diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml new file mode 100644 index 00000000000..01969d33761 --- /dev/null +++ b/.github/workflows/CI.yml @@ -0,0 +1,67 @@ +name: CI +on: + push: + tags: + - v* + branches: + - master + - 'release/**' + pull_request: + branches: + - '*' +jobs: + CI: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version: 1.18 + - name: Make OSS Build + run: make oss + - name: Make macOS build + run: make GOOS=darwin dgraph + - name: Make Windows Build + run: make GOOS=windows dgraph + - name: Make Linux Build + run: make GOOS=linux dgraph + - name: Install protobuf-compiler + run: sudo apt-get install -y protobuf-compiler + - name: Check protobuf + run: | + cd ./protos + go mod tidy + make regenerate + git diff --exit-code -- . + - name: Run unit tests + run: | + #!/bin/bash + if [ -f go.mod ]; then + export GO111MODULE=on + fi + + + # Run the Go test script. Or, run test.sh if the Go test script doesn't exist. + if [ -d ./t ]; then + #docker rmi dgraph/dgraph-lambda:latest + export GOPATH=$HOME/go + ls -alrt ~/go || true # TODO: fix later + cd t; go build . + mkdir ~/go || true # TODO: fix later + mkdir ~/go/bin || true # TODO: fix later + cp ~/work/dgraph/dgraph/dgraph ~/go/bin || true # TODO: fix later $GOPATH issue + export GOPATH=~/go + ls -alrt $GOPATH/bin + ./t -r + ./t --skip tlstest,systest/backup,systest/online-restore,systest/loader || true + else # unwanted + # Stop running containers + docker ps --filter label="cluster=test" --format "{{.Names}}" \ + | xargs -r docker stop | sed 's/^/Stopped /' + # Remove all containers + docker ps -a --filter label="cluster=test" --format "{{.Names}}" \ + | xargs -r docker rm -f | sed 's/^/Removed /' + + ./test.sh # this was the older way to run tests + fi diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml new file mode 100644 index 00000000000..54f2cbafea1 --- /dev/null +++ b/.github/workflows/golangci-lint.yml @@ -0,0 +1,26 @@ +name: golangci-lint +on: + push: + tags: + - v* + branches: + - master + - 'release/**' + pull_request: +jobs: + golangci: + name: lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: golangci-lint + env: + # prevent OOM + GOGC: 10 + uses: golangci/golangci-lint-action@v2 + with: + # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version. + version: v1.36 + only-new-issues: true + args: --timeout=10m + skip-go-installation: true diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml deleted file mode 100644 index 0254a0bfc5c..00000000000 --- a/.github/workflows/main.yml +++ /dev/null @@ -1,24 +0,0 @@ -# This is a basic workflow to help you get started with Actions - -name: Issue Closer - -# Controls when the action will run. Triggers the workflow on push or pull request -# events but only for the master branch -on: - issues: - types: [ opened, reopened ] - -# A workflow run is made up of one or more jobs that can run sequentially or in parallel -jobs: - # This workflow contains a single job called "build" - build: - # The type of runner that the job will run on - runs-on: ubuntu-latest - - # Steps represent a sequence of tasks that will be executed as part of the job - steps: - - name: Close Issue - uses: peter-evans/close-issue@v1.0.1 - with: - comment: | - **Use [Discuss Issues](https://discuss.dgraph.io/c/issues/dgraph/38) for reporting issues about this repository.** diff --git a/.gitignore b/.gitignore index a76b34bdaf6..ce968359141 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,12 @@ /dgraph-bulk-loader /osx-docker-gopath +# test +t/test*log + +# secrets +compose/hmac_secret_file + # fuzzing output gql/gql-fuzz.zip gql/fuzz-data/crashers @@ -33,3 +39,4 @@ dgraph.iml .DS_Store vendor +.minio.sys diff --git a/CHANGELOG.md b/CHANGELOG.md index 9fd9c9e46d3..3e515f0265a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,1535 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project will adhere to [Calendar Versioning](https://calver.org/) starting v20.03. +## [21.12.0] - 2021-11-30 +[21.12.0]: https://github.com/dgraph-io/dgraph/compare/v21.03.0...v21.12.0 + +### Changed + +- [BREAKING] feat(sroar): Bring sroar to Dgraph ([#7840][]) +- [BREAKING] Return error for illegal math operations. ([#7631][]) +- [BREAKING] feat: bring dgraph-lambda to dgraph, alpha launches lambda server ([#7973][]) +- [BREAKING] fix json marshal unmarshal for namespace > 127 ([#7810][]) +- [BREAKING] fix(usability): make force-namespace flag compulsory in live loader for galaxy user ([#7731][]) +- [BREAKING] perf(Transactions): Run transactions concurrently ([#7694][]) +- [BREAKING] feat(flags): expand badger to accept all valid options ([#7677][]) +- [BREAKING] fix(commit): make txn context more robust ([#7659][]) +- [BREAKING] Opt(Restore): Optimize Restore's new map-reduce based design ([#7666][]) +- [BREAKING] fix(metrics): Rename Badger metrics. ([#7507][]) +- [BREAKING] Make backup-restore an open source feature ([#8067][]) + +### Added + +- GraphQL + - Feat(GRAPHQL): adds `@default` directive for setting default field values at create and update ([#8017][]) + - Feat(GRAPHQL): Support auth with custom DQL ([#7775][]) + - Feat(GRAPHQL): This PR allows updatable and nullable `@id` fields. ([#7736][]) + - Feat(GRAPHQL): Disallow DQL schema changes for predicates used in GraphQL schema (DGRAPH-3245) ([#7742][]) + - Feat(GRAPHQL): This PR allows `@id` field in interface to be unique across all the implementing types. ([#7710][]) + - Feat(GRAPHQL): Add language tag support in GraphQL ([#7663][]) + - Feat(GRAPHQL): Zero HTTP endpoints are now available at GraphQL admin (GRAPHQL-1118) ([#6649][]) + - Feat(GRAPHQL): Webhooks on add/update/delete mutations (GRAPHQL-1045) ([#7494][]) + - Feat(GRAPHQL): Allow Multipe JWKUrls for auth. ([#7528][]) + - Feat(GRAPHQL): Add support for passing OAuth Bearer token as authorization JWT ([#7490][]) + +- Core Dgraph + - Feat(metrics): Add Badger metrics. ([#8034][]) + - Feat(magicNumber): Introduce magic number ([#8032][]) + - Feat(lambda): allow access to access jwt in lambda ([#8023][]) + - Feat(rdf-response): Support RDF response via http query request ([#8004][]) + - Feat(sroar): Use rank() API from sroar and some cleanup ([#8002][]) + - Feat(lambda): store lambda scripts within the dgraph ([#7955][]) + - Feat(query): handle extend keyword for Queries and Mutations ([#7916][]) + - Feat(Backup): Add native google cloud storage backup support ([#7829][]) + - Feat(Backup): Add native support for backup to Azure. ([#7843][]) + - Feat(cloud): add shared-instance flag in limit superflag in alpha ([#7770][]) + - Feat(Dgraph): Add task queue implementation ([#7716][]) + - Feat(DQL): `@groupby` on scalar fields and count duplicate ([#7746][]) + - Feat(Query): Add random keyword in DQL ([#7693][]) + - Feat(tool): Neo4j CSV to RDF Converter ([#7545][]) + - Feat(query): Add mechanism to have a limit on number of pending queries ([#7603][]) + - Feat(flag): remove unused badger.max-retries option from bulk command ([#7591][]) + - Feat(sentry): clusterID flag added for alpha sentry reports (gql-services) ([#7580][]) + - Feat(cmd/debuginfo) add new metrics to be collected ([#7439][]) + - Feat(flags): use Vault for ACL secrets ([#7492][]) + - Feat(Apollo): Add support for `@provides` and `@requires` directive. ([#7503][]) + - Feat(restore): Introduce incremental restore ([#7942][]) ([#7971][]) + - Feat(schema): do schema versioning and make backup non-blocking for indexing ([#7852][]) + - Feat(zero bulk): adding bulk call for alpha to inform zero about the tablets ([#8100][]) + - Feat(cdc): Add superflag to enable TLS without CA or certs. ([#8097][]) + +- Enterprise Features + - Feat(Multi-tenancy): Add namespaces field to state ([#7808][]) + - Feat(multi-tenancy): make drop data namespace aware ([#7789][]) ([#7795][]) + - Feat(cdc): Add support for SCRAM SASL mechanism ([#7765][]) + - Feat(acl): allow access to all the predicates using wildcard ([#7991][]) + - Feat(cdc): Add superflag to enable TLS without CA or certs. ([#7946][]) + +### Fixed + +- GraphQL + - Fix(GRAPHQL): add validation of null values with correct order of graphql rule validation ([#8007][]) ([#8008][]) + - Fix(GRAPHQL): fix type assertion failure in graphql if resolver is not defined ([#8003][]) + - Fix(GRAPHQL): fixing graphql schema update when the data is restored ([#7970][]) + - Fix(GRAPHQL): Nested Auth Rules not working properly. ([#7915][]) + - Fix(GRAPHQL): optimize eq filter queries ([#7895][]) + - Fix(GRAPHQL): Fix duplicate XID error in case of interface XIDs ([#7776][]) + - Fix(GRAPHQL): Pass on HTTP request headers for subscriptions ([#7806][]) + - Fix(GRAPHQL): Make mutation rewriting tests more robust ([#7768][]) + - Fix(GRAPHQL): Fix error message of lambdaOnMutate directive ([#7751][]) + - Fix(GRAPHQL): fix auth query rewriting with ID filter ([#7740][]) + - Fix(GRAPHQL): Add filter in DQL query in case of reverse predicate ([#7728][]) + - Fix(GRAPHQL): Fix GraphQL encoding in case of empty list ([#7726][]) + - Fix(GRAPHQL): fix `@cascade` with Pagination for `@auth` queries. ([#7695][]) + - Fix(GRAPHQL): Fix Execution Trace for Add and Update Mutations ([#7656][]) + - Fix(GRAPHQL): Log query along with the panic ([#7638][]) + - Fix(GRAPHQL): Add error handling for unrecognized args to generate directive. ([#7612][]) + - Fix(GRAPHQL): Fix panic when no schema exists for a new namespace ([#7630][]) + - Fix(GRAPHQL): fixed output coercing for admin fields. ([#7617][]) + - Fix(GRAPHQL): fix lambda querying a lambda field in case of no data. ([#7610][]) + - Fix(GRAPHQL): Add extra checks for deleting UpdateTypeInput ([#7595][]) + - Fix(GRAPHQL): remove support of `@id` directive on Float ([#7583][]) + - Fix(GRAPHQL): Fix mutation with Int Xid variables. ([#7565][]) + - Fix(GRAPHQL): Fix custom(dql: ...) with __typename (GRAPHQL-1098) ([#7569][]) + - Fix(GRAPHQL): Change variable name generation for interface auth rules ([#7559][]) + - Fix(GRAPHQL): Apollo federation now works with lambda (GRAPHQL-1084) ([#7558][]) + - Fix(GRAPHQL): fix empty remove in update mutation patch, that remove all the data for nodes in filter. ([#7563][]) + - Fix(GRAPHQL): fix order of entities query result ([#7542][]) + - Fix(GRAPHQL): Change variable name generation from Type to Type_ ([#7556][]) + - Fix(GRAPHQL): fix duplicate xid error for multiple xid fields. ([#7546][]) + - Fix(GRAPHQL): Added support for exact index on field having `@id` directive. ([#7534][]) ([#7551][]) + - Fix(GRAPHQL): fix query rewriting for multiple order on nested field. ([#7523][]) + - Fix(GRAPHQL) fix empty `type Query` with single extended type definition in the schema. ([#7517][]) + +- Core Dgraph + - Fix(sort): Only filter out nodes with positive offsets. ([#8077][]) + - Fix(fragment): merge the nested fragments fields ([#8075][]) + - Fix(lambda): upgrade lambda dependencies to fix vulnerabilities ([#8074][]) + - Fix(magic): fix the magic version in bulk loader etc ([#8070][]) + - Fix(split): enable split of posting list with single plist ([#8062][]) + - Fix(restore): Do not retry restore proposal ([#8058][]) + - Fix(txn): Fix data races in transaction code ([#8060][]) + - Fix(shutdown): wait for pending queries to process on alpha shutdown ([#8057][]) + - Fix(restore-test): Make offline restore use separate map directory for each group ([#8047][]) + - Fix(lambda-logs): extract namespace from body.namespace ([#8043][]) + - Fix(lambda): make lambda active only after successful start ([#8036][]) + - Fix(probe): do not contend for lock in lazy load ([#8037][]) + - Fix(lambda): shutdown node processes when alpha gets killed ([#8027][]) + - Fix(snapshot): Fix snapshot calculation after restore ([#8024][]) + - Fix(badger): Upgrade badger to fix deadlock ([#8025][]) + - Fix(bulk): quote the schema correctly in bulk loader ([#8019][]) + - Fix(sbs): handle response error gracefully ([#8018][]) + - Fix(admin): make config changes to pass through gog middlewares ([#8014][]) + - Fix(lambda): fix race condition in lambda server spin up ([#8013][]) + - Fix(proposals): Incremental proposal key for zero proposals ([#8005][]) + - Fix(lambda): monitor lambda server, fix performance issue, remove lambda logs from extensions ([#8006][]) + - Fix(live): quote the xid when doing upsert ([#7983][]) + - Fix(sroar): Bring latest sroar to master ([#7977][]) + - Fix(query): Do not execute filters if there are no source uids ([#7962][]) ([#7969][]) + - Fix(snapshot): update last snapshot time across members ([#7968][]) + - Fix(pool): use write lock when getting health info ([#7963][]) + - Fix(JoinCluster): Avoid retrying JoinCluster indefinitely ([#7961][]) + - Fix(rollups): Write rolled-up keys at ts+1 ([#7957][]) ([#7959][]) + - Fix(conn): JoinCluster loop should use latest conn ([#7950][]) + - Fix(restore): Set kv version to restoreTs for rolled up keys and schema keys ([#7930][]) ([#7935][]) + - Fix(backup): Fix full backup request ([#7932][]) + - Fix(cmd/debug): Print banned namespaces correctly. ([#7929][]) + - Reconnect via a redial in case of disconnection. ([#7918][]) + - Fix(metrics): Expose dgraph_num_backups_failed_total metric view. ([#7900][]) + - Fix(sroar): Fix TestAuthWithCustomDQL failure because of roaring bitmaps ([#7902][]) + - Fix(DQL): revert changes related to cascade pagination with sort ([#7885][]) + - Fix(restore): append galaxy namespace to type name ([#7880][]) + - Fix(Backup): use validReadTs from manifest for backward compatibility ([#7601][]) ([#7863][]) + - fix the predicate move ([#7862][]) + - Fix(restore): consider the banned namespaces while bumping ([#7839][]) + - Fix(restore): update the schema and type from 2103 ([#7838][]) + - Fix(updatemanifest): update the version of manifest after update ([#7828][]) + - Fix(backup): handle manifest version logic, update manifest version to 2105 ([#7825][]) + - Fix(schema-update): Start opIndexing only when index creation is required. ([#7845][]) + - Fix(admin): remove exportedFiles field ([#7835][]) + - Fix(restore): reset the kv.StreamId before sending to stream writer ([#7833][]) + - Fix(auth): preserve the status code while returning error ([#7832][]) + - bug fix to permit audit streaming to stdout writer([#7803][]) + - Fix(lease): don't do rate limiting when not limit is not specified ([#7787][]) ([#7801][]) + - Fix(restore): Bump uid and namespace after restore ([#7790][]) + - Fix(txn): ensure that txn hash is set ([#7782][]) + - Fix(export-backup): Fix double free in export backup ([#7780][]) + - Fix(Dgraph): Forward task status requests to correct Alpha ([#7774][]) + - Fix(bulk): upsert guardian/groot for all existing namespaces ([#7759][]) + - Fix(export): Fix facet export of reference type postings to JSON format ([#7744][]) + - fix: Prevent proposal from being dropped accidentally ([#7741][]) + - Fix(live): make live loader progress on a cluster with very high maxUid ([#7743][]) + - Fix(Chunker): don't delete node with empty facet in mutation ([#7737][]) + - Fix(DQL): optimize query for has function with offset. ([#7727][]) + - fixing readme for dgraph after latest release ([#7732][]) + - Fix(lease): prevent ID lease overflow ([#7724][]) + - Fix(lsbackup): Fix profiler in lsBackup ([#7729][]) + - Fix(bulk): throw the error instead of crashing ([#7722][]) + - Fix(ee): GetKeys should return an error ([#7713][]) + - Fix(raftwal): take snapshot after restore ([#7719][]) + - Fix(pagination): Fix after for regexp, match functions ([#7700][]) + - Fix(query): Prevent multiple entries for same predicate/type in schema mutations. ([#7715][]) + - Fix(vault): Hide ACL flags when not required ([#7701][]) + - Fix(flag): fix bulk loader flag and remove flag parsing from critical path ([#7679][]) + - Fix(upgrade): make upgrade tool to work with non-acl cluster ([#7674][]) + - Fix(query): Fix pagination with match functions ([#7668][]) + - Fix(postingList): Acquire lock before reading the cached posting list ([#7632][]) + - Fix(zero): add a ratelimiter to limit the uid lease per namespace ([#7568][]) + - Fix(export): use UriHandler for exports ([#7690][]) + - Fix s3 backup copy ([#7669][]) + - return if no uids exist in queries for Geo ([#7651][]) + - Fix(DGRAPH): fix `@normalize` response when multiple fields at different levels with same alias are selected. ([#7639][]) + - Fix(/commit): protect the commit endpoint via acl ([#7608][]) + - Use GetString for vault path ([#7605][]) + - Fix query logging for mutations ([#7646][]) + - Fix(login): fix login based on refresh token logic ([#7637][]) + - Fix(Query): Fix cascade pagination with 0 offset. ([#7636][]) + - feat(flags): Add query timeout as a limit config ([#7599][]) + - Fix(flags): Add empty defaults to Vault superflag ([#7598][]) + - Fix(persistent): make persistent query namespace aware ([#7570][]) + - Fix(rollups): Fix splits in roll-up ([#7609][]) + - fix for xgo version to use ([#7620][]) + - Fix(flags): Expose global flags to dgraph subcommands. ([#7530][]) + - Fix(telemetry): fix zero crash due to telemetry ([#7575][]) + - Fix(telemetry): Track enterprise feature usage ([#7495][]) + - Fix(release): update support for xgo tool ([#7576][]) + - Fix(super-flags): Use GetPath for path arguments in superflags ([#7541][]) + - Fix(dql): Fix error message in case of wrong argument to val() ([#7543][]) + - Fix(Roaring): Remove pack from posting list ([#7535][]) + - Fix(Flags): immediately panic on SuperFlag user errors ([#7529][]) + - Fix(Rollups): Don't try splitting a posting list with cardinality less than 2. ([#7525][]) + - Fix(export): fix namespace parameter in export ([#7524][]) + - Fix(live): fix usage of force-namespace parameter in export ([#7526][]) + - Fix largeSchemaUpdate test ([#7522][]) + - Fix(Configs): Allow hierarchical notation in JSON/YAML configs ([#7498][]) + - Fix(Bulk): Remove stale allocator in reduce ([#7510][]) + - Fix upsert mutations ([#7515][]) + - Fix(standalone): Set whitelist flag using superflag. ([#7512][]) + - Fix(admin-endpoints): Error out if the request is rejected by the server ([#7511][]) + - Fix(Dgraph): Throttle number of files to open while schema update ([#7480][]) + - Fix(metrics): Expose Badger LSM and vlog size bytes. ([#7488][]) + - Fix(schema): log error instead of panic if schema not found for predicate ([#7502][]) + - Fix(tool): Don't ban namespace in export_backup ([#8099][]) + - Fix(state): fix hex to uint64 response of list of namespaces ([#8101][]) + - Fix(restore): return nil if there is error ([#8098][]) + +- Enterprise Features + - Fix(audit): fixing audit logs for websocket connections ([#8048][]) + - Fix(acl): subscribe for the correct predicates ([#7992][]) + - Fix(acl): filter out the results based on type ([#7978][]) ([#7980][]) + - Fix(groot): do not upsert groot for all namespaces on restart ([#7917][]) + - Fix(cdc): Show namespace info in event meta ([#7721][]) + - Fix(learner): Don't start a learner node with no peers ([#7582][]) + - Fix(audit): logs not getting deleted after N days ([#7567][]) + - Fix(release/v21.03) - Use worker.GetEEFeatureList instead of ee.GetEEFeatureList ([#7564][]) + - Fix(multi-tenancy): Format namespace to human readable form ([#7552][]) + - Fix(learner nodes): Reconnect to learner nodes after restart ([#7554][]) + - Fix(multi-tenancy): fix live loader for case when namespace does not exist for data ([#7505][]) + +### Performance: + +- Opt(schema): Optimize populateSchema() by avoiding repeated lock acquisition ([#8068][]) +- Perf: Speed up parsing of a huge query with a lot of conditional mutations ([#7871][]) +- Opt(Restore): Make restore map phase faster ([#8038][]) +- Opt(codec): return nil instead of a new bitmap ([#7997][]) +- Opt(cache): Use Ristretto to store posting lists ([#7995][]) +- Opt(rdf-output): Make RDF output generation concurrent ([#7988][]) +- Opt(recurse): Optimise recurse and bring range iterators from sroar ([#7989][]) +- Opt(restore): Sort the buffer before spinning the writeToDisk goroutine ([#7984][]) +- Perf(sroar): Use latest sroar and add histogram in the sbs tool ([#7982][]) +- Opt(Alpha): Load schema and types using Stream framework ([#7938][]) ([#7940][]) +- Opt(query): Use sroar in pb.List ([#7864][]) +- Opt(snapshot): use full table copy when streaming the entire data ([#7870][]) +- Opt(snapshot): Optimize snapshot by using sinceTs ([#7826][]) +- Opt(predMove): iterate Phase I till there is major data to move ([#7792][]) +- Opt(dropPrefix): allow logical drop for deleting predicates and indexing ([#7764][]) +- Opt(txn commits): Optimize txns by passing Skiplists to Badger ([#7777][]) +- Opt(GraphQL): filter existence queries on GraphQL side instead of using `@filter(type)` ([#7757][]) +- Opt(predMove): hot tablet move ([#7703][]) +- Opt(Backup): Make backups faster ([#7680][]) +- Perf(restore): Implement map-reduce based restore ([#7664][]) +- Opt(reindex): do not try building indices when inserting a new predicate ([#7109][]) +- Perf(txn): de-duplicate the context keys and predicates ([#7478][]) +- perf(rollup): use NSplit API from sroar to improve rollup performance ([#8092][]) + +[#7957]: https://github.com/dgraph-io/dgraph/issues/7957 +[#7978]: https://github.com/dgraph-io/dgraph/issues/7978 +[#7938]: https://github.com/dgraph-io/dgraph/issues/7938 +[#8099]: https://github.com/dgraph-io/dgraph/issues/8099 +[#8101]: https://github.com/dgraph-io/dgraph/issues/8101 +[#8100]: https://github.com/dgraph-io/dgraph/issues/8100 +[#8097]: https://github.com/dgraph-io/dgraph/issues/8097 +[#8098]: https://github.com/dgraph-io/dgraph/issues/8098 +[#7946]: https://github.com/dgraph-io/dgraph/issues/7946 +[#7942]: https://github.com/dgraph-io/dgraph/issues/7942 +[#7490]: https://github.com/dgraph-io/dgraph/issues/7490 +[#7789]: https://github.com/dgraph-io/dgraph/issues/7789 +[#8007]: https://github.com/dgraph-io/dgraph/issues/8007 +[#7534]: https://github.com/dgraph-io/dgraph/issues/7534 +[#7787]: https://github.com/dgraph-io/dgraph/issues/7787 +[#7601]: https://github.com/dgraph-io/dgraph/issues/7601 +[#7930]: https://github.com/dgraph-io/dgraph/issues/7930 +[#7962]: https://github.com/dgraph-io/dgraph/issues/7962 +[#7840]: https://github.com/dgraph-io/dgraph/issues/7840 +[#7631]: https://github.com/dgraph-io/dgraph/issues/7631 +[#7973]: https://github.com/dgraph-io/dgraph/issues/7973 +[#7810]: https://github.com/dgraph-io/dgraph/issues/7810 +[#7731]: https://github.com/dgraph-io/dgraph/issues/7731 +[#7694]: https://github.com/dgraph-io/dgraph/issues/7694 +[#7677]: https://github.com/dgraph-io/dgraph/issues/7677 +[#7659]: https://github.com/dgraph-io/dgraph/issues/7659 +[#7666]: https://github.com/dgraph-io/dgraph/issues/7666 +[#7507]: https://github.com/dgraph-io/dgraph/issues/7507 +[#8067]: https://github.com/dgraph-io/dgraph/issues/8067 +[#8017]: https://github.com/dgraph-io/dgraph/issues/8017 +[#7775]: https://github.com/dgraph-io/dgraph/issues/7775 +[#7736]: https://github.com/dgraph-io/dgraph/issues/7736 +[#7742]: https://github.com/dgraph-io/dgraph/issues/7742 +[#7710]: https://github.com/dgraph-io/dgraph/issues/7710 +[#7663]: https://github.com/dgraph-io/dgraph/issues/7663 +[#6649]: https://github.com/dgraph-io/dgraph/issues/6649 +[#7494]: https://github.com/dgraph-io/dgraph/issues/7494 +[#7528]: https://github.com/dgraph-io/dgraph/issues/7528 +[#8034]: https://github.com/dgraph-io/dgraph/issues/8034 +[#8032]: https://github.com/dgraph-io/dgraph/issues/8032 +[#8023]: https://github.com/dgraph-io/dgraph/issues/8023 +[#8004]: https://github.com/dgraph-io/dgraph/issues/8004 +[#8002]: https://github.com/dgraph-io/dgraph/issues/8002 +[#7955]: https://github.com/dgraph-io/dgraph/issues/7955 +[#7916]: https://github.com/dgraph-io/dgraph/issues/7916 +[#7829]: https://github.com/dgraph-io/dgraph/issues/7829 +[#7843]: https://github.com/dgraph-io/dgraph/issues/7843 +[#7770]: https://github.com/dgraph-io/dgraph/issues/7770 +[#7716]: https://github.com/dgraph-io/dgraph/issues/7716 +[#7746]: https://github.com/dgraph-io/dgraph/issues/7746 +[#7693]: https://github.com/dgraph-io/dgraph/issues/7693 +[#7545]: https://github.com/dgraph-io/dgraph/issues/7545 +[#7603]: https://github.com/dgraph-io/dgraph/issues/7603 +[#7591]: https://github.com/dgraph-io/dgraph/issues/7591 +[#7580]: https://github.com/dgraph-io/dgraph/issues/7580 +[#7439]: https://github.com/dgraph-io/dgraph/issues/7439 +[#7492]: https://github.com/dgraph-io/dgraph/issues/7492 +[#7503]: https://github.com/dgraph-io/dgraph/issues/7503 +[#7971]: https://github.com/dgraph-io/dgraph/issues/7971 +[#7852]: https://github.com/dgraph-io/dgraph/issues/7852 +[#7808]: https://github.com/dgraph-io/dgraph/issues/7808 +[#7795]: https://github.com/dgraph-io/dgraph/issues/7795 +[#7765]: https://github.com/dgraph-io/dgraph/issues/7765 +[#7991]: https://github.com/dgraph-io/dgraph/issues/7991 +[#8008]: https://github.com/dgraph-io/dgraph/issues/8008 +[#8003]: https://github.com/dgraph-io/dgraph/issues/8003 +[#7970]: https://github.com/dgraph-io/dgraph/issues/7970 +[#7915]: https://github.com/dgraph-io/dgraph/issues/7915 +[#7895]: https://github.com/dgraph-io/dgraph/issues/7895 +[#7776]: https://github.com/dgraph-io/dgraph/issues/7776 +[#7806]: https://github.com/dgraph-io/dgraph/issues/7806 +[#7768]: https://github.com/dgraph-io/dgraph/issues/7768 +[#7751]: https://github.com/dgraph-io/dgraph/issues/7751 +[#7740]: https://github.com/dgraph-io/dgraph/issues/7740 +[#7728]: https://github.com/dgraph-io/dgraph/issues/7728 +[#7726]: https://github.com/dgraph-io/dgraph/issues/7726 +[#7695]: https://github.com/dgraph-io/dgraph/issues/7695 +[#7656]: https://github.com/dgraph-io/dgraph/issues/7656 +[#7638]: https://github.com/dgraph-io/dgraph/issues/7638 +[#7612]: https://github.com/dgraph-io/dgraph/issues/7612 +[#7630]: https://github.com/dgraph-io/dgraph/issues/7630 +[#7617]: https://github.com/dgraph-io/dgraph/issues/7617 +[#7610]: https://github.com/dgraph-io/dgraph/issues/7610 +[#7595]: https://github.com/dgraph-io/dgraph/issues/7595 +[#7583]: https://github.com/dgraph-io/dgraph/issues/7583 +[#7565]: https://github.com/dgraph-io/dgraph/issues/7565 +[#7569]: https://github.com/dgraph-io/dgraph/issues/7569 +[#7559]: https://github.com/dgraph-io/dgraph/issues/7559 +[#7558]: https://github.com/dgraph-io/dgraph/issues/7558 +[#7563]: https://github.com/dgraph-io/dgraph/issues/7563 +[#7542]: https://github.com/dgraph-io/dgraph/issues/7542 +[#7556]: https://github.com/dgraph-io/dgraph/issues/7556 +[#7546]: https://github.com/dgraph-io/dgraph/issues/7546 +[#7551]: https://github.com/dgraph-io/dgraph/issues/7551 +[#7523]: https://github.com/dgraph-io/dgraph/issues/7523 +[#7517]: https://github.com/dgraph-io/dgraph/issues/7517 +[#8077]: https://github.com/dgraph-io/dgraph/issues/8077 +[#8075]: https://github.com/dgraph-io/dgraph/issues/8075 +[#8074]: https://github.com/dgraph-io/dgraph/issues/8074 +[#8070]: https://github.com/dgraph-io/dgraph/issues/8070 +[#8062]: https://github.com/dgraph-io/dgraph/issues/8062 +[#8058]: https://github.com/dgraph-io/dgraph/issues/8058 +[#8060]: https://github.com/dgraph-io/dgraph/issues/8060 +[#8057]: https://github.com/dgraph-io/dgraph/issues/8057 +[#8047]: https://github.com/dgraph-io/dgraph/issues/8047 +[#8043]: https://github.com/dgraph-io/dgraph/issues/8043 +[#8036]: https://github.com/dgraph-io/dgraph/issues/8036 +[#8037]: https://github.com/dgraph-io/dgraph/issues/8037 +[#8027]: https://github.com/dgraph-io/dgraph/issues/8027 +[#8024]: https://github.com/dgraph-io/dgraph/issues/8024 +[#8025]: https://github.com/dgraph-io/dgraph/issues/8025 +[#8019]: https://github.com/dgraph-io/dgraph/issues/8019 +[#8018]: https://github.com/dgraph-io/dgraph/issues/8018 +[#8014]: https://github.com/dgraph-io/dgraph/issues/8014 +[#8013]: https://github.com/dgraph-io/dgraph/issues/8013 +[#8005]: https://github.com/dgraph-io/dgraph/issues/8005 +[#8006]: https://github.com/dgraph-io/dgraph/issues/8006 +[#7983]: https://github.com/dgraph-io/dgraph/issues/7983 +[#7977]: https://github.com/dgraph-io/dgraph/issues/7977 +[#7969]: https://github.com/dgraph-io/dgraph/issues/7969 +[#7968]: https://github.com/dgraph-io/dgraph/issues/7968 +[#7963]: https://github.com/dgraph-io/dgraph/issues/7963 +[#7961]: https://github.com/dgraph-io/dgraph/issues/7961 +[#7959]: https://github.com/dgraph-io/dgraph/issues/7959 +[#7950]: https://github.com/dgraph-io/dgraph/issues/7950 +[#7935]: https://github.com/dgraph-io/dgraph/issues/7935 +[#7932]: https://github.com/dgraph-io/dgraph/issues/7932 +[#7929]: https://github.com/dgraph-io/dgraph/issues/7929 +[#7918]: https://github.com/dgraph-io/dgraph/issues/7918 +[#7900]: https://github.com/dgraph-io/dgraph/issues/7900 +[#7902]: https://github.com/dgraph-io/dgraph/issues/7902 +[#7885]: https://github.com/dgraph-io/dgraph/issues/7885 +[#7880]: https://github.com/dgraph-io/dgraph/issues/7880 +[#7863]: https://github.com/dgraph-io/dgraph/issues/7863 +[#7862]: https://github.com/dgraph-io/dgraph/issues/7862 +[#7839]: https://github.com/dgraph-io/dgraph/issues/7839 +[#7838]: https://github.com/dgraph-io/dgraph/issues/7838 +[#7828]: https://github.com/dgraph-io/dgraph/issues/7828 +[#7825]: https://github.com/dgraph-io/dgraph/issues/7825 +[#7845]: https://github.com/dgraph-io/dgraph/issues/7845 +[#7835]: https://github.com/dgraph-io/dgraph/issues/7835 +[#7833]: https://github.com/dgraph-io/dgraph/issues/7833 +[#7832]: https://github.com/dgraph-io/dgraph/issues/7832 +[#7803]: https://github.com/dgraph-io/dgraph/issues/7803 +[#7801]: https://github.com/dgraph-io/dgraph/issues/7801 +[#7790]: https://github.com/dgraph-io/dgraph/issues/7790 +[#7782]: https://github.com/dgraph-io/dgraph/issues/7782 +[#7780]: https://github.com/dgraph-io/dgraph/issues/7780 +[#7774]: https://github.com/dgraph-io/dgraph/issues/7774 +[#7759]: https://github.com/dgraph-io/dgraph/issues/7759 +[#7744]: https://github.com/dgraph-io/dgraph/issues/7744 +[#7741]: https://github.com/dgraph-io/dgraph/issues/7741 +[#7743]: https://github.com/dgraph-io/dgraph/issues/7743 +[#7737]: https://github.com/dgraph-io/dgraph/issues/7737 +[#7727]: https://github.com/dgraph-io/dgraph/issues/7727 +[#7732]: https://github.com/dgraph-io/dgraph/issues/7732 +[#7724]: https://github.com/dgraph-io/dgraph/issues/7724 +[#7729]: https://github.com/dgraph-io/dgraph/issues/7729 +[#7722]: https://github.com/dgraph-io/dgraph/issues/7722 +[#7713]: https://github.com/dgraph-io/dgraph/issues/7713 +[#7719]: https://github.com/dgraph-io/dgraph/issues/7719 +[#7700]: https://github.com/dgraph-io/dgraph/issues/7700 +[#7715]: https://github.com/dgraph-io/dgraph/issues/7715 +[#7701]: https://github.com/dgraph-io/dgraph/issues/7701 +[#7679]: https://github.com/dgraph-io/dgraph/issues/7679 +[#7674]: https://github.com/dgraph-io/dgraph/issues/7674 +[#7668]: https://github.com/dgraph-io/dgraph/issues/7668 +[#7632]: https://github.com/dgraph-io/dgraph/issues/7632 +[#7568]: https://github.com/dgraph-io/dgraph/issues/7568 +[#7690]: https://github.com/dgraph-io/dgraph/issues/7690 +[#7669]: https://github.com/dgraph-io/dgraph/issues/7669 +[#7651]: https://github.com/dgraph-io/dgraph/issues/7651 +[#7639]: https://github.com/dgraph-io/dgraph/issues/7639 +[#7608]: https://github.com/dgraph-io/dgraph/issues/7608 +[#7605]: https://github.com/dgraph-io/dgraph/issues/7605 +[#7646]: https://github.com/dgraph-io/dgraph/issues/7646 +[#7637]: https://github.com/dgraph-io/dgraph/issues/7637 +[#7636]: https://github.com/dgraph-io/dgraph/issues/7636 +[#7599]: https://github.com/dgraph-io/dgraph/issues/7599 +[#7598]: https://github.com/dgraph-io/dgraph/issues/7598 +[#7570]: https://github.com/dgraph-io/dgraph/issues/7570 +[#7609]: https://github.com/dgraph-io/dgraph/issues/7609 +[#7620]: https://github.com/dgraph-io/dgraph/issues/7620 +[#7530]: https://github.com/dgraph-io/dgraph/issues/7530 +[#7575]: https://github.com/dgraph-io/dgraph/issues/7575 +[#7495]: https://github.com/dgraph-io/dgraph/issues/7495 +[#7576]: https://github.com/dgraph-io/dgraph/issues/7576 +[#7541]: https://github.com/dgraph-io/dgraph/issues/7541 +[#7543]: https://github.com/dgraph-io/dgraph/issues/7543 +[#7535]: https://github.com/dgraph-io/dgraph/issues/7535 +[#7529]: https://github.com/dgraph-io/dgraph/issues/7529 +[#7525]: https://github.com/dgraph-io/dgraph/issues/7525 +[#7524]: https://github.com/dgraph-io/dgraph/issues/7524 +[#7526]: https://github.com/dgraph-io/dgraph/issues/7526 +[#7522]: https://github.com/dgraph-io/dgraph/issues/7522 +[#7498]: https://github.com/dgraph-io/dgraph/issues/7498 +[#7510]: https://github.com/dgraph-io/dgraph/issues/7510 +[#7515]: https://github.com/dgraph-io/dgraph/issues/7515 +[#7512]: https://github.com/dgraph-io/dgraph/issues/7512 +[#7511]: https://github.com/dgraph-io/dgraph/issues/7511 +[#7480]: https://github.com/dgraph-io/dgraph/issues/7480 +[#7488]: https://github.com/dgraph-io/dgraph/issues/7488 +[#7502]: https://github.com/dgraph-io/dgraph/issues/7502 +[#8048]: https://github.com/dgraph-io/dgraph/issues/8048 +[#7992]: https://github.com/dgraph-io/dgraph/issues/7992 +[#7980]: https://github.com/dgraph-io/dgraph/issues/7980 +[#7917]: https://github.com/dgraph-io/dgraph/issues/7917 +[#7721]: https://github.com/dgraph-io/dgraph/issues/7721 +[#7582]: https://github.com/dgraph-io/dgraph/issues/7582 +[#7567]: https://github.com/dgraph-io/dgraph/issues/7567 +[#7564]: https://github.com/dgraph-io/dgraph/issues/7564 +[#7552]: https://github.com/dgraph-io/dgraph/issues/7552 +[#7554]: https://github.com/dgraph-io/dgraph/issues/7554 +[#7505]: https://github.com/dgraph-io/dgraph/issues/7505 +[#8068]: https://github.com/dgraph-io/dgraph/issues/8068 +[#7871]: https://github.com/dgraph-io/dgraph/issues/7871 +[#8038]: https://github.com/dgraph-io/dgraph/issues/8038 +[#7997]: https://github.com/dgraph-io/dgraph/issues/7997 +[#7995]: https://github.com/dgraph-io/dgraph/issues/7995 +[#7988]: https://github.com/dgraph-io/dgraph/issues/7988 +[#7989]: https://github.com/dgraph-io/dgraph/issues/7989 +[#7984]: https://github.com/dgraph-io/dgraph/issues/7984 +[#7982]: https://github.com/dgraph-io/dgraph/issues/7982 +[#7940]: https://github.com/dgraph-io/dgraph/issues/7940 +[#7864]: https://github.com/dgraph-io/dgraph/issues/7864 +[#7870]: https://github.com/dgraph-io/dgraph/issues/7870 +[#7826]: https://github.com/dgraph-io/dgraph/issues/7826 +[#7792]: https://github.com/dgraph-io/dgraph/issues/7792 +[#7764]: https://github.com/dgraph-io/dgraph/issues/7764 +[#7777]: https://github.com/dgraph-io/dgraph/issues/7777 +[#7757]: https://github.com/dgraph-io/dgraph/issues/7757 +[#7703]: https://github.com/dgraph-io/dgraph/issues/7703 +[#7680]: https://github.com/dgraph-io/dgraph/issues/7680 +[#7664]: https://github.com/dgraph-io/dgraph/issues/7664 +[#7109]: https://github.com/dgraph-io/dgraph/issues/7109 +[#7478]: https://github.com/dgraph-io/dgraph/issues/7478 +[#8092]: https://github.com/dgraph-io/dgraph/issues/8092 + +## [21.03.2] - 2021-08-26 +[21.03.2]: https://github.com/dgraph-io/dgraph/compare/v21.03.1...v21.03.2 + +### Fixed + +- GraphQL + - Handle extend keyword for Queries and Mutations ([#7923][]) + +- Core Dgraph + - fix(Raft): Detect network partition when streaming ([#7908][]) + - fix(Raft): Reconnect via a redial in case of disconnection. ([#7921][]) + - fix(conn): JoinCluster loop should use latest conn ([#7952][]) + - fix(pool): use write lock when getting health info ([#7967][]) + - fix(acl): The Acl cache should be updated on restart and restore. ([#7964][]) + - fix(acl): filter out the results based on type ([#7981][]) + - fix(backup): Fix full backup request ([#7934][]) + - fix(live): quote the xid when doing upsert ([#7999][]) + - fix(export): Write temporary files for export to the t directory. ([#7998][]) + +### Changed + +- protobuf: upgrade golang/protobuf library v1.4.1 -> v1.5.2 ([#7949][]) +- chore(raft): Log packets message less frequently. ([#7913][]) + +### Added + +- feat(acl): allow access to all the predicates using wildcard. ([#7993][]) +- feat(Multi-tenancy): Add namespaces field to state. ([#7936][]) + +[#7923]: https://github.com/dgraph-io/dgraph/issues/7923 +[#7908]: https://github.com/dgraph-io/dgraph/issues/7908 +[#7921]: https://github.com/dgraph-io/dgraph/issues/7921 +[#7952]: https://github.com/dgraph-io/dgraph/issues/7952 +[#7967]: https://github.com/dgraph-io/dgraph/issues/7967 +[#7964]: https://github.com/dgraph-io/dgraph/issues/7964 +[#7981]: https://github.com/dgraph-io/dgraph/issues/7981 +[#7934]: https://github.com/dgraph-io/dgraph/issues/7934 +[#7999]: https://github.com/dgraph-io/dgraph/issues/7999 +[#7998]: https://github.com/dgraph-io/dgraph/issues/7998 +[#7949]: https://github.com/dgraph-io/dgraph/issues/7949 +[#7913]: https://github.com/dgraph-io/dgraph/issues/7913 +[#7993]: https://github.com/dgraph-io/dgraph/issues/7993 +[#7936]: https://github.com/dgraph-io/dgraph/issues/7936 + +## [21.03.1] - 2021-06-16 +[21.03.1]: https://github.com/dgraph-io/dgraph/compare/v21.03.0...v21.03.1 + +### Fixed +- GraphQL + - fix(GraphQL): fix @cascade with Pagination for @auth queries ([#7695][]) + - Fix(GraphQL): Fix GraphQL encoding in case of empty list ([#7726][]) ([#7730][]) + - Fix(GraphQL): Add filter in DQL query in case of reverse predicate ([#7728][]) ([#7733][]) + - Fix(graphql): Fix error message of lambdaOnMutate directive ([#7751][]) ([#7754][]) + +- Core Dgraph + - fix(vault): Hide ACL flags when not required ([#7701][]) + - fix(Chunker): don't delete node with empty facet in mutation ([#7737][]) ([#7745][]) + - fix(bulk): throw the error instead of crashing ([#7722][]) ([#7749][]) + - fix(raftwal): take snapshot after restore ([#7719][]) ([#7750][]) + - fix(bulk): upsert guardian/groot for all existing namespaces ([#7759][]) ([#7769][]) + - fix(txn): ensure that txn hash is set ([#7782][]) ([#7784][]) + - bug fix to permit audit streaming to stdout writer([#7803][]) ([#7804][]) + - fix(drop): attach galaxy namespace to drop attr done on 20.11 backup ([#7827][]) + - fix: Prevent proposal from being dropped accidentally ([#7741][]) ([#7811][]) + - fix(schema-update): Start opIndexing only when index creation is required. ([#7845][]) ([#7847][]) + - fix(export): Fix facet export of reference type postings to JSON format ([#7744][]) ([#7756][]) + - fix(lease): don't do rate limiting when not limit is not specified ([#7787][]) + - fix(lease): prevent ID lease overflow ([#7802][]) + - fix(auth): preserve the status code while returning error ([#7832][]) ([#7834][]) + - fix(ee): GetKeys should return an error ([#7713][]) ([#7797][]) + - fix(admin): remove exportedFiles field ([#7835][]) ([#7836][]) + - fix(restore): append galaxy namespace to type name ([#7881][]) + - fix(DQL): revert changes related to cascade pagination with sort ([#7885][]) ([#7888][]) + - fix(metrics): Expose dgraph_num_backups_failed_total metric view. ([#7900][]) ([#7904][]) + +### Changed + - opt(GraphQL): filter existence queries on GraphQL side instead of using @filter(type) ([#7757][]) ([#7760][]) + +### Added + - feat(cdc): Add support for SCRAM SASL mechanism ([#7765][]) ([#7767][]) + - Add asynchronous task API ([#7781][]) + - make exports synchronous again ([#7877][]) + - feat(schema): do schema versioning and make backup non-blocking for i… ([#7856][]) ([#7873][]) + +[#7701]: https://github.com/dgraph-io/dgraph/issues/7701 +[#7737]: https://github.com/dgraph-io/dgraph/issues/7737 +[#7745]: https://github.com/dgraph-io/dgraph/issues/7745 +[#7722]: https://github.com/dgraph-io/dgraph/issues/7722 +[#7749]: https://github.com/dgraph-io/dgraph/issues/7749 +[#7719]: https://github.com/dgraph-io/dgraph/issues/7719 +[#7750]: https://github.com/dgraph-io/dgraph/issues/7750 +[#7765]: https://github.com/dgraph-io/dgraph/issues/7765 +[#7767]: https://github.com/dgraph-io/dgraph/issues/7767 +[#7759]: https://github.com/dgraph-io/dgraph/issues/7759 +[#7769]: https://github.com/dgraph-io/dgraph/issues/7769 +[#7782]: https://github.com/dgraph-io/dgraph/issues/7782 +[#7784]: https://github.com/dgraph-io/dgraph/issues/7784 +[#7803]: https://github.com/dgraph-io/dgraph/issues/7803 +[#7804]: https://github.com/dgraph-io/dgraph/issues/7804 +[#7827]: https://github.com/dgraph-io/dgraph/issues/7827 +[#7741]: https://github.com/dgraph-io/dgraph/issues/7741 +[#7811]: https://github.com/dgraph-io/dgraph/issues/7811 +[#7845]: https://github.com/dgraph-io/dgraph/issues/7845 +[#7847]: https://github.com/dgraph-io/dgraph/issues/7847 +[#7744]: https://github.com/dgraph-io/dgraph/issues/7744 +[#7756]: https://github.com/dgraph-io/dgraph/issues/7756 +[#7787]: https://github.com/dgraph-io/dgraph/issues/7787 +[#7802]: https://github.com/dgraph-io/dgraph/issues/7802 +[#7832]: https://github.com/dgraph-io/dgraph/issues/7832 +[#7834]: https://github.com/dgraph-io/dgraph/issues/7834 +[#7796]: https://github.com/dgraph-io/dgraph/issues/7796 +[#7781]: https://github.com/dgraph-io/dgraph/issues/7781 +[#7713]: https://github.com/dgraph-io/dgraph/issues/7713 +[#7797]: https://github.com/dgraph-io/dgraph/issues/7797 +[#7835]: https://github.com/dgraph-io/dgraph/issues/7835 +[#7836]: https://github.com/dgraph-io/dgraph/issues/7836 +[#7856]: https://github.com/dgraph-io/dgraph/issues/7856 +[#7873]: https://github.com/dgraph-io/dgraph/issues/7873 +[#7881]: https://github.com/dgraph-io/dgraph/issues/7881 +[#7885]: https://github.com/dgraph-io/dgraph/issues/7885 +[#7888]: https://github.com/dgraph-io/dgraph/issues/7888 +[#7877]: https://github.com/dgraph-io/dgraph/issues/7877 +[#7695]: https://github.com/dgraph-io/dgraph/issues/7695 +[#7726]: https://github.com/dgraph-io/dgraph/issues/7726 +[#7730]: https://github.com/dgraph-io/dgraph/issues/7730 +[#7728]: https://github.com/dgraph-io/dgraph/issues/7728 +[#7733]: https://github.com/dgraph-io/dgraph/issues/7733 +[#7751]: https://github.com/dgraph-io/dgraph/issues/7751 +[#7754]: https://github.com/dgraph-io/dgraph/issues/7754 +[#7757]: https://github.com/dgraph-io/dgraph/issues/7757 +[#7760]: https://github.com/dgraph-io/dgraph/issues/7760 +[#7900]: https://github.com/dgraph-io/dgraph/issues/7900 +[#7904]: https://github.com/dgraph-io/dgraph/issues/7904 + + +## [21.03.0] - 2021-04-07 +[21.03.0]: https://github.com/dgraph-io/dgraph/compare/v20.11.0...v21.03.0 + +### Changed + +- [BREAKING] Feat(flags): expand badger to accept all valid options ([#7677][]) +- [BREAKING] Feat(Dgraph): Read-Only replicas ([#7272][]) +- [BREAKING] Consolidate multiple flags into a few SuPerflags ([#7436][]) ([#7337][]) ([#7560][]) ([#7652][]) ([#7675][]) +- [BREAKING] Feat(zero): Make zero lease out namespace IDs ([#7341][]) +- [BREAKING] Fix(commit): make txn context more robust ([#7659][]) +- [BREAKING] Fix(Query): Return error for illegal math operations. ([#7631][]) +- [BREAKING] Rename Badger metrics. ([#7507][]) +- [BREAKING] Fix(Backups): new badger Superflag, NumGoroutines option solves OOM crashes ([#7387][]) +- [BREAKING] Remove restore tracker as its not necessary ([#7148][]) +- [BREAKING] Chore(GraphQL): Remove `dgraph.graphql.p_sha256hash` predicate and merge it into `dgraph.graphql.p_query` ([#7451][]) +- [BREAKING] Introducing Multi-Tenancy in dgraph ([#7293][]) ([#7400][]) ([#7397][]) ([#7399][]) ([#7377][]) ([#7414][]) ([#7418][]) + +### Added + +- GraphQL + - Feat(GraphQL): Zero HTTP endpoints are now available at GraphQL admin (GraphQL-1118) ([#6649][]) ([#7670][]) + - Feat(GraphQL): Webhooks on add/update/delete mutations (GraphQL-1045) ([#7494][]) ([#7616][]) + - Feat(GraphQL): Allow Multiple JWKUrls for auth. ([#7528][]) ([#7581][]) + - Feat(GraphQL): allow string --> Int64 hardcoded coercing ([#7584][]) + - Feat(Apollo): Add support for `@provides` and `@requires` directive. ([#7503][]) + - Feat(GraphQL): Handle upsert with multiple XIDs in case one of the XIDs does not exist ([#7472][]) + - Feat(GraphQL): Delete redundant reference to inverse object ([#7469][]) + - Feat(GraphQL): upgarde GraphQL-transport-ws module ([#7441][]) + - Feat(GraphQL): This PR allow multiple `@id` fields in a type. ([#7235][]) + - Feat(GraphQL): Add support for GraphQL Upsert Mutations ([#7433][]) + - Feat(GraphQL): This PR adds subscriptions to custom DQL. ([#7385][]) + - Feat(GraphQL): Make XID node referencing invariant of order in which XIDs are referenced in Mutation Rewriting ([#7448][]) + - Feat(GraphQL): Dgraph.Authorization should with irrespective of number of spaces after # ([#7410][]) + - Feat(GraphQL): adding auth token support for regexp, in and arrays ([#7039][]) + - Feat(GraphQL): Extend Support of IN filter to all the scalar data types ([#7340][]) + - Feat(GraphQL): Add `@include` and `@skip` to the Directives ([#7314][]) + - Feat(GraphQL): add support for has filter with list of arguments. ([#7406][]) + - Feat(GraphQL): Add support for has filter on list of fields. ([#7363][]) + - Feat(GraphQL): Allow standard claims into auth variables ([#7381][]) + - Perf(GraphQL): Generate GraphQL query response by optimized JSON encoding (GraphQL-730) ([#7371][]) + - Feat(GraphQL): Extend Support For Apollo Federation ([#7275][]) + - Feat(GraphQL): Support using custom DQL with `@groupby` ([#7476][]) + - Feat(GraphQL): Add support for passing OAuth Bearer token as authorization JWT ([#7490][]) + +- Core Dgraph + - Feat(query): Add mechanism to have a limit on number of pending queries ([#7603][]) + - Perf(bulk): Reuse allocator ([#7360][]) + - Perf(compression): Use gzip with BestSpeed in export and backup ([#7643][]) ([#7683][]) + - Feat(flags): Add query timeout as a limit config ([#7599][]) + - Opt(reindex): do not try building indices when inserting a new predicate ([#7109][]) + - Perf(txn): de-duplicate the context keys and predicates ([#7478][]) + - Feat(flags): use Vault for ACL secrets ([#7492][]) + - Feat(bulk): Add /jemalloc HTTP endpoint. ([#7165][]) + - Feat(metrics): Add Dgraph txn metrics (commits and discards). ([#7339][]) + - Feat(Bulk Loader + Live Loader): Supporting Loading files via s3/minio ([#7359][]) + - Feat(metrics): Add Raft leadership metrics. ([#7338][]) + - Use Badger's value log threshold of 1MB ([#7415][]) + - Feat(Monitoring): Adding Monitoring for Disk Space and Number of Backups ([#7404][]) + - Perf: simple simdjson solution with 30% speed increase ([#7316][]) + +- Enterprise Features + - Perf(Backup): Improve backup Performance ([#7601][]) + - Make backup API asynchronous + - Perf(backups): Reduce latency of list backups ([#7435][]) + - Feat(acl): allow setting a password at the time of creation of namespace ([#7446][]) + - Feat(enterprise): audit logs for alpha and zero ([#7295][]) + - Feat(enterpise): Change data capture (CDC) integration with kafka ([#7395][]) + - Perf(dgraph) - Use badger sinceTs in backups ([#7392][]) + - Perf(backup): Reorganize the output of lsbackup command ([#7354][]) + +### Fixed +- GraphQL + - Fix(GraphQL): Fix Execution Trace for Add and Update Mutations ([#7656][]) + - Fix(GraphQL): Add error handling for unrecognized args to generate directive. ([#7612][]) + - Fix(GraphQL): Fix panic when no schema exists for a new namespace ([#7630][]) + - Fix(GraphQL): Fixed output coercing for admin fields. ([#7617][]) + - Fix(GraphQL): Fix lambda querying a lambda field in case of no data. ([#7610][]) + - Fix(GraphQL): Undo the breaking change and tag it as deprecated. ([#7602][]) + - Fix(GraphQL): Add extra checks for deleting UpdateTypeInput ([#7595][]) + - Fix(persistent): make persistent query namespace aware ([#7570][]) + - Fix(GraphQL): remove support of `@id` directive on Float ([#7583][]) + - Fix(GraphQL): Fix mutation with Int Xid variables. ([#7565][]) ([#7588][]) + - Fix(GraphQL): Fix error message when dgraph and GraphQL schema differ. + - Fix(GraphQL): Fix custom(dql: ...) with `__typename` (GraphQL-1098) ([#7569][]) + - Fix(GraphQL): Change variable name generation for interface auth rules ([#7559][]) + - Fix(GraphQL): Apollo federation now works with lambda (GraphQL-1084) ([#7558][]) + - Fix(GraphQL): Fix empty remove in update mutation patch, that remove all the data for nodes in filter. ([#7563][]) + - Fix(GraphQL): Fix order of entities query result ([#7542][]) + - Fix(GraphQL): Change variable name generation from `Type` to `Type_` ([#7556][]) + - Fix(GraphQL): Fix duplicate xid error for multiple xid fields. ([#7546][]) + - Fix(GraphQL): Fix query rewriting for multiple order on nested field. ([#7523][]) + - Fix(GraphQL) Fix empty `type Query` with single extended type definition in the schema. ([#7517][]) + - Fix(GraphQL): Added support for parameterized cascade with variables. ([#7477][]) + - Fix(GraphQL): Fix fragment expansion in auth queries (GraphQL-1030) ([#7467][]) + - Fix(GraphQL): Refactor Mutation Rewriter for Add and Update Mutations ([#7409][]) + - Fix(GraphQL): Fix `@auth` rules evaluation in case of null variables in custom claims. ([#7380][]) + - Fix(GraphQL): Fix interface query with auth rules. ([#7401][]) + - Fix(GraphQL): Added error for case when multiple filter functions are used in filter. ([#7368][]) + - Fix(subscriptions): Fix subscription to use the kv with the max version ([#7349][]) + - Fix(GraphQL):This PR Fix a panic when we pass a single ID as a integer and expected type is `[ID]`.We now coerce that to type array of string. ([#7325][]) + - Fix(GraphQL): This PR Fix multi cors and multi schema nodes issue by selecting one of the latest added nodes, and add dgraph type to cors. ([#7270][]) + - Fix(GraphQL): This PR allow to use `__typename` in mutation. ([#7285][]) + - Fix(GraphQL): Fix auth-token propagation for HTTP endpoints resolved through GraphQL (GraphQL-946) ([#7245][]) + - Fix(GraphQL): This PR addd input coercion from single object to list and Fix panic when we pass single ID in filter as a string. ([#7133][]) + - Fix(GraphQL): adding support for `@id` with type other than strings ([#7019][]) + - Fix(GraphQL): Fix panic caused by incorrect input coercion of scalar to list ([#7405][]) + +- Core Dgraph + - Fix(flag): Fix bulk loader flag and remove flag parsing from critical path ([#7679][]) + - Fix(query): Fix pagination with match functions ([#7668][]) + - Fix(postingList): Acquire lock before reading the cached posting list ([#7632][]) + - Fix(zero): add a ratelimiter to limit the uid lease per namespace ([#7568][]) + - Fixing type inversion in ludicrous mode ([#7614][]) + - Fix(/commit): protect the commit endpoint via acl ([#7608][]) + - Fix(login): Fix login based on refresh token logic ([#7637][]) + - Fix(Query): Fix cascade pagination with 0 offset. ([#7636][]) + - Fix(telemetry): Track enterprise Feature usage ([#7495][]) + - Fix(dql): Fix error message in case of wrong argument to val() ([#7543][]) + - Fix(export): Fix namespace parameter in export ([#7524][]) + - Fix(live): Fix usage of force-namespace parameter in export ([#7526][]) + - Fix(Configs): Allow hierarchical notation in JSON/YAML configs ([#7498][]) + - Fix upsert mutations ([#7515][]) + - Fix(admin-endpoints): Error out if the request is rejected by the server ([#7511][]) + - Fix(Dgraph): Throttle number of files to open while schema update ([#7480][]) + - Fix(metrics): Expose Badger LSM and vlog size bytes. ([#7488][]) + - Fix(schema): log error instead of panic if schema not found for predicate ([#7502][]) + - Fix(moveTablet): make move tablet namespace aware ([#7468][]) + - Fix(dgraph): Do not return reverse edges from expandEdges ([#7461][]) + - Fix(Query): Fix cascade with pagination ([#7440][]) + - Fix(Mutation): Deeply-nested uid facets ([#7455][]) + - Fix(live): Fix live loader to load with force namespace ([#7445][]) + - Fix(sort): Fix multi-sort with nils ([#7432][]) + - Fix(GC): Reduce DiscardRatio from 0.9 to 0.7 ([#7412][]) + - Fix(jsonpb): use gogo/jsonpb for unmarshalling string ([#7382][]) + - Fix: Calling Discard only adds to `txn_discards` metric, not `txn_aborts`. ([#7365][]) + - Fix(Dgraph): check for deleteBelowTs in pIterator.valid ([#7288][]) + - Fix(dgraph): Add X-Dgraph-AuthToken to list of access control allowed headers + - Fix(sort): Make sort consistent for indexed and without indexed predicates ([#7241][]) + - Fix(ludicrous): Fix logical race in concurrent execution of mutations ([#7269][]) + - Fix(restore): Handle MaxUid=0 appropriately ([#7258][]) + - Fix(indexing): use encrypted tmpDBs for index building if encryption is enabled ([#6828][]) + - Fix(bulk): save schemaMap after map phase ([#7188][]) + - Fix(DQL): Fix Aggregate Functions on empty data ([#7176][]) + - Fixing unique proposal key error ([#7218][]) + - Fix(Chunker): JSON parsing Performance ([#7171][]) + - Fix(bulk): Fix memory held by b+ tree in reduce phase ([#7161][]) + - Fix(bulk): Fixing bulk loader when encryption + mtls is enabled ([#7154][]) + +- Enterprise Features + - Fix(restore): append the object path preFix while reading backup ([#7686][]) + - Fix restoring from old version for type ([#7456][]) + - Fix(backup): Fix Perf issues with full backups ([#7434][]) + - Fix(export-backup): Fix memory leak in backup export ([#7452][]) + - Fix(ACL): use acl for export, add GoG admin resolvers ([#7420][]) + - Fix(restore): reset acl accounts once restore is done if necessary ([#7202][]) + - Fix(restore): multiple restore requests should be rejected and proposals should not be submitted ([#7118][]) + +[#7677]: https://github.com/dgraph-io/dgraph/issues/7677 +[#7272]: https://github.com/dgraph-io/dgraph/issues/7272 +[#7436]: https://github.com/dgraph-io/dgraph/issues/7436 +[#7337]: https://github.com/dgraph-io/dgraph/issues/7337 +[#7560]: https://github.com/dgraph-io/dgraph/issues/7560 +[#7652]: https://github.com/dgraph-io/dgraph/issues/7652 +[#7675]: https://github.com/dgraph-io/dgraph/issues/7675 +[#7341]: https://github.com/dgraph-io/dgraph/issues/7341 +[#7659]: https://github.com/dgraph-io/dgraph/issues/7659 +[#7631]: https://github.com/dgraph-io/dgraph/issues/7631 +[#7507]: https://github.com/dgraph-io/dgraph/issues/7507 +[#7387]: https://github.com/dgraph-io/dgraph/issues/7387 +[#7148]: https://github.com/dgraph-io/dgraph/issues/7148 +[#7143]: https://github.com/dgraph-io/dgraph/issues/7143 +[#7451]: https://github.com/dgraph-io/dgraph/issues/7451 +[#6649]: https://github.com/dgraph-io/dgraph/issues/6649 +[#7670]: https://github.com/dgraph-io/dgraph/issues/7670 +[#7494]: https://github.com/dgraph-io/dgraph/issues/7494 +[#7616]: https://github.com/dgraph-io/dgraph/issues/7616 +[#7528]: https://github.com/dgraph-io/dgraph/issues/7528 +[#7581]: https://github.com/dgraph-io/dgraph/issues/7581 +[#7584]: https://github.com/dgraph-io/dgraph/issues/7584 +[#7503]: https://github.com/dgraph-io/dgraph/issues/7503 +[#7472]: https://github.com/dgraph-io/dgraph/issues/7472 +[#7469]: https://github.com/dgraph-io/dgraph/issues/7469 +[#7441]: https://github.com/dgraph-io/dgraph/issues/7441 +[#7235]: https://github.com/dgraph-io/dgraph/issues/7235 +[#7433]: https://github.com/dgraph-io/dgraph/issues/7433 +[#7385]: https://github.com/dgraph-io/dgraph/issues/7385 +[#7448]: https://github.com/dgraph-io/dgraph/issues/7448 +[#7410]: https://github.com/dgraph-io/dgraph/issues/7410 +[#7039]: https://github.com/dgraph-io/dgraph/issues/7039 +[#7340]: https://github.com/dgraph-io/dgraph/issues/7340 +[#7314]: https://github.com/dgraph-io/dgraph/issues/7314 +[#7406]: https://github.com/dgraph-io/dgraph/issues/7406 +[#7363]: https://github.com/dgraph-io/dgraph/issues/7363 +[#7381]: https://github.com/dgraph-io/dgraph/issues/7381 +[#7371]: https://github.com/dgraph-io/dgraph/issues/7371 +[#7275]: https://github.com/dgraph-io/dgraph/issues/7275 +[#7476]: https://github.com/dgraph-io/dgraph/issues/7476 +[#7490]: https://github.com/dgraph-io/dgraph/issues/7490 +[#7603]: https://github.com/dgraph-io/dgraph/issues/7603 +[#7360]: https://github.com/dgraph-io/dgraph/issues/7360 +[#7643]: https://github.com/dgraph-io/dgraph/issues/7643 +[#7683]: https://github.com/dgraph-io/dgraph/issues/7683 +[#7599]: https://github.com/dgraph-io/dgraph/issues/7599 +[#7109]: https://github.com/dgraph-io/dgraph/issues/7109 +[#7478]: https://github.com/dgraph-io/dgraph/issues/7478 +[#7492]: https://github.com/dgraph-io/dgraph/issues/7492 +[#7165]: https://github.com/dgraph-io/dgraph/issues/7165 +[#7339]: https://github.com/dgraph-io/dgraph/issues/7339 +[#7359]: https://github.com/dgraph-io/dgraph/issues/7359 +[#7338]: https://github.com/dgraph-io/dgraph/issues/7338 +[#7415]: https://github.com/dgraph-io/dgraph/issues/7415 +[#7404]: https://github.com/dgraph-io/dgraph/issues/7404 +[#7316]: https://github.com/dgraph-io/dgraph/issues/7316 +[#7601]: https://github.com/dgraph-io/dgraph/issues/7601 +[#7435]: https://github.com/dgraph-io/dgraph/issues/7435 +[#7446]: https://github.com/dgraph-io/dgraph/issues/7446 +[#7293]: https://github.com/dgraph-io/dgraph/issues/7293 +[#7400]: https://github.com/dgraph-io/dgraph/issues/7400 +[#7397]: https://github.com/dgraph-io/dgraph/issues/7397 +[#7399]: https://github.com/dgraph-io/dgraph/issues/7399 +[#7377]: https://github.com/dgraph-io/dgraph/issues/7377 +[#7414]: https://github.com/dgraph-io/dgraph/issues/7414 +[#7418]: https://github.com/dgraph-io/dgraph/issues/7418 +[#7295]: https://github.com/dgraph-io/dgraph/issues/7295 +[#7395]: https://github.com/dgraph-io/dgraph/issues/7395 +[#7392]: https://github.com/dgraph-io/dgraph/issues/7392 +[#7354]: https://github.com/dgraph-io/dgraph/issues/7354 +[#7656]: https://github.com/dgraph-io/dgraph/issues/7656 +[#7612]: https://github.com/dgraph-io/dgraph/issues/7612 +[#7630]: https://github.com/dgraph-io/dgraph/issues/7630 +[#7617]: https://github.com/dgraph-io/dgraph/issues/7617 +[#7610]: https://github.com/dgraph-io/dgraph/issues/7610 +[#7602]: https://github.com/dgraph-io/dgraph/issues/7602 +[#7595]: https://github.com/dgraph-io/dgraph/issues/7595 +[#7570]: https://github.com/dgraph-io/dgraph/issues/7570 +[#7583]: https://github.com/dgraph-io/dgraph/issues/7583 +[#7565]: https://github.com/dgraph-io/dgraph/issues/7565 +[#7588]: https://github.com/dgraph-io/dgraph/issues/7588 +[#7569]: https://github.com/dgraph-io/dgraph/issues/7569 +[#7559]: https://github.com/dgraph-io/dgraph/issues/7559 +[#7558]: https://github.com/dgraph-io/dgraph/issues/7558 +[#7563]: https://github.com/dgraph-io/dgraph/issues/7563 +[#7542]: https://github.com/dgraph-io/dgraph/issues/7542 +[#7556]: https://github.com/dgraph-io/dgraph/issues/7556 +[#7546]: https://github.com/dgraph-io/dgraph/issues/7546 +[#7523]: https://github.com/dgraph-io/dgraph/issues/7523 +[#7517]: https://github.com/dgraph-io/dgraph/issues/7517 +[#7477]: https://github.com/dgraph-io/dgraph/issues/7477 +[#7467]: https://github.com/dgraph-io/dgraph/issues/7467 +[#7409]: https://github.com/dgraph-io/dgraph/issues/7409 +[#7380]: https://github.com/dgraph-io/dgraph/issues/7380 +[#7401]: https://github.com/dgraph-io/dgraph/issues/7401 +[#7368]: https://github.com/dgraph-io/dgraph/issues/7368 +[#7349]: https://github.com/dgraph-io/dgraph/issues/7349 +[#7325]: https://github.com/dgraph-io/dgraph/issues/7325 +[#7270]: https://github.com/dgraph-io/dgraph/issues/7270 +[#7285]: https://github.com/dgraph-io/dgraph/issues/7285 +[#7245]: https://github.com/dgraph-io/dgraph/issues/7245 +[#7133]: https://github.com/dgraph-io/dgraph/issues/7133 +[#7019]: https://github.com/dgraph-io/dgraph/issues/7019 +[#7405]: https://github.com/dgraph-io/dgraph/issues/7405 +[#7679]: https://github.com/dgraph-io/dgraph/issues/7679 +[#7668]: https://github.com/dgraph-io/dgraph/issues/7668 +[#7632]: https://github.com/dgraph-io/dgraph/issues/7632 +[#7568]: https://github.com/dgraph-io/dgraph/issues/7568 +[#7614]: https://github.com/dgraph-io/dgraph/issues/7614 +[#7608]: https://github.com/dgraph-io/dgraph/issues/7608 +[#7637]: https://github.com/dgraph-io/dgraph/issues/7637 +[#7636]: https://github.com/dgraph-io/dgraph/issues/7636 +[#7495]: https://github.com/dgraph-io/dgraph/issues/7495 +[#7543]: https://github.com/dgraph-io/dgraph/issues/7543 +[#7524]: https://github.com/dgraph-io/dgraph/issues/7524 +[#7526]: https://github.com/dgraph-io/dgraph/issues/7526 +[#7498]: https://github.com/dgraph-io/dgraph/issues/7498 +[#7515]: https://github.com/dgraph-io/dgraph/issues/7515 +[#7511]: https://github.com/dgraph-io/dgraph/issues/7511 +[#7480]: https://github.com/dgraph-io/dgraph/issues/7480 +[#7488]: https://github.com/dgraph-io/dgraph/issues/7488 +[#7502]: https://github.com/dgraph-io/dgraph/issues/7502 +[#7468]: https://github.com/dgraph-io/dgraph/issues/7468 +[#7461]: https://github.com/dgraph-io/dgraph/issues/7461 +[#7440]: https://github.com/dgraph-io/dgraph/issues/7440 +[#7455]: https://github.com/dgraph-io/dgraph/issues/7455 +[#7445]: https://github.com/dgraph-io/dgraph/issues/7445 +[#7432]: https://github.com/dgraph-io/dgraph/issues/7432 +[#7412]: https://github.com/dgraph-io/dgraph/issues/7412 +[#7382]: https://github.com/dgraph-io/dgraph/issues/7382 +[#7365]: https://github.com/dgraph-io/dgraph/issues/7365 +[#7288]: https://github.com/dgraph-io/dgraph/issues/7288 +[#7241]: https://github.com/dgraph-io/dgraph/issues/7241 +[#7269]: https://github.com/dgraph-io/dgraph/issues/7269 +[#7258]: https://github.com/dgraph-io/dgraph/issues/7258 +[#6828]: https://github.com/dgraph-io/dgraph/issues/6828 +[#7188]: https://github.com/dgraph-io/dgraph/issues/7188 +[#7176]: https://github.com/dgraph-io/dgraph/issues/7176 +[#7218]: https://github.com/dgraph-io/dgraph/issues/7218 +[#7171]: https://github.com/dgraph-io/dgraph/issues/7171 +[#7161]: https://github.com/dgraph-io/dgraph/issues/7161 +[#7154]: https://github.com/dgraph-io/dgraph/issues/7154 +[#7686]: https://github.com/dgraph-io/dgraph/issues/7686 +[#7456]: https://github.com/dgraph-io/dgraph/issues/7456 +[#7434]: https://github.com/dgraph-io/dgraph/issues/7434 +[#7452]: https://github.com/dgraph-io/dgraph/issues/7452 +[#7420]: https://github.com/dgraph-io/dgraph/issues/7420 +[#7202]: https://github.com/dgraph-io/dgraph/issues/7202 +[#7118]: https://github.com/dgraph-io/dgraph/issues/7118 + +## [20.11.3] - 2021-03-31 +[20.11.3]: https://github.com/dgraph-io/dgraph/compare/v20.11.2...v20.11.3 + +### Fixed +- GraphQL + - Fix(GRAPHQL): fix query rewriting for multiple order on nested field ([#7523][]) ([#7536][]) + - Fix(GRAPHQL): Added support for exact index on field having @id directive ([#7534][]) ([#7550][]) + - Fix(GraphQL): Add extra checks for deleting UpdateTypeInput ([#7595][]) ([#7600][]) + - Fix(GRAPHQL): Undo the breaking change and tag it as deprecated. ([#7607][]) + - Fix(GraphQL): Log query along with the panic ([#7638][]) ([#7645][]) + - Fix(GraphQL): Fix Execution Trace for Add and Update Mutations ([#7656][]) ([#7658][]) + +- Core Dgraph + - Fix(schema): log error instead of panic if schema not found for predicate ([#7502][]) ([#7509][]) + - Chore(cmd/debuginfo) add new metrics to be collected ([#7439][]) ([#7562][]) + - Fix(vlog): Use Badger's value log threshold of 1MB ([#7415][]) ([#7474][]) + - Chore(bulk): Improve perf of bulk loader with Reuse allocator and assinging tags to allocator ([#7360][]) ([#7547][]) + - Fix(query): Fix pagination with match functions ([#7668][]) ([#7672][]) + +[#7523]: https://github.com/dgraph-io/dgraph/issues/7523 +[#7536]: https://github.com/dgraph-io/dgraph/issues/7536 +[#7534]: https://github.com/dgraph-io/dgraph/issues/7534 +[#7550]: https://github.com/dgraph-io/dgraph/issues/7550 +[#7595]: https://github.com/dgraph-io/dgraph/issues/7595 +[#7600]: https://github.com/dgraph-io/dgraph/issues/7600 +[#7607]: https://github.com/dgraph-io/dgraph/issues/7607 +[#7638]: https://github.com/dgraph-io/dgraph/issues/7638 +[#7645]: https://github.com/dgraph-io/dgraph/issues/7645 +[#7656]: https://github.com/dgraph-io/dgraph/issues/7656 +[#7658]: https://github.com/dgraph-io/dgraph/issues/7658 +[#7502]: https://github.com/dgraph-io/dgraph/issues/7502 +[#7509]: https://github.com/dgraph-io/dgraph/issues/7509 +[#7439]: https://github.com/dgraph-io/dgraph/issues/7439 +[#7562]: https://github.com/dgraph-io/dgraph/issues/7562 +[#7415]: https://github.com/dgraph-io/dgraph/issues/7415 +[#7474]: https://github.com/dgraph-io/dgraph/issues/7474 +[#7360]: https://github.com/dgraph-io/dgraph/issues/7360 +[#7547]: https://github.com/dgraph-io/dgraph/issues/7547 +[#7668]: https://github.com/dgraph-io/dgraph/issues/7668 +[#7672]: https://github.com/dgraph-io/dgraph/issues/7672 + +## [20.11.2] - 2021-02-23 +[20.11.2]: https://github.com/dgraph-io/dgraph/compare/v20.11.1...v20.11.2 + +### Fixed +- GraphQL + - Fix(Mutation): Deeply-nested uid facets ([#7457][]) + - Fix(GraphQL): Fix panic caused by incorrect input coercion of scalar to list ([#7405][]) ([#7428][]) + - Fix(GraphQL): Refactor Mutation Rewriter for Add and Update Mutations ([#7409][]) ([#7413][]) + - Fix(GraphQL): fix `@auth` rules evaluation in case of null values. ([#7411][]) + - Fix(GraphQL): fix interface query with auth rules ([#7408][]) + - Fix(GraphQL): Added error for case when multiple filter functions are used in filter. ([#7368][]) ([#7384][]) + +- Core Dgraph + - Fix(sort): Fix multi-sort with nils ([#7432][]) ([#7444][]) + - Fix(GC): Reduce DiscardRatio from 0.9 to 0.7 ([#7412][]) ([#7421][]) + +- Enterprise Features + - Fix(export-backup): fix memory leak in backup export ([#7452][]) ([#7453][]) + +[#7457]: https://github.com/dgraph-io/dgraph/issues/7457 +[#7405]: https://github.com/dgraph-io/dgraph/issues/7405 +[#7428]: https://github.com/dgraph-io/dgraph/issues/7428 +[#7409]: https://github.com/dgraph-io/dgraph/issues/7409 +[#7413]: https://github.com/dgraph-io/dgraph/issues/7413 +[#7411]: https://github.com/dgraph-io/dgraph/issues/7411 +[#7408]: https://github.com/dgraph-io/dgraph/issues/7408 +[#7368]: https://github.com/dgraph-io/dgraph/issues/7368 +[#7384]: https://github.com/dgraph-io/dgraph/issues/7384 +[#7432]: https://github.com/dgraph-io/dgraph/issues/7432 +[#7444]: https://github.com/dgraph-io/dgraph/issues/7444 +[#7412]: https://github.com/dgraph-io/dgraph/issues/7412 +[#7421]: https://github.com/dgraph-io/dgraph/issues/7421 +[#7452]: https://github.com/dgraph-io/dgraph/issues/7452 +[#7453]: https://github.com/dgraph-io/dgraph/issues/7453 + +## [20.11.1] - 2021-01-27 +[20.11.1]: https://github.com/dgraph-io/dgraph/compare/v20.11.0...v20.11.1 + +### Fixed +- GraphQL + - Fix(subscriptions): fix subscription to use the kv with the max version ([#7349][]) ([#7355][]) + - Fix(GraphQl): fix a panic when we pass a single ID as a integer and expected type is `[ID]`.We + now coerce that to type array of string. ([#7325][]) ([#7353][]) + - Fix(GRAPHQL): update gqlparser release to v2.1.4 ([#7347][]) ([#7352][]) + - Fix(GraphQL): Fix graphql flaky tests which were caused by receiving extra schema updates + ([#7329][]) ([#7348][]) + - Fix(GraphQL): This PR addd input coercion from single object to list and fix panic when we + pass single ID in filter as a string. ([#7133][]) ([#7306][]) + - Fix(GRAPHQL): Don't generate get query on interface if it doesn't have field of type ID and + also disallow get query on field of type `@id` in inerface. ([#7158][]) ([#7305][]) + - Fix(GraphQL): This PR fix multi cors and multi schema nodes issue by selecting one of the + latest added nodes, and add dgraph type to cors. ([#7270][]) ([#7302][]) + - Fix(GraphQL): This PR allow to use __typename in mutation. ([#7285][]) ([#7303][]) + - Fix(GraphQL): Fix auth-token propagation for HTTP endpoints resolved through GraphQL (GRAPHQL + -946) ([#7245][]) ([#7251][]) + +- Core Dgraph + - Fix(bulk): save schemaMap after map phase ([#7188][]) ([#7351][]) + - Fix(Dgraph): check for deleteBelowTs in pIterator.valid ([#7288][]) ([#7350][]) + - Fix(indexing): use encrypted tmpDBs for index building if encryption is enabled ([#6828][]) ([#7343][]) + - Fix(bulk): Fix memory held by b+ tree in reduce phase ([#7161][]) ([#7333][]) + - Feat(bulk): Add /jemalloc HTTP endpoint. ([#7165][]) ([#7331][]) + - Fix(sort): Make sort consistent for indexed and without indexed predicates ([#7241][]) ([#7323][]) + - Fix(dgraph): Add X-Dgraph-AuthToken to list of access control allowed headers ([#7311][]) + - Fix(ludicrous): Fix logical race in concurrent execution of mutations ([#7269][]) ([#7309][]) + - Fix(ludicrous): Fix data race in executor ([#7203][]) ([#7307][]) + - Opt(rollup): change the way rollups are done ([#7253][]) ([#7277][]) + - Fix(indexing): use --tmp directory for building indexes ([#7289][]) ([#7300][]) + - Fix(dgraph): Fix dgraph crash on windows ([#7261][]) ([#7299][]) + - Fix(dgraph): making jemalloc to work with dgraph on macos ([#7247][]) ([#7282][]) + - Fix(dgraph): Fixing multiple race conditions ([#7278][]) + - Fixing unique proposal key error ([#7218][]) ([#7281][]) + - Fix(raft): Unmarshal zero snapshot into pb.ZeroSnaphot ([#7244][]) + - Fix(bulk): fixing bulk loader when encryption + mtls is enabled ([#7154][]) ([#7155][]) + +- Enterprise Features + - Fix(restore): reset acl accounts once restore is done if necessary ([#7202][]) ([#7280][]) + - Fix(restore): multiple restore requests should be rejected and proposals should not be submitted ([#7118][]) ([#7276][]) + - Fix(restore): Handle MaxUid=0 appropriately ([#7258][]) ([#7265][]) + +[#7349]: https://github.com/dgraph-io/dgraph/issues/7349 +[#7355]: https://github.com/dgraph-io/dgraph/issues/7355 +[#7188]: https://github.com/dgraph-io/dgraph/issues/7188 +[#7351]: https://github.com/dgraph-io/dgraph/issues/7351 +[#7288]: https://github.com/dgraph-io/dgraph/issues/7288 +[#7350]: https://github.com/dgraph-io/dgraph/issues/7350 +[#7325]: https://github.com/dgraph-io/dgraph/issues/7325 +[#7353]: https://github.com/dgraph-io/dgraph/issues/7353 +[#7347]: https://github.com/dgraph-io/dgraph/issues/7347 +[#7352]: https://github.com/dgraph-io/dgraph/issues/7352 +[#6828]: https://github.com/dgraph-io/dgraph/issues/6828 +[#7343]: https://github.com/dgraph-io/dgraph/issues/7343 +[#7329]: https://github.com/dgraph-io/dgraph/issues/7329 +[#7348]: https://github.com/dgraph-io/dgraph/issues/7348 +[#7161]: https://github.com/dgraph-io/dgraph/issues/7161 +[#7333]: https://github.com/dgraph-io/dgraph/issues/7333 +[#7165]: https://github.com/dgraph-io/dgraph/issues/7165 +[#7331]: https://github.com/dgraph-io/dgraph/issues/7331 +[#7241]: https://github.com/dgraph-io/dgraph/issues/7241 +[#7323]: https://github.com/dgraph-io/dgraph/issues/7323 +[#7311]: https://github.com/dgraph-io/dgraph/issues/7311 +[#7269]: https://github.com/dgraph-io/dgraph/issues/7269 +[#7309]: https://github.com/dgraph-io/dgraph/issues/7309 +[#7133]: https://github.com/dgraph-io/dgraph/issues/7133 +[#7306]: https://github.com/dgraph-io/dgraph/issues/7306 +[#7158]: https://github.com/dgraph-io/dgraph/issues/7158 +[#7305]: https://github.com/dgraph-io/dgraph/issues/7305 +[#7270]: https://github.com/dgraph-io/dgraph/issues/7270 +[#7302]: https://github.com/dgraph-io/dgraph/issues/7302 +[#7285]: https://github.com/dgraph-io/dgraph/issues/7285 +[#7303]: https://github.com/dgraph-io/dgraph/issues/7303 +[#7203]: https://github.com/dgraph-io/dgraph/issues/7203 +[#7307]: https://github.com/dgraph-io/dgraph/issues/7307 +[#7253]: https://github.com/dgraph-io/dgraph/issues/7253 +[#7277]: https://github.com/dgraph-io/dgraph/issues/7277 +[#7289]: https://github.com/dgraph-io/dgraph/issues/7289 +[#7300]: https://github.com/dgraph-io/dgraph/issues/7300 +[#7261]: https://github.com/dgraph-io/dgraph/issues/7261 +[#7299]: https://github.com/dgraph-io/dgraph/issues/7299 +[#7247]: https://github.com/dgraph-io/dgraph/issues/7247 +[#7282]: https://github.com/dgraph-io/dgraph/issues/7282 +[#7278]: https://github.com/dgraph-io/dgraph/issues/7278 +[#7202]: https://github.com/dgraph-io/dgraph/issues/7202 +[#7280]: https://github.com/dgraph-io/dgraph/issues/7280 +[#7218]: https://github.com/dgraph-io/dgraph/issues/7218 +[#7281]: https://github.com/dgraph-io/dgraph/issues/7281 +[#7118]: https://github.com/dgraph-io/dgraph/issues/7118 +[#7276]: https://github.com/dgraph-io/dgraph/issues/7276 +[#7258]: https://github.com/dgraph-io/dgraph/issues/7258 +[#7265]: https://github.com/dgraph-io/dgraph/issues/7265 +[#7245]: https://github.com/dgraph-io/dgraph/issues/7245 +[#7251]: https://github.com/dgraph-io/dgraph/issues/7251 +[#7244]: https://github.com/dgraph-io/dgraph/issues/7244 +[#7154]: https://github.com/dgraph-io/dgraph/issues/7154 +[#7155]: https://github.com/dgraph-io/dgraph/issues/7155 + +## [20.11.0] - 2020-12-16 +[20.11.0]: https://github.com/dgraph-io/dgraph/compare/v20.07.0...v20.11.0 + +### Changed + +- [BREAKING] Feat: Use snappy compression by default. ([#6697][]) +- [BREAKING] Fix(OOM): Don't unmarshal pb.Proposals until we need them ([#7059][]) +- [BREAKING] Feat(Dgraph): Use Badger with new WAL format. ([#6643][]) +- [BREAKING] Switch Raft WAL to use simple files ([#6572][]) +- Feat(tls): splitting tls_dir + making health point available on HTTP ([#6821][]) + +### Added + +- GraphQL + - Feat(GraphQL): Add Aggregation Queries at Child Level ([#7022][]) + - Feat(GraphQL): Add aggregate query at root level ([#6985][]) + - Feat(GraphQL): Mutations with Auth on interfaces should work correctly. ([#6839][]) + - Feat(GraphQL): This PR adds support for "application/dql" in content header. ([#6849][]) + - Feat(GraphQL): Add count queries Feature at non-root levels ([#6834][]) + - Fix(GraphQL): AND/OR filters now accept an array while also accepting objects. ([#6801][]) + - Feat(GraphQL): Allow Query with Auth rules on Interfaces ([#6776][]) + - Feat(GraphQL): This PR adds auth switch in GraphQL authorization header. ([#6779][]) + - Feat(GraphQL): Add count query Feature at root to GraphQL ([#6786][]) + - Feat(GraphQL): Add generate directive to graphql schema ([#6760][]) + - Feat(GraphQL): add support for all RSA and HMAC algorithms supported by github.com/dgrijalva/jwt-go/v4 ([#6750][]) + - Feat(GraphQL): allow duplicate XIDs if only XID value is repeated ([#6762][]) + - Feat(GraphQL): Add support for Polygon and Multi-Polygon in GraphQL ([#6618][]) + - Feat(GraphQL): add support for between filter in GraphQL ([#6651][]) + - Feat(GraphQL): Unions ([#6722][]) + - Feat(GraphQL): add support for IN filter ([#6662][]) + - Feat(GraphQL): Add support for Geo point type in Graphql. ([#6481][]) + - Feat(GraphQL): GraphQL now has lambda resolvers ([#6574][]) + - Feat(GraphQL): Support authorization with jwk_url ([#6564][]) + - Feat(GraphQL): GQL Logging MW for admin query/mutation ([#6562][]) + - Feat: add schema history to graphql ([#6324][]) + - Feat(GraphQL): Add GraphQL schema validation Endpoint. ([#6250][]) + - Feat(GraphQL): This PR adds parameterised cascade in graphql. ([#6251][]) + - Feat(GraphQL): add has filter support ([#6258][]) + - Feat(GraphQL): GraphQL now has Int64 as scalar type ([#6200][]) + - Feat(GraphQL): `@custom` HTTP body now supports hardcoded scalars ([#6157][]) + - Feat(GraphQL): Custom logic now supports DQL queries ([#6115][]) + - Feat(GraphQL): This PR allows to return errors from custom REST endpoint. ([#6604][]) + +- Core Dgraph + - Feat(dgraph): Add suport for RDF query. ([#6038][]) + - perf(xidmap): Use btree with hash of keys for xidmap ([#6902][]) + - Feat(Query): Enable persistent queries in dgraph ([#6788][]) + - Feat(Dgraph): Add ability to change size of caches through the admin interface. ([#6644][]) + - Feat(query): Support for between func with count at root ([#6556][]) + - Feat(querylang): language support for term tokenization ([#6269][]) + - Feat(ludicrous): Run mutations from the same predicate concurrently in ludicrous mode ([#6060][]) + - Feat(Dgraph): Add experimental cache for posting lists ([#6245][]) + - Feat(dgraph): making all internal communications with tls configured ([#6692][]) + - Feat(dgraph): enabling TLS config in http zero ([#6691][]) + - Feat(raftwal): Add support for encryption in raftwal ([#6714][]) + - Feat(Dgraph): add utility to export backup data. ([#6550][]) + - Feature: dgraph_txn_aborts metric for prometheus ([#6171][]) + - Feat(live): added upsert in live loader ([#6057][]) + +- Enterprise Features + - Feat(Dgraph): Online restores allows to restore a specific backup. ([#6411][]) + +### Fixed + +- GraphQL + - Fix(GraphQL): Fix internal Aliases name generation ([#7009][]) + - Fix(GraphQL): Allows repetition of fields inside implementing type in + interface and allow to inherit field of same name of type ID from multiple interfaces. ([#7053][]) + - Fix(GraphQL): Fix password query rewriting in release/v20.11 ([#7012][]) + - Fix(GraphQL): Fix bug with password query rewriting ([#7011][]) + - Fix(GraphQL): Use fragments on interfaces while querying other interface. ([#6964][]) + - Fix(GraphQL): Fix multiple alias in query ([#6940][]) + - Fix(GraphQL): Add support for using auth with secret directive ([#6920][]) + - Fix(GraphQL): Fix exclusion of filters in Query generation ([#6917][]) + - Fix(GraphQL): handle filters for enum properly ([#6916][]) + - Fix(GraphQL): Fixes issue of multiple responses in a subscription for an update. ([#6868][]) + - Fix(GraphQL): Fix panic caused when trying to delete a nested object which doesn't have id/xid ([#6810][]) + - Fix(GraphQL): Fix between filter bugs ([#6822][]) + - Fix(GraphQL): Fix panic error when we give null value in filter connectives. ([#6707][]) + - Fix(GraphQL): Remove extra fields when querying interfaces ([#6596][]) + - Fix(GraphQL): disallowing field names with as ([#6579][]) + - Fix(GraphQL): Fix object Linking with `hasInverse` ([#6557][]) + - Fix(GraphQL): Fix cascade with auth query when RBAC is false ([#6444][]) + - Fix(GraphQL): Generate correct schema when no orderable field in a type ([#6456][]) + - Fix(GraphQL): Fix restoreStatus query with query variables ([#6414][]) + - Fix(GraphQL): Fix for deletion on interfaces with no non Id field ([#6387][]) + - Fix(GraphQL): don't generate orderable enum value for list fields ([#6392][]) + - Fix(GraphQL): Fix introspection completion bug ([#6385][]) + - Fix(GraphQL): Extend int64 range to 64-bit numeric values and adds input coercing and + validation for integers. ([#6275][]) + - Fix(GraphQL): Remove auth error from mutation. ([#6329][]) + - Fix(GraphQL): Fix query rewriting for auth delete when deleting types with inverse field. ([#6350][]) + - Fix(GraphQL): incorrect generatedSchema in updateGQLSchema ([#6349][]) + - Fix(GraphQL): Link xids properly if there are duplicate xids within the same add request. ([#6265][]) + - Fix(GraphQL): Fix internal error when doing GraphQL schema introspection after drop all ([#6268][]) + - Fix(GraphQL): Fixes unexpected fragment behaviour ([#6228][]) + - Fix(GraphQL): Fix order and offset in auth queries. ([#6221][]) + - Fix(GraphQL): Linking of xids for deep mutations ([#6172][]) + - Fix(GraphQL): Don't reserve certain queries/mutations/inputs when a type is remote. ([#6055][]) + - Fix(GraphQl): Allow case insensitive auth header for graphql subscriptions. ([#6141][]) + - Fix(GraphQl): Panic Fix when subscription expiry is not present in jwt. ([#6129][]) + - Fix(GraphQL): Fix bug in custom resolver, now body need not have all the fields. ([#6054][]) + - Fix(GraphQL): Disallow Subscription typename. ([#6077][]) + - Fix(GraphQL): Fixes wrong query parameter value for custom field URL ([#6074][]) + - Fix(GraphQL): Fixes panic in update mutation without set & remove ([#6073][]) + - Fix(GraphQL): Fix auth rewriting for nested queries when RBAC rule is true. ([#6053][]) + - Fix(GraphQL): Fix getType queries when id was used as a name for types other than ID ([#6130][]) + +- Core Dgraph + - Fix(ludicrous mode): Handle deletes correctly ([#6773][]) + - Fix(Zero): Fix how Zero snapshots and purge works ([#7096][]) + - Fix: Check for nil ServerCloser in shutdown handler ([#7048][]) + - Fix(health): Update health only after upserting schema and types ([#7006][]) + - Fix(worker): Flush the stream writer on error (DGRAPH-2499) ([#6609][]) + - Fix(export): don't return an error if there was no GraphQL schema ([#6815][]) + - Fix pointer misalignment ([#6795][]) + - Fix(metrics): Show memory metrics for zero ([#6743][]) + - feat(Query): Allow filters in expand(_all_) queries on predicates pointing to nodes ([#6752][]) + - Fix(Ludicrous): Upserts on list type in Dgraph ([#6754][]) + - Fix(worker): Avoid panic in handleUidPostings ([#6607][]) + - Fix(config): Set glog -v flag correctly from config files. ([#6678][]) + - Fix ErrIndexingInProgress if schema update fails ([#6583][]) + - feat(bulk): Allow encrypted input with unencrypted output in bulk. ([#6541][]) + - Fix(Dgraph): Subscribe to ACL updates instead of polling. ([#6459][]) + - Fix(Alpha): Immediately take a snapshot if we don't have one ([#6458][]) + - Fix(Dgraph): Fix bug when deleting and adding to a single UID predicate in the same transaction. ([#6431][]) + - Fix(raft): Only leader should check the quorum ([#6323][]) + - Fix(Dgraph): Parse Content-Type in headers correctly ([#6370][]) + - Fix(shutdown): Force exit if CTRL-C is caught before initialization ([#6359][]) + - Fix(Query) Fix Star_All delete query when used with ACL enabled ([#6331][]) + - Fix(Alpha): MASA: Make Alpha Shutdown Again ([#6313][]) + - Fix(Dgraph): Fix how visited nodes are detected in recurse queries. ([#6272][]) + - Fix(Dgraph): make backups cancel other tasks ([#6152][]) + - Fix(Dgraph): Don't store start_ts in postings. ([#6206][]) + - Fix(Dgraph): Perform rollups more aggresively. ([#6143][]) + - Fix(rollups): rollup a batch if more than 2 seconds elapsed since last batch ([#6118][]) + +- Enterprise Features + - Fix(enterprise): Set version correctly post marshalling during restore ([#7018][]) + - Add badger.compression to Dgraph restore ([#6987][]) + - Fix(backup/restore): Fixes backup and restore with DROP operations (GRAPHQL-735) ([#6844][]) + - Fix(ACL) : Disallow deleting of groot user and guardians group ([#6580][]) + - Fix: Online Restore honors credentials passed in ([#6295][]) + - Fix(ACL Query): Fixes queries which use variable at the top level ([#6290][]) + - Fix(Dgraph): race condition in EnterpriseEnabled() ([#6793][]) + +[#6697]: https://github.com/dgraph-io/dgraph/issues/6697 +[#7059]: https://github.com/dgraph-io/dgraph/issues/7059 +[#6643]: https://github.com/dgraph-io/dgraph/issues/6643 +[#6572]: https://github.com/dgraph-io/dgraph/issues/6572 +[#6821]: https://github.com/dgraph-io/dgraph/issues/6821 +[#7022]: https://github.com/dgraph-io/dgraph/issues/7022 +[#6985]: https://github.com/dgraph-io/dgraph/issues/6985 +[#6839]: https://github.com/dgraph-io/dgraph/issues/6839 +[#6849]: https://github.com/dgraph-io/dgraph/issues/6849 +[#6834]: https://github.com/dgraph-io/dgraph/issues/6834 +[#6801]: https://github.com/dgraph-io/dgraph/issues/6801 +[#6776]: https://github.com/dgraph-io/dgraph/issues/6776 +[#6779]: https://github.com/dgraph-io/dgraph/issues/6779 +[#6786]: https://github.com/dgraph-io/dgraph/issues/6786 +[#6760]: https://github.com/dgraph-io/dgraph/issues/6760 +[#6750]: https://github.com/dgraph-io/dgraph/issues/6750 +[#6762]: https://github.com/dgraph-io/dgraph/issues/6762 +[#6618]: https://github.com/dgraph-io/dgraph/issues/6618 +[#6651]: https://github.com/dgraph-io/dgraph/issues/6651 +[#6722]: https://github.com/dgraph-io/dgraph/issues/6722 +[#6662]: https://github.com/dgraph-io/dgraph/issues/6662 +[#6481]: https://github.com/dgraph-io/dgraph/issues/6481 +[#6574]: https://github.com/dgraph-io/dgraph/issues/6574 +[#6564]: https://github.com/dgraph-io/dgraph/issues/6564 +[#6562]: https://github.com/dgraph-io/dgraph/issues/6562 +[#6324]: https://github.com/dgraph-io/dgraph/issues/6324 +[#6250]: https://github.com/dgraph-io/dgraph/issues/6250 +[#6251]: https://github.com/dgraph-io/dgraph/issues/6251 +[#6258]: https://github.com/dgraph-io/dgraph/issues/6258 +[#6200]: https://github.com/dgraph-io/dgraph/issues/6200 +[#6157]: https://github.com/dgraph-io/dgraph/issues/6157 +[#6038]: https://github.com/dgraph-io/dgraph/issues/6038 +[#6115]: https://github.com/dgraph-io/dgraph/issues/6115 +[#6604]: https://github.com/dgraph-io/dgraph/issues/6604 +[#6902]: https://github.com/dgraph-io/dgraph/issues/6902 +[#6788]: https://github.com/dgraph-io/dgraph/issues/6788 +[#6773]: https://github.com/dgraph-io/dgraph/issues/6773 +[#6644]: https://github.com/dgraph-io/dgraph/issues/6644 +[#6556]: https://github.com/dgraph-io/dgraph/issues/6556 +[#6269]: https://github.com/dgraph-io/dgraph/issues/6269 +[#6060]: https://github.com/dgraph-io/dgraph/issues/6060 +[#6245]: https://github.com/dgraph-io/dgraph/issues/6245 +[#6692]: https://github.com/dgraph-io/dgraph/issues/6692 +[#6691]: https://github.com/dgraph-io/dgraph/issues/6691 +[#6714]: https://github.com/dgraph-io/dgraph/issues/6714 +[#6550]: https://github.com/dgraph-io/dgraph/issues/6550 +[#6171]: https://github.com/dgraph-io/dgraph/issues/6171 +[#6057]: https://github.com/dgraph-io/dgraph/issues/6057 +[#6411]: https://github.com/dgraph-io/dgraph/issues/6411 +[#7009]: https://github.com/dgraph-io/dgraph/issues/7009 +[#7053]: https://github.com/dgraph-io/dgraph/issues/7053 +[#7012]: https://github.com/dgraph-io/dgraph/issues/7012 +[#7011]: https://github.com/dgraph-io/dgraph/issues/7011 +[#6964]: https://github.com/dgraph-io/dgraph/issues/6964 +[#6940]: https://github.com/dgraph-io/dgraph/issues/6940 +[#6920]: https://github.com/dgraph-io/dgraph/issues/6920 +[#6917]: https://github.com/dgraph-io/dgraph/issues/6917 +[#6916]: https://github.com/dgraph-io/dgraph/issues/6916 +[#6868]: https://github.com/dgraph-io/dgraph/issues/6868 +[#6810]: https://github.com/dgraph-io/dgraph/issues/6810 +[#6822]: https://github.com/dgraph-io/dgraph/issues/6822 +[#6707]: https://github.com/dgraph-io/dgraph/issues/6707 +[#6596]: https://github.com/dgraph-io/dgraph/issues/6596 +[#6579]: https://github.com/dgraph-io/dgraph/issues/6579 +[#6557]: https://github.com/dgraph-io/dgraph/issues/6557 +[#6444]: https://github.com/dgraph-io/dgraph/issues/6444 +[#6456]: https://github.com/dgraph-io/dgraph/issues/6456 +[#6414]: https://github.com/dgraph-io/dgraph/issues/6414 +[#6387]: https://github.com/dgraph-io/dgraph/issues/6387 +[#6392]: https://github.com/dgraph-io/dgraph/issues/6392 +[#6385]: https://github.com/dgraph-io/dgraph/issues/6385 +[#6275]: https://github.com/dgraph-io/dgraph/issues/6275 +[#6329]: https://github.com/dgraph-io/dgraph/issues/6329 +[#6350]: https://github.com/dgraph-io/dgraph/issues/6350 +[#6349]: https://github.com/dgraph-io/dgraph/issues/6349 +[#6265]: https://github.com/dgraph-io/dgraph/issues/6265 +[#6268]: https://github.com/dgraph-io/dgraph/issues/6268 +[#6228]: https://github.com/dgraph-io/dgraph/issues/6228 +[#6221]: https://github.com/dgraph-io/dgraph/issues/6221 +[#6172]: https://github.com/dgraph-io/dgraph/issues/6172 +[#6055]: https://github.com/dgraph-io/dgraph/issues/6055 +[#6141]: https://github.com/dgraph-io/dgraph/issues/6141 +[#6129]: https://github.com/dgraph-io/dgraph/issues/6129 +[#6054]: https://github.com/dgraph-io/dgraph/issues/6054 +[#6077]: https://github.com/dgraph-io/dgraph/issues/6077 +[#6074]: https://github.com/dgraph-io/dgraph/issues/6074 +[#6073]: https://github.com/dgraph-io/dgraph/issues/6073 +[#6053]: https://github.com/dgraph-io/dgraph/issues/6053 +[#6130]: https://github.com/dgraph-io/dgraph/issues/6130 +[#7096]: https://github.com/dgraph-io/dgraph/issues/7096 +[#7048]: https://github.com/dgraph-io/dgraph/issues/7048 +[#7006]: https://github.com/dgraph-io/dgraph/issues/7006 +[#6609]: https://github.com/dgraph-io/dgraph/issues/6609 +[#6815]: https://github.com/dgraph-io/dgraph/issues/6815 +[#6795]: https://github.com/dgraph-io/dgraph/issues/6795 +[#6743]: https://github.com/dgraph-io/dgraph/issues/6743 +[#6752]: https://github.com/dgraph-io/dgraph/issues/6752 +[#6754]: https://github.com/dgraph-io/dgraph/issues/6754 +[#6607]: https://github.com/dgraph-io/dgraph/issues/6607 +[#6678]: https://github.com/dgraph-io/dgraph/issues/6678 +[#6583]: https://github.com/dgraph-io/dgraph/issues/6583 +[#6541]: https://github.com/dgraph-io/dgraph/issues/6541 +[#6459]: https://github.com/dgraph-io/dgraph/issues/6459 +[#6458]: https://github.com/dgraph-io/dgraph/issues/6458 +[#6431]: https://github.com/dgraph-io/dgraph/issues/6431 +[#6323]: https://github.com/dgraph-io/dgraph/issues/6323 +[#6370]: https://github.com/dgraph-io/dgraph/issues/6370 +[#6359]: https://github.com/dgraph-io/dgraph/issues/6359 +[#6331]: https://github.com/dgraph-io/dgraph/issues/6331 +[#6313]: https://github.com/dgraph-io/dgraph/issues/6313 +[#6272]: https://github.com/dgraph-io/dgraph/issues/6272 +[#6152]: https://github.com/dgraph-io/dgraph/issues/6152 +[#6206]: https://github.com/dgraph-io/dgraph/issues/6206 +[#6143]: https://github.com/dgraph-io/dgraph/issues/6143 +[#6118]: https://github.com/dgraph-io/dgraph/issues/6118 +[#7018]: https://github.com/dgraph-io/dgraph/issues/7018 +[#6987]: https://github.com/dgraph-io/dgraph/issues/6987 +[#6844]: https://github.com/dgraph-io/dgraph/issues/6844 +[#6580]: https://github.com/dgraph-io/dgraph/issues/6580 +[#6295]: https://github.com/dgraph-io/dgraph/issues/6295 +[#6290]: https://github.com/dgraph-io/dgraph/issues/6290 +[#6793]: https://github.com/dgraph-io/dgraph/issues/6793 + +## [20.07.3] - 2020-12-29 +[20.07.3]: https://github.com/dgraph-io/dgraph/compare/v20.07.2...v20.07.3 + +### Changed + +- Chore(bulk): Change default compression to zstd:3. ([#6995][]) +- Build(dockerfile): Set GODEBUG=madvdontneed=1. ([#6955][]) +- Updating badger version 3f846b3. ([#7212][]) + +### Added: + +- Update kubernetes links to match 20.07 config files ([#7049][]) +- Fix(dgraph): giving users the option to control tls versions ([#6820][]) +- Feat(dgraph): making all internal communications with tls configured ([#6876][]) +- Feat(dgraph): enabling TLS config in http zero ([#6691) ([#6867][]) + +### Fixed: + +- GraphQL + - Fix(GraphQL): don't update cacheMb if not specified by user ([#7103][]) + - Fix: added comment docstring for ExportInput format ([#6991][]) + - Fix(GraphQL): fixes issue of multiple responses in a subscription for an update. ([#6868][]) + - Fix ErrIndexingInProgress if schema update fails ([#6583][]) + - Fix(GraphQL): fix panic error when we give null value in filter connectives. ([#6707][]) + - Fix(GraphQL): reduces polling duration of subscriptions. ([#6661][]) + - Fix(GraphQL): add enable schema cleaning in GraphQL and reduce schema update time. ([#6725][]) + - Fix(GraphQL): fixes flaky test for subscriptions. ([#6065][]) +- Fix(DQL): ignore ordering of indexes in schema with eq function (DGRAPH-2601) ([#6996][]) +- Fix(worker): fix eq filter for non-index predicates. ([#6986][]) +- Fix(Alpha): Immediately take a snapshot if we don't have one ([#6458][]) +- Fix(Dgraph): Type names in exported schema are surrounded by brackets. ([#6679][]) +- Fix(ludicrous mode): Handle deletes correctly ([#6773][]) +- Fix(worker): Avoid panic in handleUidPostings ([#6607][]) +- Fix(gqlParser): Handle strings with only whitespace in parseID ([#6615][]) +- Fix(Ludicrous): Upserts on list type in Dgraph ([#6796][]) +- Enterprise features + - Fix(backup/restore): fixes backup and restore with DROP operations ([#6922][]) + - Generic alpha log error message for failed ACL login ([#6848][]) + +[#6995]: https://github.com/dgraph-io/dgraph/issues/6995 +[#6955]: https://github.com/dgraph-io/dgraph/issues/6955 +[#7212]: https://github.com/dgraph-io/dgraph/issues/7212 +[#7049]: https://github.com/dgraph-io/dgraph/issues/7049 +[#6820]: https://github.com/dgraph-io/dgraph/issues/6820 +[#6876]: https://github.com/dgraph-io/dgraph/issues/6876 +[#6867]: https://github.com/dgraph-io/dgraph/issues/6867 +[#7103]: https://github.com/dgraph-io/dgraph/issues/7103 +[#6991]: https://github.com/dgraph-io/dgraph/issues/6991 +[#6868]: https://github.com/dgraph-io/dgraph/issues/6868 +[#6583]: https://github.com/dgraph-io/dgraph/issues/6583 +[#6707]: https://github.com/dgraph-io/dgraph/issues/6707 +[#6661]: https://github.com/dgraph-io/dgraph/issues/6661 +[#6725]: https://github.com/dgraph-io/dgraph/issues/6725 +[#6065]: https://github.com/dgraph-io/dgraph/issues/6065 +[#6996]: https://github.com/dgraph-io/dgraph/issues/6996 +[#6986]: https://github.com/dgraph-io/dgraph/issues/6986 +[#6458]: https://github.com/dgraph-io/dgraph/issues/6458 +[#6679]: https://github.com/dgraph-io/dgraph/issues/6679 +[#6773]: https://github.com/dgraph-io/dgraph/issues/6773 +[#6607]: https://github.com/dgraph-io/dgraph/issues/6607 +[#6615]: https://github.com/dgraph-io/dgraph/issues/6615 +[#6796]: https://github.com/dgraph-io/dgraph/issues/6796 +[#6922]: https://github.com/dgraph-io/dgraph/issues/6922 +[#6848]: https://github.com/dgraph-io/dgraph/issues/6848 + + +## [20.07.2] - 2020-10-22 +[20.07.2]: https://github.com/dgraph-io/dgraph/compare/v20.07.1...v20.07.2 + +### Changed + +- Update badger to 5e3d4b9. ([#6669][]) +- Makefile to build Dgraph inside docker container. ([#6601][]) +- Return content length header for queries. ([#6480][]) +- Use raft storage in managedmode. ([#6547][]) +- Update index.md. ([#6567][]) +- Changes github.com/dgraph-io/graphql-transport-ws version. ([#6529][]) + +### Added + +- Add utility to export backup data. ([#6590][]) +- Add separate compression flag for z and wal dirs. ([#6421][]) + +### Fixed + +- GraphQL + - Disallowing field names with as. ([#6645][]) + - Remove extra fields when querying interfaces. ([#6647][]) + - fix object Linking with `hasInverse`. ([#6648][]) + - Update gqlgen in go.mod. ([#6646][]) + - Hide info when performing mutation on id field with auth rule. ([#6534][]) + - Fix cascade with auth query when RBAC is false. ([#6535][]) + - Fix squashIntoObject so that results are correctly merged. ([#6530][]) + - Fix errors from authorization examples given in docs. ([#6522][]) + - Fix restoreStatus query with query variables. ([#6424][]) + - Fix for deletion on interfaces with no non Id field. ([#6417][]) + - Fix internal error when doing GraphQL schema introspection after drop all. ([#6525][]) + - Link xids properly if there are duplicate xids within type. ([#6521][]) + - Fix query rewriting for auth delete when deleting types with inverse field. ([#6524][]) + - Fix order and offset in auth queries. ([#6366][]) + - Generate correct schema when no orderable field in a type. ([#6460][]) + - Don't generate orderable enum value for list fields. ([#6413][]) + - Dix introspection completion bug. ([#6389][]) +- Fix Poor-man's auth for admin operations. ([#6686][]) +- Break out if g.Ctx is done. ([#6675][]) +- Fix wrong path response for k-shortest paths. ([#6654][]) +- Update nextRaftId when starting a node with a raftId > 0. ([#6597][]) +- Pagination param "after" does not work when using func: uid(v). ([#6634][]) +- CID never created if Zero stops early after first init. ([#6637][]) +- Pause rollups during snapshot streaming. ([#6611][]) +- Use flags for cache. ([#6467][]) +- Remove auth error from mutation. ([#6532][]) +- Fix readTs less than minTs. ([#6517][]) +- Fix bug when deleting and adding to a single UID predicate in the same transaction. ([#6449][]) + +[#6669]: https://github.com/dgraph-io/dgraph/issues/6669 +[#6601]: https://github.com/dgraph-io/dgraph/issues/6601 +[#6480]: https://github.com/dgraph-io/dgraph/issues/6480 +[#6547]: https://github.com/dgraph-io/dgraph/issues/6547 +[#6567]: https://github.com/dgraph-io/dgraph/issues/6567 +[#6529]: https://github.com/dgraph-io/dgraph/issues/6529 +[#6590]: https://github.com/dgraph-io/dgraph/issues/6590 +[#6421]: https://github.com/dgraph-io/dgraph/issues/6421 +[#6645]: https://github.com/dgraph-io/dgraph/issues/6645 +[#6647]: https://github.com/dgraph-io/dgraph/issues/6647 +[#6648]: https://github.com/dgraph-io/dgraph/issues/6648 +[#6646]: https://github.com/dgraph-io/dgraph/issues/6646 +[#6534]: https://github.com/dgraph-io/dgraph/issues/6534 +[#6535]: https://github.com/dgraph-io/dgraph/issues/6535 +[#6530]: https://github.com/dgraph-io/dgraph/issues/6530 +[#6522]: https://github.com/dgraph-io/dgraph/issues/6522 +[#6424]: https://github.com/dgraph-io/dgraph/issues/6424 +[#6417]: https://github.com/dgraph-io/dgraph/issues/6417 +[#6525]: https://github.com/dgraph-io/dgraph/issues/6525 +[#6521]: https://github.com/dgraph-io/dgraph/issues/6521 +[#6524]: https://github.com/dgraph-io/dgraph/issues/6524 +[#6366]: https://github.com/dgraph-io/dgraph/issues/6366 +[#6460]: https://github.com/dgraph-io/dgraph/issues/6460 +[#6413]: https://github.com/dgraph-io/dgraph/issues/6413 +[#6389]: https://github.com/dgraph-io/dgraph/issues/6389 +[#6686]: https://github.com/dgraph-io/dgraph/issues/6686 +[#6675]: https://github.com/dgraph-io/dgraph/issues/6675 +[#6654]: https://github.com/dgraph-io/dgraph/issues/6654 +[#6597]: https://github.com/dgraph-io/dgraph/issues/6597 +[#6634]: https://github.com/dgraph-io/dgraph/issues/6634 +[#6637]: https://github.com/dgraph-io/dgraph/issues/6637 +[#6611]: https://github.com/dgraph-io/dgraph/issues/6611 +[#6467]: https://github.com/dgraph-io/dgraph/issues/6467 +[#6532]: https://github.com/dgraph-io/dgraph/issues/6532 +[#6517]: https://github.com/dgraph-io/dgraph/issues/6517 +[#6449]: https://github.com/dgraph-io/dgraph/issues/6449 + ## [20.07.1] - 2020-09-17 [20.07.1]: https://github.com/dgraph-io/dgraph/compare/v20.07.0...v20.07.1 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3be5ca8e196..b9498d8c209 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -133,7 +133,7 @@ Copyright 2015-2018 Dgraph Labs, Inc. ``` For release images, follow [Doing a release](#doing-a-release). It creates -Docker images that contains `dgraph`, `dgraph-ratel`, and `badger` commands. +Docker images that contains `dgraph` and `badger` commands. ### Testing diff --git a/Makefile b/Makefile index 4e4344e51f8..df47a13cbf4 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,7 @@ # BUILD ?= $(shell git rev-parse --short HEAD) -BUILD_CODENAME = unnamed +BUILD_CODENAME = zion BUILD_DATE ?= $(shell git log -1 --format=%ci) BUILD_BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD) BUILD_VERSION ?= $(shell git describe --always --tags) @@ -64,7 +64,7 @@ test: image: @GOOS=linux $(MAKE) dgraph - @mkdir linux + @mkdir -p linux @mv ./dgraph/dgraph ./linux/dgraph @docker build -f contrib/Dockerfile -t dgraph/dgraph:$(subst /,-,${BUILD_BRANCH}) . @rm -r linux diff --git a/README.md b/README.md index adb01435dd2..606f82050d0 100644 --- a/README.md +++ b/README.md @@ -7,25 +7,25 @@ [![Coverage Status](https://coveralls.io/repos/github/dgraph-io/dgraph/badge.svg?branch=master)](https://coveralls.io/github/dgraph-io/dgraph?branch=master) [![Go Report Card](https://goreportcard.com/badge/github.com/dgraph-io/dgraph)](https://goreportcard.com/report/github.com/dgraph-io/dgraph) -Dgraph is a horizontally scalable and distributed GraphQL database with a graph backend. It provides ACID transactions, consistent replication and linearizable reads. It's built from ground up to perform for +Dgraph is a horizontally scalable and distributed GraphQL database with a graph backend. It provides ACID transactions, consistent replication, and linearizable reads. It's built from the ground up to perform for a rich set of queries. Being a native GraphQL database, it tightly controls how the data is arranged on disk to optimize for query performance and throughput, reducing disk seeks and network calls in a cluster. Dgraph's goal is to provide [Google](https://www.google.com) production level scale and throughput, -with low enough latency to be serving real time user queries, over terabytes of structured data. +with low enough latency to be serving real-time user queries, over terabytes of structured data. Dgraph supports [GraphQL query syntax](https://dgraph.io/docs/master/query-language/), and responds in [JSON](http://www.json.org/) and [Protocol Buffers](https://developers.google.com/protocol-buffers/) over [GRPC](http://www.grpc.io/) and HTTP. **Use [Discuss Issues](https://discuss.dgraph.io/c/issues/dgraph/38) for reporting issues about this repository.** ## Status -Dgraph is [at version 20.07.2][rel] and is production ready. Apart from the vast open source community, it is being used in +Dgraph is [at version v21.03.0][rel] and is production-ready. Apart from the vast open source community, it is being used in production at multiple Fortune 500 companies, and by [Intuit Katlas](https://github.com/intuit/katlas) and [VMware Purser](https://github.com/vmware/purser). -[rel]: https://github.com/dgraph-io/dgraph/releases +[rel]: https://github.com/dgraph-io/dgraph/releases/tag/v21.03.0 ## Quick Install @@ -47,14 +47,16 @@ docker pull dgraph/dgraph:latest If you want to install from source, install Go 1.13+ or later and the following dependencies: -Ubuntu: +### Ubuntu + ```bash sudo apt-get update sudo apt-get install gcc make ``` -Then clone the Dgraph repository and use `make install` to install the Dgraph binary to `$GOPATH/bin`. +### Build and Install +Then clone the Dgraph repository and use `make install` to install the Dgraph binary to `$GOPATH/bin`. ```bash git clone https://github.com/dgraph-io/dgraph.git @@ -72,7 +74,7 @@ presentation videos on [YouTube channel](https://www.youtube.com/channel/UCghE41 ## Is Dgraph the right choice for me? -- Do you have more than 10 SQL tables, connected to each other via foreign keys? +- Do you have more than 10 SQL tables connected via foreign keys? - Do you have sparse data, which doesn't elegantly fit into SQL tables? - Do you want a simple and flexible schema, which is readable and maintainable over time? @@ -80,8 +82,8 @@ presentation videos on [YouTube channel](https://www.youtube.com/channel/UCghE41 If the answers to the above are YES, then Dgraph would be a great fit for your application. Dgraph provides NoSQL like scalability while providing SQL like -transactions and ability to select, filter and aggregate data points. It -combines that with distributed joins, traversals and graph operations, which +transactions and the ability to select, filter, and aggregate data points. It +combines that with distributed joins, traversals, and graph operations, which makes it easy to build applications with it. ## Dgraph compared to other graph DBs @@ -94,7 +96,7 @@ makes it easy to build applications with it. | Language | GraphQL inspired | Cypher, Gremlin | Gremlin | | Protocols | Grpc / HTTP + JSON / RDF | Bolt + Cypher | Websocket / HTTP | | Transactions | Distributed ACID transactions | Single server ACID transactions | Not typically ACID -| Full Text Search | Native support | Native support | Via External Indexing System | +| Full-Text Search | Native support | Native support | Via External Indexing System | | Regular Expressions | Native support | Native support | Via External Indexing System | | Geo Search | Native support | External support only | Via External Indexing System | | License | Apache 2.0 | GPL v3 | Apache 2.0 | @@ -107,9 +109,9 @@ makes it easy to build applications with it. play.dgraph.io](http://play.dgraph.io/). - Please see [releases tab](https://github.com/dgraph-io/dgraph/releases) to find the latest release and corresponding release notes. -- [See the Roadmap](https://discuss.dgraph.io/t/product-roadmap-2020/8479) for list of +- [See the Roadmap](https://discuss.dgraph.io/t/dgraph-product-roadmap-2021/12284) for a list of working and planned features. -- Read about the latest updates from Dgraph team [on our +- Read about the latest updates from the Dgraph team [on our blog](https://open.dgraph.io/). - Watch tech talks on our [YouTube channel](https://www.youtube.com/channel/UCghE41LR8nkKFlR3IFTRO4w/featured). @@ -119,7 +121,7 @@ makes it easy to build applications with it. - Please see [Contributing to Dgraph](https://github.com/dgraph-io/dgraph/blob/master/CONTRIBUTING.md) for guidelines on contributions. ## Client Libraries -The Dgraph team maintain a number of [officially supported client libraries](https://dgraph.io/docs/clients/). There are also libraries contributed by the community [unofficial client libraries](https://dgraph.io/docs/clients#unofficial-dgraph-clients). +The Dgraph team maintains several [officially supported client libraries](https://dgraph.io/docs/clients/). There are also libraries contributed by the community [unofficial client libraries](https://dgraph.io/docs/clients#unofficial-dgraph-clients). ## Contact - Please use [discuss.dgraph.io](https://discuss.dgraph.io) for documentation, questions, feature requests and discussions. diff --git a/algo/heap.go b/algo/heap.go deleted file mode 100644 index 7c36dcad0e9..00000000000 --- a/algo/heap.go +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright 2016-2018 Dgraph Labs, Inc. and Contributors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package algo - -import ( - "github.com/dgraph-io/dgraph/codec" -) - -type elem struct { - val uint64 // Value of this element. - listIdx int // Which list this element comes from. - - // The following fields are only used when merging pb.UidPack objects. - decoder *codec.Decoder // pointer to the decoder for this pb.UidPack object. - blockIdx int // The current position in the current pb.UidBlock object. - blockUids []uint64 // The current block. -} - -type uint64Heap []elem - -func (h uint64Heap) Len() int { return len(h) } -func (h uint64Heap) Less(i, j int) bool { return h[i].val < h[j].val } -func (h uint64Heap) Swap(i, j int) { h[i], h[j] = h[j], h[i] } -func (h *uint64Heap) Push(x interface{}) { - *h = append(*h, x.(elem)) -} - -func (h *uint64Heap) Pop() interface{} { - old := *h - n := len(old) - x := old[n-1] - *h = old[0 : n-1] - return x -} diff --git a/algo/heap_test.go b/algo/heap_test.go deleted file mode 100644 index b677f3dfea6..00000000000 --- a/algo/heap_test.go +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright 2016-2018 Dgraph Labs, Inc. and Contributors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package algo - -import ( - "container/heap" - "testing" - - "github.com/stretchr/testify/require" -) - -func TestPush(t *testing.T) { - h := &uint64Heap{} - heap.Init(h) - - e := elem{val: 5} - heap.Push(h, e) - e.val = 3 - heap.Push(h, e) - e.val = 4 - heap.Push(h, e) - - require.Equal(t, h.Len(), 3) - require.EqualValues(t, (*h)[0].val, 3) - - e.val = 10 - (*h)[0] = e - heap.Fix(h, 0) - require.EqualValues(t, (*h)[0].val, 4) - - e.val = 11 - (*h)[0] = e - heap.Fix(h, 0) - require.EqualValues(t, (*h)[0].val, 5) - - e = heap.Pop(h).(elem) - require.EqualValues(t, e.val, 5) - - e = heap.Pop(h).(elem) - require.EqualValues(t, e.val, 10) - - e = heap.Pop(h).(elem) - require.EqualValues(t, e.val, 11) - - require.Equal(t, h.Len(), 0) -} diff --git a/algo/packed.go b/algo/packed.go deleted file mode 100644 index d87e943ee10..00000000000 --- a/algo/packed.go +++ /dev/null @@ -1,341 +0,0 @@ -/* - * Copyright 2019 Dgraph Labs, Inc. and Contributors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package algo - -import ( - "container/heap" - "sort" - - "github.com/dgraph-io/dgraph/codec" - "github.com/dgraph-io/dgraph/protos/pb" -) - -// ApplyFilterPacked applies the filter to a list of packed uids. -func ApplyFilterPacked(u *pb.UidPack, f func(uint64, int) bool) *pb.UidPack { - index := 0 - decoder := codec.NewDecoder(u) - encoder := codec.Encoder{BlockSize: int(u.BlockSize)} - - for ; decoder.Valid(); decoder.Next() { - for _, uid := range decoder.Uids() { - if f(uid, index) { - encoder.Add(uid) - } - index++ - } - } - - return encoder.Done() -} - -// IntersectWithLinPacked performs the liner intersection between two compressed uid lists. -func IntersectWithLinPacked(u, v *pb.UidPack) *pb.UidPack { - if u == nil || v == nil { - return nil - } - - uDec := codec.NewDecoder(u) - uuids := uDec.Uids() - vDec := codec.NewDecoder(v) - vuids := vDec.Uids() - uIdx, vIdx := 0, 0 - result := codec.Encoder{BlockSize: int(u.BlockSize)} - - for { - // Break if the end of a list has been reached. - if len(uuids) == 0 || len(vuids) == 0 { - break - } - - // Load the next block of the encoded lists if necessary. - if uIdx == len(uuids) { - if uDec.Valid() { - uuids = uDec.Next() - uIdx = 0 - } else { - break - } - - } - if vIdx == len(vuids) { - if vDec.Valid() { - vuids = vDec.Next() - vIdx = 0 - } else { - break - } - } - - uLen := len(uuids) - vLen := len(vuids) - - for uIdx < uLen && vIdx < vLen { - uid := uuids[uIdx] - vid := vuids[vIdx] - switch { - case uid > vid: - for vIdx = vIdx + 1; vIdx < vLen && vuids[vIdx] < uid; vIdx++ { - } - case uid == vid: - result.Add(uid) - vIdx++ - uIdx++ - default: - for uIdx = uIdx + 1; uIdx < uLen && uuids[uIdx] < vid; uIdx++ { - } - } - } - } - return result.Done() -} - -// listInfoPacked stores the packed list in a format that allows lists to be sorted by size. -type listInfoPacked struct { - l *pb.UidPack - length int -} - -// IntersectSortedPacked calculates the intersection of multiple lists and performs -// the intersections from the smallest to the largest list. -func IntersectSortedPacked(lists []*pb.UidPack) *pb.UidPack { - if len(lists) == 0 { - encoder := codec.Encoder{BlockSize: 10} - return encoder.Done() - } - ls := make([]listInfoPacked, 0, len(lists)) - for _, list := range lists { - ls = append(ls, listInfoPacked{ - l: list, - length: codec.ExactLen(list), - }) - } - // Sort the lists based on length. - sort.Slice(ls, func(i, j int) bool { - return ls[i].length < ls[j].length - }) - - if len(ls) == 1 { - // Return a copy of the UidPack. - return codec.CopyUidPack(ls[0].l) - } - - // TODO(martinmr): Add the rest of the algorithms. - out := IntersectWithLinPacked(ls[0].l, ls[1].l) - // Intersect from smallest to largest. - for i := 2; i < len(ls); i++ { - out := IntersectWithLinPacked(out, ls[i].l) - // Break if we reach size 0 as we can no longer - // add any element. - if codec.ExactLen(out) == 0 { - break - } - } - return out -} - -// DifferencePacked performs the difference operation between two UidPack objects. -func DifferencePacked(u, v *pb.UidPack) *pb.UidPack { - if u == nil || v == nil { - // If v == nil, then it's empty so the value of u - v is just u. - // Return a copy of u. - if v == nil { - return codec.CopyUidPack(u) - } - - return nil - } - - result := codec.Encoder{BlockSize: int(u.BlockSize)} - - uDec := codec.NewDecoder(u) - uuids := uDec.Uids() - vDec := codec.NewDecoder(v) - vuids := vDec.Uids() - uIdx, vIdx := 0, 0 - - for { - // Break if the end of a list has been reached. - if len(uuids) == 0 || len(vuids) == 0 { - break - } - - // Load the next block of the encoded lists if necessary. - if uIdx == len(uuids) { - if uDec.Valid() { - uuids = uDec.Next() - uIdx = 0 - } else { - break - } - - } - - if vIdx == len(vuids) { - if vDec.Valid() { - vuids = vDec.Next() - vIdx = 0 - } else { - break - } - } - - uLen := len(uuids) - vLen := len(vuids) - - for uIdx < uLen && vIdx < vLen { - uid := uuids[uIdx] - vid := vuids[vIdx] - - switch { - case uid < vid: - for uIdx < uLen && uuids[uIdx] < vid { - result.Add(uuids[uIdx]) - uIdx++ - } - case uid == vid: - uIdx++ - vIdx++ - default: - vIdx++ - for { - if !(vIdx < vLen && vuids[vIdx] < uid) { - break - } - vIdx++ - } - } - } - - for uIdx < uLen && vIdx >= vLen { - result.Add(uuids[uIdx]) - uIdx++ - } - } - - return result.Done() -} - -// MergeSortedPacked merges already sorted UidPack objects into a single UidPack. -func MergeSortedPacked(lists []*pb.UidPack) *pb.UidPack { - if len(lists) == 0 { - return nil - } - - h := &uint64Heap{} - heap.Init(h) - maxSz := 0 - blockSize := 0 - - for i, l := range lists { - if l == nil { - continue - } - if blockSize == 0 { - blockSize = int(l.BlockSize) - } - decoder := codec.NewDecoder(lists[i]) - block := decoder.Uids() - if len(block) == 0 { - continue - } - - listLen := codec.ExactLen(lists[i]) - if listLen > 0 { - heap.Push(h, elem{ - val: block[0], - listIdx: i, - decoder: decoder, - blockIdx: 0, - blockUids: block, - }) - if listLen > maxSz { - maxSz = listLen - } - } - } - - // Our final result. - result := codec.Encoder{BlockSize: blockSize} - // emptyResult is used to keep track of whether the encoder contains data since the - // encoder storing the final result does not have an equivalent of len. - emptyResult := true - var last uint64 // Last element added to sorted / final result. - - for h.Len() > 0 { // While heap is not empty. - me := &(*h)[0] // Peek at the top element in heap. - if emptyResult || me.val != last { - result.Add(me.val) // Add if unique. - last = me.val - emptyResult = false - } - - // Reached the end of this list. Remove it from the heap. - lastBlock := me.decoder.BlockIdx() == len(me.decoder.Pack.GetBlocks())-1 - if me.blockIdx == len(me.blockUids)-1 && lastBlock { - heap.Pop(h) - continue - } - - // Increment counters. - me.blockIdx++ - if me.blockIdx >= len(me.blockUids) { - // Reached the end of the current block. Decode the next block - // and reset the block counter. - me.blockUids = me.decoder.Next() - me.blockIdx = 0 - } - - // Update current value and re-heapify. - me.val = me.blockUids[me.blockIdx] - heap.Fix(h, 0) // Faster than Pop() followed by Push(). - } - - return result.Done() -} - -// IndexOfPacked finds the index of the given uid in the UidPack. If it doesn't find it, -// it returns -1. -func IndexOfPacked(u *pb.UidPack, uid uint64) int { - if u == nil { - return -1 - } - - index := 0 - decoder := codec.Decoder{Pack: u} - decoder.Seek(uid, codec.SeekStart) - // Need to re-unpack this block since Seek might make Uids return an array with missing - // elements in this block. We need them to make the correct calculation. - decoder.UnpackBlock() - - uids := decoder.Uids() - if len(uids) == 0 { - return -1 - } - - searchFunc := func(i int) bool { return uids[i] >= uid } - uidx := sort.Search(len(uids), searchFunc) - if uids[uidx] != uid { - return -1 - } - - index += uidx - for i := 0; i < decoder.BlockIdx(); i++ { - index += int(u.Blocks[i].GetNumUids()) - } - - return index -} diff --git a/algo/packed_test.go b/algo/packed_test.go deleted file mode 100644 index 898dec340f5..00000000000 --- a/algo/packed_test.go +++ /dev/null @@ -1,355 +0,0 @@ -/* - * Copyright 2016-2018 Dgraph Labs, Inc. and Contributors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package algo - -import ( - "math" - "testing" - - "github.com/dgraph-io/dgraph/codec" - "github.com/dgraph-io/dgraph/protos/pb" - "github.com/stretchr/testify/require" -) - -func newUidPack(data []uint64) *pb.UidPack { - // Using a small block size to make sure multiple blocks are used by the tests. - encoder := codec.Encoder{BlockSize: 5} - for _, uid := range data { - encoder.Add(uid) - } - return encoder.Done() -} - -func TestMergeSorted1Packed(t *testing.T) { - input := []*pb.UidPack{ - newUidPack([]uint64{55}), - } - require.Equal(t, []uint64{55}, codec.Decode(MergeSortedPacked(input), 0)) -} - -func printPack(t *testing.T, pack *pb.UidPack) { - for _, block := range pack.Blocks { - t.Logf("[%x]Block base: %d. Num uids: %d. Deltas: %x\n", - pack.AllocRef, block.Base, block.NumUids, block.Deltas) - } -} - -func TestMergeSorted2Packed(t *testing.T) { - input := []*pb.UidPack{ - newUidPack([]uint64{1, 3, 6, 8, 10}), - newUidPack([]uint64{2, 4, 5, 7, 15}), - } - require.Equal(t, []uint64{1, 3, 6, 8, 10}, codec.Decode(input[0], 0)) - require.Equal(t, []uint64{2, 4, 5, 7, 15}, codec.Decode(input[1], 0)) - - // printPack(t, input[1]) - merged := MergeSortedPacked(input) - // printPack(t, merged) - require.Equal(t, []uint64{1, 2, 3, 4, 5, 6, 7, 8, 10, 15}, - codec.Decode(merged, 0)) -} - -func TestMergeSorted3Packed(t *testing.T) { - input := []*pb.UidPack{ - newUidPack([]uint64{1, 3, 6, 8, 10}), - newUidPack([]uint64{}), - } - require.Equal(t, []uint64{1, 3, 6, 8, 10}, codec.Decode(MergeSortedPacked(input), 0)) -} - -func TestMergeSorted4Packed(t *testing.T) { - input := []*pb.UidPack{ - newUidPack([]uint64{}), - newUidPack([]uint64{1, 3, 6, 8, 10}), - } - require.Equal(t, []uint64{1, 3, 6, 8, 10}, codec.Decode(MergeSortedPacked(input), 0)) -} - -func TestMergeSorted5Packed(t *testing.T) { - input := []*pb.UidPack{ - newUidPack([]uint64{}), - newUidPack([]uint64{}), - } - require.Empty(t, codec.Decode(MergeSortedPacked(input), 0)) -} - -func TestMergeSorted6Packed(t *testing.T) { - input := []*pb.UidPack{ - newUidPack([]uint64{11, 13, 16, 18, 20}), - newUidPack([]uint64{12, 14, 15, 15, 16, 16, 17, 25}), - newUidPack([]uint64{1, 2}), - } - require.Equal(t, - []uint64{1, 2, 11, 12, 13, 14, 15, 16, 17, 18, 20, 25}, - codec.Decode(MergeSortedPacked(input), 0)) -} - -func TestMergeSorted7Packed(t *testing.T) { - input := []*pb.UidPack{ - newUidPack([]uint64{5, 6, 7}), - newUidPack([]uint64{3, 4}), - newUidPack([]uint64{1, 2}), - newUidPack([]uint64{}), - } - require.Equal(t, []uint64{1, 2, 3, 4, 5, 6, 7}, codec.Decode(MergeSortedPacked(input), 0)) -} - -func TestMergeSorted8Packed(t *testing.T) { - input := []*pb.UidPack{} - require.Empty(t, codec.Decode(MergeSortedPacked(input), 0)) -} - -func TestMergeSorted9Packed(t *testing.T) { - input := []*pb.UidPack{ - newUidPack([]uint64{1, 1, 1}), - } - require.Equal(t, []uint64{1}, codec.Decode(MergeSortedPacked(input), 0)) -} - -func TestMergeSorted10Packed(t *testing.T) { - input := []*pb.UidPack{ - newUidPack([]uint64{1, 2, 3, 3, 6}), - newUidPack([]uint64{4, 8, 9}), - } - require.Equal(t, []uint64{1, 2, 3, 4, 6, 8, 9}, codec.Decode(MergeSortedPacked(input), 0)) -} - -func TestUIDListIntersect1Packed(t *testing.T) { - u := newUidPack([]uint64{1, 2, 3}) - v := newUidPack([]uint64{}) - o := IntersectWithLinPacked(u, v) - require.Empty(t, codec.Decode(o, 0)) -} - -func TestUIDListIntersect2Packed(t *testing.T) { - u := newUidPack([]uint64{1, 2, 3}) - v := newUidPack([]uint64{1, 2, 3, 4, 5}) - o := IntersectWithLinPacked(u, v) - require.Equal(t, []uint64{1, 2, 3}, codec.Decode(o, 0)) -} - -func TestUIDListIntersect3Packed(t *testing.T) { - u := newUidPack([]uint64{1, 2, 3}) - v := newUidPack([]uint64{2}) - o := IntersectWithLinPacked(u, v) - require.Equal(t, []uint64{2}, codec.Decode(o, 0)) -} - -func TestUIDListIntersect4Packed(t *testing.T) { - u := newUidPack([]uint64{1, 2, 3}) - v := newUidPack([]uint64{0, 5}) - o := IntersectWithLinPacked(u, v) - require.Empty(t, codec.Decode(o, 0)) -} - -func TestUIDListIntersect5Packed(t *testing.T) { - u := newUidPack([]uint64{1, 2, 3}) - v := newUidPack([]uint64{3, 5}) - o := IntersectWithLinPacked(u, v) - require.Equal(t, []uint64{3}, codec.Decode(o, 0)) -} - -func TestUIDListIntersect6Packed(t *testing.T) { - u := newUidPack([]uint64{1, 2, 3, 4, 5, 6, 7, 9}) - v := newUidPack([]uint64{1, 3, 5, 7, 8, 9}) - o := IntersectWithLinPacked(u, v) - require.Equal(t, []uint64{1, 3, 5, 7, 9}, codec.Decode(o, 0)) -} - -func TestUIDListIntersectDupFirstPacked(t *testing.T) { - u := newUidPack([]uint64{1, 1, 2, 3}) - v := newUidPack([]uint64{1, 2}) - o := IntersectWithLinPacked(u, v) - require.Equal(t, []uint64{1, 2}, codec.Decode(o, 0)) -} - -func TestUIDListIntersectDupBothPacked(t *testing.T) { - u := newUidPack([]uint64{1, 1, 2, 3, 5}) - v := newUidPack([]uint64{1, 1, 2, 4}) - o := IntersectWithLinPacked(u, v) - require.Equal(t, []uint64{1, 1, 2}, codec.Decode(o, 0)) -} - -func TestUIDListIntersectDupSecondPacked(t *testing.T) { - u := newUidPack([]uint64{1, 2, 3, 5}) - v := newUidPack([]uint64{1, 1, 2, 4}) - o := IntersectWithLinPacked(u, v) - require.Equal(t, []uint64{1, 2}, codec.Decode(o, 0)) -} - -func TestIntersectSorted1Packed(t *testing.T) { - input := []*pb.UidPack{ - newUidPack([]uint64{1, 2, 3}), - newUidPack([]uint64{2, 3, 4, 5}), - } - require.Equal(t, []uint64{2, 3}, codec.Decode(IntersectSortedPacked(input), 0)) -} - -func TestIntersectSorted2Packed(t *testing.T) { - input := []*pb.UidPack{ - newUidPack([]uint64{1, 2, 3}), - } - require.Equal(t, []uint64{1, 2, 3}, codec.Decode(IntersectSortedPacked(input), 0)) -} - -func TestIntersectSorted3Packed(t *testing.T) { - input := []*pb.UidPack{} - require.Empty(t, codec.Decode(IntersectSortedPacked(input), 0)) -} - -func TestIntersectSorted4Packed(t *testing.T) { - input := []*pb.UidPack{ - newUidPack([]uint64{100, 101}), - } - require.Equal(t, []uint64{100, 101}, codec.Decode(IntersectSortedPacked(input), 0)) -} - -func TestIntersectSorted5Packed(t *testing.T) { - input := []*pb.UidPack{ - newUidPack([]uint64{1, 2, 3}), - newUidPack([]uint64{2, 3, 4, 5}), - newUidPack([]uint64{4, 5, 6}), - } - require.Empty(t, codec.Decode(IntersectSortedPacked(input), 0)) -} - -func TestIntersectSorted6Packed(t *testing.T) { - input := []*pb.UidPack{ - newUidPack([]uint64{10, 12, 13}), - newUidPack([]uint64{2, 3, 4, 13}), - newUidPack([]uint64{4, 5, 6}), - } - require.Empty(t, codec.Decode(IntersectSortedPacked(input), 0)) -} - -func TestIntersectSorted7Packed(t *testing.T) { - input := []*pb.UidPack{ - newUidPack([]uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}), - newUidPack([]uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}), - newUidPack([]uint64{1, 2, 3, 4, 5, 6, 7, 8, 9}), - newUidPack([]uint64{1, 2, 3, 4, 5, 6, 7, 8}), - newUidPack([]uint64{1, 2, 3, 4, 5, 6, 7}), - newUidPack([]uint64{1, 2, 3, 4, 5, 6}), - newUidPack([]uint64{1, 2, 3, 4, 5}), - newUidPack([]uint64{1, 2, 3, 4}), - newUidPack([]uint64{1, 2, 3}), - newUidPack([]uint64{1, 2}), - newUidPack([]uint64{1}), - } - require.Equal(t, []uint64{1}, codec.Decode(IntersectSortedPacked(input), 0)) -} - -func TestDiffSorted1Packed(t *testing.T) { - input := []*pb.UidPack{ - newUidPack([]uint64{1, 2, 3}), - newUidPack([]uint64{1}), - } - output := DifferencePacked(input[0], input[1]) - require.Equal(t, []uint64{2, 3}, codec.Decode(output, 0)) -} - -func TestDiffSorted2Packed(t *testing.T) { - input := []*pb.UidPack{ - newUidPack([]uint64{1, 2, 3}), - newUidPack([]uint64{2}), - } - output := DifferencePacked(input[0], input[1]) - require.Equal(t, []uint64{1, 3}, codec.Decode(output, 0)) -} - -func TestDiffSorted3Packed(t *testing.T) { - input := []*pb.UidPack{ - newUidPack([]uint64{1, 2, 3}), - newUidPack([]uint64{3}), - } - output := DifferencePacked(input[0], input[1]) - require.Equal(t, []uint64{1, 2}, codec.Decode(output, 0)) -} - -func TestDiffSorted4Packed(t *testing.T) { - input := []*pb.UidPack{ - newUidPack([]uint64{1, 2, 3}), - newUidPack([]uint64{}), - } - output := DifferencePacked(input[0], input[1]) - require.Equal(t, []uint64{1, 2, 3}, codec.Decode(output, 0)) -} - -func TestDiffSorted5Packed(t *testing.T) { - input := []*pb.UidPack{ - newUidPack([]uint64{}), - newUidPack([]uint64{1, 2}), - } - output := DifferencePacked(input[0], input[1]) - require.Equal(t, []uint64{}, codec.Decode(output, 0)) -} - -func TestSubSorted1Packed(t *testing.T) { - input := []*pb.UidPack{ - newUidPack([]uint64{1, 2, 3}), - newUidPack([]uint64{2, 3, 4, 5}), - } - output := DifferencePacked(input[0], input[1]) - require.Equal(t, []uint64{1}, codec.Decode(output, 0)) -} - -func TestSubSorted6Packed(t *testing.T) { - input := []*pb.UidPack{ - newUidPack([]uint64{10, 12, 13}), - newUidPack([]uint64{2, 3, 4, 13}), - } - output := DifferencePacked(input[0], input[1]) - require.Equal(t, []uint64{10, 12}, codec.Decode(output, 0)) -} - -func TestIndexOfPacked1(t *testing.T) { - encoder := codec.Encoder{BlockSize: 10} - for i := 0; i < 1000; i++ { - encoder.Add(uint64(i)) - } - pack := encoder.Done() - - for i := 0; i < 1000; i++ { - require.Equal(t, i, IndexOfPacked(pack, uint64(i))) - } - require.Equal(t, -1, IndexOfPacked(pack, 1000)) -} - -func TestIndexOfPacked2(t *testing.T) { - encoder := codec.Encoder{BlockSize: 10} - for i := 0; i < 100; i++ { - encoder.Add(uint64(i)) - } - pack := encoder.Done() - - require.Equal(t, -1, IndexOfPacked(pack, 100)) - require.Equal(t, -1, IndexOfPacked(pack, 101)) - require.Equal(t, -1, IndexOfPacked(pack, 1000)) - require.Equal(t, -1, IndexOfPacked(pack, math.MaxUint64)) -} - -func TestIndexOfPacked3(t *testing.T) { - require.Equal(t, -1, IndexOfPacked(nil, 0)) - require.Equal(t, -1, IndexOfPacked(nil, math.MaxUint64)) -} - -func TestApplyFilterUintPacked(t *testing.T) { - l := []uint64{1, 2, 3, 4, 5, 6, 7, 8, 9} - u := newUidPack(l) - res := ApplyFilterPacked(u, func(a uint64, idx int) bool { return (l[idx] % 2) == 1 }) - require.Equal(t, []uint64{1, 3, 5, 7, 9}, codec.Decode(res, 0)) -} diff --git a/algo/uidlist.go b/algo/uidlist.go index e2ca6dbc906..bb494f56d66 100644 --- a/algo/uidlist.go +++ b/algo/uidlist.go @@ -17,412 +17,37 @@ package algo import ( - "container/heap" - "sort" - "github.com/dgraph-io/dgraph/codec" "github.com/dgraph-io/dgraph/protos/pb" + "github.com/dgraph-io/sroar" ) const jump = 32 // Jump size in InsersectWithJump. // ApplyFilter applies a filter to our UIDList. +// TODO: ApplyFilter in this way should only happen for sorted uids. For normal +// filter, it should use Bitmap FastAnd or And. func ApplyFilter(u *pb.List, f func(uint64, int) bool) { - out := u.Uids[:0] - for i, uid := range u.Uids { - if f(uid, i) { - out = append(out, uid) - } - } - u.Uids = out -} - -// IntersectCompressedWith intersects a packed list of UIDs with another list -// and writes the output to o. -func IntersectCompressedWith(pack *pb.UidPack, afterUID uint64, v, o *pb.List) { - if pack == nil { - return - } - dec := codec.Decoder{Pack: pack} - dec.Seek(afterUID, codec.SeekStart) - n := dec.ApproxLen() - m := len(v.Uids) - - if n > m { - n, m = m, n - } - dst := o.Uids[:0] - - // If n equals 0, set it to 1 to avoid division by zero. - if n == 0 { - n = 1 - } - - // Select appropriate function based on heuristics. - ratio := float64(m) / float64(n) - if ratio < 500 { - IntersectCompressedWithLinJump(&dec, v.Uids, &dst) - } else { - IntersectCompressedWithBin(&dec, v.Uids, &dst) - } - o.Uids = dst -} - -// IntersectCompressedWithLinJump performs the intersection linearly. -func IntersectCompressedWithLinJump(dec *codec.Decoder, v []uint64, o *[]uint64) { - m := len(v) - k := 0 - _, off := IntersectWithLin(dec.Uids(), v[k:], o) - k += off - - for k < m { - u := dec.LinearSeek(v[k]) - if len(u) == 0 { - break - } - _, off := IntersectWithLin(u, v[k:], o) - if off == 0 { - off = 1 // If v[k] isn't in u, move forward. + uids := codec.GetUids(u) + var out []uint64 + for i, x := range uids { + if f(x, i) { + out = append(out, x) } - - k += off - } -} - -// IntersectCompressedWithBin is based on the paper -// "Fast Intersection Algorithms for Sorted Sequences" -// https://link.springer.com/chapter/10.1007/978-3-642-12476-1_3 -func IntersectCompressedWithBin(dec *codec.Decoder, q []uint64, o *[]uint64) { - ld := dec.ApproxLen() - lq := len(q) - - if lq == 0 { - // Not checking whether ld == 0 because that is an approximate size. - // If the actual length of the list is zero, the for loop below will - // do nothing as expected. - return - } - - // Pick the shorter list and do binary search - if ld < lq { - uids := dec.Uids() - for len(uids) > 0 { - for _, u := range uids { - qidx := sort.Search(len(q), func(idx int) bool { - return q[idx] >= u - }) - if qidx >= len(q) { - return - } - if q[qidx] == u { - *o = append(*o, u) - qidx++ - } - q = q[qidx:] - } - uids = dec.Next() - } - return - } - - for _, u := range q { - uids := dec.Seek(u, codec.SeekStart) - if len(uids) == 0 { - return - } - if uids[0] == u { - *o = append(*o, u) - } - } -} - -// IntersectWith intersects u with v. The update is made to o. -// u, v should be sorted. -func IntersectWith(u, v, o *pb.List) { - n := len(u.Uids) - m := len(v.Uids) - - if n > m { - n, m = m, n - } - if o.Uids == nil { - o.Uids = make([]uint64, 0, n) - } - dst := o.Uids[:0] - if n == 0 { - n = 1 - } - // Select appropriate function based on heuristics. - ratio := float64(m) / float64(n) - switch { - case ratio < 100: - IntersectWithLin(u.Uids, v.Uids, &dst) - case ratio < 500: - IntersectWithJump(u.Uids, v.Uids, &dst) - default: - IntersectWithBin(u.Uids, v.Uids, &dst) - } - o.Uids = dst -} - -// IntersectWithLin performs the intersection linearly. -func IntersectWithLin(u, v []uint64, o *[]uint64) (int, int) { - n := len(u) - m := len(v) - i, k := 0, 0 - for i < n && k < m { - uid := u[i] - vid := v[k] - switch { - case uid > vid: - for k = k + 1; k < m && v[k] < uid; k++ { - } - case uid == vid: - *o = append(*o, uid) - k++ - i++ - default: - for i = i + 1; i < n && u[i] < vid; i++ { - } - } - } - return i, k -} - -// IntersectWithJump performs the intersection linearly but jumping jump steps -// between iterations. -func IntersectWithJump(u, v []uint64, o *[]uint64) (int, int) { - n := len(u) - m := len(v) - i, k := 0, 0 - for i < n && k < m { - uid := u[i] - vid := v[k] - switch { - case uid == vid: - *o = append(*o, uid) - k++ - i++ - case k+jump < m && uid > v[k+jump]: - k += jump - case i+jump < n && vid > u[i+jump]: - i += jump - case uid > vid: - for k = k + 1; k < m && v[k] < uid; k++ { - } - default: - for i = i + 1; i < n && u[i] < vid; i++ { - } - } - } - return i, k -} - -// IntersectWithBin is based on the paper -// "Fast Intersection Algorithms for Sorted Sequences" -// https://link.springer.com/chapter/10.1007/978-3-642-12476-1_3 -func IntersectWithBin(d, q []uint64, o *[]uint64) { - ld := len(d) - lq := len(q) - - if ld < lq { - ld, lq = lq, ld - d, q = q, d - } - if ld == 0 || lq == 0 || d[ld-1] < q[0] || q[lq-1] < d[0] { - return - } - - val := d[0] - minq := sort.Search(len(q), func(i int) bool { - return q[i] >= val - }) - - val = d[len(d)-1] - maxq := sort.Search(len(q), func(i int) bool { - return q[i] > val - }) - - binIntersect(d, q[minq:maxq], o) -} - -// binIntersect is the recursive function used. -// NOTE: len(d) >= len(q) (Must hold) -func binIntersect(d, q []uint64, final *[]uint64) { - if len(d) == 0 || len(q) == 0 { - return - } - midq := len(q) / 2 - qval := q[midq] - midd := sort.Search(len(d), func(i int) bool { - return d[i] >= qval - }) - - dd := d[0:midd] - qq := q[0:midq] - if len(dd) > len(qq) { // D > Q - binIntersect(dd, qq, final) - } else { - binIntersect(qq, dd, final) - } - - if midd >= len(d) { - return - } - if d[midd] == qval { - *final = append(*final, qval) - } else { - midd-- } - dd = d[midd+1:] - qq = q[midq+1:] - if len(dd) > len(qq) { // D > Q - binIntersect(dd, qq, final) + if len(u.SortedUids) > 0 { + u.SortedUids = out } else { - binIntersect(qq, dd, final) + b := sroar.NewBitmap() + b.SetMany(out) + u.Bitmap = b.ToBuffer() } } -type listInfo struct { - l *pb.List - length int -} - -// IntersectSorted calculates the intersection of multiple lists and performs -// the intersections from the smallest to the largest list. -func IntersectSorted(lists []*pb.List) *pb.List { - if len(lists) == 0 { - return &pb.List{} - } - ls := make([]listInfo, 0, len(lists)) - for _, list := range lists { - ls = append(ls, listInfo{ - l: list, - length: len(list.Uids), - }) - } - // Sort the lists based on length. - sort.Slice(ls, func(i, j int) bool { - return ls[i].length < ls[j].length - }) - out := &pb.List{Uids: make([]uint64, ls[0].length)} - if len(ls) == 1 { - copy(out.Uids, ls[0].l.Uids) - return out - } - - IntersectWith(ls[0].l, ls[1].l, out) - // Intersect from smallest to largest. - for i := 2; i < len(ls); i++ { - IntersectWith(out, ls[i].l, out) - // Break if we reach size 0 as we can no longer - // add any element. - if len(out.Uids) == 0 { - break - } - } - return out -} - -// Difference returns the difference of two lists. -func Difference(u, v *pb.List) *pb.List { - if u == nil || v == nil { - return &pb.List{Uids: make([]uint64, 0)} - } - n := len(u.Uids) - m := len(v.Uids) - out := make([]uint64, 0, n/2) - i, k := 0, 0 - for i < n && k < m { - uid := u.Uids[i] - vid := v.Uids[k] - switch { - case uid < vid: - for i < n && u.Uids[i] < vid { - out = append(out, u.Uids[i]) - i++ - } - case uid == vid: - i++ - k++ - default: - for k = k + 1; k < m && v.Uids[k] < uid; k++ { - } - } - } - for i < n && k >= m { - out = append(out, u.Uids[i]) - i++ - } - return &pb.List{Uids: out} -} - -// MergeSorted merges sorted lists. -func MergeSorted(lists []*pb.List) *pb.List { - if len(lists) == 0 { - return new(pb.List) - } - - h := &uint64Heap{} - heap.Init(h) - maxSz := 0 - - for i, l := range lists { - if l == nil { - continue - } - lenList := len(l.Uids) - if lenList > 0 { - heap.Push(h, elem{ - val: l.Uids[0], - listIdx: i, - }) - if lenList > maxSz { - maxSz = lenList - } - } - } - - // Our final output. Give it an approximate capacity as copies are expensive. - output := make([]uint64, 0, maxSz) - // idx[i] is the element we are looking at for lists[i]. - idx := make([]int, len(lists)) - var last uint64 // Last element added to sorted / final output. - for h.Len() > 0 { // While heap is not empty. - me := (*h)[0] // Peek at the top element in heap. - if len(output) == 0 || me.val != last { - output = append(output, me.val) // Add if unique. - last = me.val - } - l := lists[me.listIdx] - if idx[me.listIdx] >= len(l.Uids)-1 { - heap.Pop(h) - } else { - idx[me.listIdx]++ - val := l.Uids[idx[me.listIdx]] - (*h)[0].val = val - heap.Fix(h, 0) // Faster than Pop() followed by Push(). - } - } - return &pb.List{Uids: output} -} - // IndexOf performs a binary search on the uids slice and returns the index at // which it finds the uid, else returns -1 func IndexOf(u *pb.List, uid uint64) int { - i := sort.Search(len(u.Uids), func(i int) bool { return u.Uids[i] >= uid }) - if i < len(u.Uids) && u.Uids[i] == uid { - return i - } - return -1 -} - -// ToUintsListForTest converts to list of uints for testing purpose only. -func ToUintsListForTest(ul []*pb.List) [][]uint64 { - out := make([][]uint64, 0, len(ul)) - for _, u := range ul { - out = append(out, u.Uids) - } - return out + bm := codec.FromListNoCopy(u) + return bm.Rank(uid) } diff --git a/algo/uidlist_test.go b/algo/uidlist_test.go index 068dea5d505..0fc46d2cd3d 100644 --- a/algo/uidlist_test.go +++ b/algo/uidlist_test.go @@ -17,11 +17,7 @@ package algo import ( - "fmt" - "math/rand" - "sort" "testing" - "time" "github.com/dgraph-io/dgraph/codec" "github.com/dgraph-io/dgraph/protos/pb" @@ -29,497 +25,12 @@ import ( ) func newList(data []uint64) *pb.List { - return &pb.List{Uids: data} -} - -func TestMergeSorted1(t *testing.T) { - input := []*pb.List{ - newList([]uint64{55}), - } - require.Equal(t, MergeSorted(input).Uids, []uint64{55}) -} - -func TestMergeSorted2(t *testing.T) { - input := []*pb.List{ - newList([]uint64{1, 3, 6, 8, 10}), - newList([]uint64{2, 4, 5, 7, 15}), - } - require.Equal(t, MergeSorted(input).Uids, - []uint64{1, 2, 3, 4, 5, 6, 7, 8, 10, 15}) -} - -func TestMergeSorted3(t *testing.T) { - input := []*pb.List{ - newList([]uint64{1, 3, 6, 8, 10}), - newList([]uint64{}), - } - require.Equal(t, MergeSorted(input).Uids, []uint64{1, 3, 6, 8, 10}) -} - -func TestMergeSorted4(t *testing.T) { - input := []*pb.List{ - newList([]uint64{}), - newList([]uint64{1, 3, 6, 8, 10}), - } - require.Equal(t, MergeSorted(input).Uids, []uint64{1, 3, 6, 8, 10}) -} - -func TestMergeSorted5(t *testing.T) { - input := []*pb.List{ - newList([]uint64{}), - newList([]uint64{}), - } - require.Empty(t, MergeSorted(input).Uids) -} - -func TestMergeSorted6(t *testing.T) { - input := []*pb.List{ - newList([]uint64{11, 13, 16, 18, 20}), - newList([]uint64{12, 14, 15, 15, 16, 16, 17, 25}), - newList([]uint64{1, 2}), - } - require.Equal(t, MergeSorted(input).Uids, - []uint64{1, 2, 11, 12, 13, 14, 15, 16, 17, 18, 20, 25}) -} - -func TestMergeSorted7(t *testing.T) { - input := []*pb.List{ - newList([]uint64{5, 6, 7}), - newList([]uint64{3, 4}), - newList([]uint64{1, 2}), - newList([]uint64{}), - } - require.Equal(t, MergeSorted(input).Uids, []uint64{1, 2, 3, 4, 5, 6, 7}) -} - -func TestMergeSorted8(t *testing.T) { - input := []*pb.List{} - require.Empty(t, MergeSorted(input).Uids) -} - -func TestMergeSorted9(t *testing.T) { - input := []*pb.List{ - newList([]uint64{1, 1, 1}), - } - require.Equal(t, MergeSorted(input).Uids, []uint64{1}) -} - -func TestMergeSorted10(t *testing.T) { - input := []*pb.List{ - newList([]uint64{1, 2, 3, 3, 6}), - newList([]uint64{4, 8, 9}), - } - require.Equal(t, MergeSorted(input).Uids, []uint64{1, 2, 3, 4, 6, 8, 9}) -} - -func TestIntersectSorted1(t *testing.T) { - input := []*pb.List{ - newList([]uint64{1, 2, 3}), - newList([]uint64{2, 3, 4, 5}), - } - require.Equal(t, []uint64{2, 3}, IntersectSorted(input).Uids) -} - -func TestIntersectSorted2(t *testing.T) { - input := []*pb.List{ - newList([]uint64{1, 2, 3}), - } - require.Equal(t, IntersectSorted(input).Uids, []uint64{1, 2, 3}) -} - -func TestIntersectSorted3(t *testing.T) { - input := []*pb.List{} - require.Empty(t, IntersectSorted(input).Uids) -} - -func TestIntersectSorted4(t *testing.T) { - input := []*pb.List{ - newList([]uint64{100, 101}), - } - require.Equal(t, IntersectSorted(input).Uids, []uint64{100, 101}) -} - -func TestIntersectSorted5(t *testing.T) { - input := []*pb.List{ - newList([]uint64{1, 2, 3}), - newList([]uint64{2, 3, 4, 5}), - newList([]uint64{4, 5, 6}), - } - require.Empty(t, IntersectSorted(input).Uids) -} - -func TestIntersectSorted6(t *testing.T) { - input := []*pb.List{ - newList([]uint64{10, 12, 13}), - newList([]uint64{2, 3, 4, 13}), - newList([]uint64{4, 5, 6}), - } - require.Empty(t, IntersectSorted(input).Uids) -} - -func TestDiffSorted1(t *testing.T) { - input := []*pb.List{ - newList([]uint64{1, 2, 3}), - newList([]uint64{1}), - } - output := Difference(input[0], input[1]) - require.Equal(t, []uint64{2, 3}, output.Uids) -} - -func TestDiffSorted2(t *testing.T) { - input := []*pb.List{ - newList([]uint64{1, 2, 3}), - newList([]uint64{2}), - } - output := Difference(input[0], input[1]) - require.Equal(t, []uint64{1, 3}, output.Uids) -} - -func TestDiffSorted3(t *testing.T) { - input := []*pb.List{ - newList([]uint64{1, 2, 3}), - newList([]uint64{3}), - } - output := Difference(input[0], input[1]) - require.Equal(t, []uint64{1, 2}, output.Uids) -} - -func TestDiffSorted4(t *testing.T) { - input := []*pb.List{ - newList([]uint64{1, 2, 3}), - newList([]uint64{}), - } - output := Difference(input[0], input[1]) - require.Equal(t, []uint64{1, 2, 3}, output.Uids) -} - -func TestDiffSorted5(t *testing.T) { - input := []*pb.List{ - newList([]uint64{}), - newList([]uint64{1, 2}), - } - output := Difference(input[0], input[1]) - require.Equal(t, []uint64{}, output.Uids) -} - -func TestSubSorted1(t *testing.T) { - input := []*pb.List{ - newList([]uint64{1, 2, 3}), - newList([]uint64{2, 3, 4, 5}), - } - output := Difference(input[0], input[1]) - require.Equal(t, []uint64{1}, output.Uids) -} - -func TestSubSorted6(t *testing.T) { - input := []*pb.List{ - newList([]uint64{10, 12, 13}), - newList([]uint64{2, 3, 4, 13}), - } - output := Difference(input[0], input[1]) - require.Equal(t, []uint64{10, 12}, output.Uids) -} - -func TestUIDListIntersect1(t *testing.T) { - u := newList([]uint64{1, 2, 3}) - v := newList([]uint64{}) - IntersectWith(u, v, u) - require.Empty(t, u.Uids) -} - -func TestUIDListIntersect2(t *testing.T) { - u := newList([]uint64{1, 2, 3}) - v := newList([]uint64{1, 2, 3, 4, 5}) - IntersectWith(u, v, u) - require.Equal(t, []uint64{1, 2, 3}, u.Uids) - require.Equal(t, []uint64{1, 2, 3, 4, 5}, v.Uids) -} - -func TestUIDListIntersect3(t *testing.T) { - u := newList([]uint64{1, 2, 3}) - v := newList([]uint64{2}) - IntersectWith(u, v, u) - require.Equal(t, []uint64{2}, u.Uids) - require.Equal(t, []uint64{2}, v.Uids) -} - -func TestUIDListIntersect4(t *testing.T) { - u := newList([]uint64{1, 2, 3}) - v := newList([]uint64{0, 5}) - IntersectWith(u, v, u) - require.Empty(t, u.Uids) - require.Equal(t, []uint64{0, 5}, v.Uids) -} - -func TestUIDListIntersect5(t *testing.T) { - u := newList([]uint64{1, 2, 3}) - v := newList([]uint64{3, 5}) - IntersectWith(u, v, u) - require.Equal(t, []uint64{3}, u.Uids) -} - -func TestUIDListIntersectDupFirst(t *testing.T) { - u := newList([]uint64{1, 1, 2, 3}) - v := newList([]uint64{1, 2}) - IntersectWith(u, v, u) - require.Equal(t, []uint64{1, 2}, u.Uids) -} - -func TestUIDListIntersectDupBoth(t *testing.T) { - u := newList([]uint64{1, 1, 2, 3, 5}) - v := newList([]uint64{1, 1, 2, 4}) - IntersectWith(u, v, u) - require.Equal(t, []uint64{1, 1, 2}, u.Uids) -} - -func TestUIDListIntersectDupSecond(t *testing.T) { - u := newList([]uint64{1, 2, 3, 5}) - v := newList([]uint64{1, 1, 2, 4}) - IntersectWith(u, v, u) - require.Equal(t, []uint64{1, 2}, u.Uids) + return &pb.List{SortedUids: data} } func TestApplyFilterUint(t *testing.T) { l := []uint64{1, 2, 3, 4, 5} u := newList(l) ApplyFilter(u, func(a uint64, idx int) bool { return (l[idx] % 2) == 1 }) - require.Equal(t, []uint64{1, 3, 5}, u.Uids) -} - -// Benchmarks for IntersectWith -func BenchmarkListIntersectRandom(b *testing.B) { - randomTests := func(arrSz int, overlap float64) { - limit := int64(float64(arrSz) / overlap) - u1, v1 := make([]uint64, arrSz), make([]uint64, arrSz) - for i := 0; i < arrSz; i++ { - u1[i] = uint64(rand.Int63n(limit)) - v1[i] = uint64(rand.Int63n(limit)) - } - sort.Slice(u1, func(i, j int) bool { return u1[i] < u1[j] }) - sort.Slice(v1, func(i, j int) bool { return v1[i] < v1[j] }) - - u := newList(u1) - v := newList(v1) - dst1 := &pb.List{} - dst2 := &pb.List{} - compressedUids := codec.Encode(u1, 256) - - b.Run(fmt.Sprintf(":size=%d:overlap=%.2f:", arrSz, overlap), - func(b *testing.B) { - for k := 0; k < b.N; k++ { - IntersectWith(u, v, dst1) - } - }) - - b.Run(fmt.Sprintf(":compressed:size=%d:overlap=%.2f:", arrSz, overlap), - func(b *testing.B) { - for k := 0; k < b.N; k++ { - IntersectCompressedWith(compressedUids, 0, v, dst2) - } - }) - i := 0 - j := 0 - for i < len(dst1.Uids) { - if dst1.Uids[i] != dst2.Uids[j] { - b.Errorf("Unexpected error in intersection") - } - // Behaviour of bin intersect is not defined when duplicates are present - i = skipDuplicate(dst1.Uids, i) - j = skipDuplicate(dst2.Uids, j) - } - if j < len(dst2.Uids) { - b.Errorf("Unexpected error in intersection") - } - - codec.FreePack(compressedUids) - } - - randomTests(10240, 0.3) - randomTests(1024000, 0.3) - randomTests(10240, 0.1) - randomTests(1024000, 0.1) - randomTests(10240, 0.01) - randomTests(1024000, 0.01) -} - -func BenchmarkListIntersectRatio(b *testing.B) { - randomTests := func(sz int, overlap float64) { - rs := []int{1, 10, 50, 100, 500, 1000, 10000, 100000, 1000000} - for _, r := range rs { - sz1 := sz - sz2 := sz * r - if sz2 > 1000000 { - break - } - - u1, v1 := make([]uint64, sz1), make([]uint64, sz2) - limit := int64(float64(sz) / overlap) - for i := 0; i < sz1; i++ { - u1[i] = uint64(rand.Int63n(limit)) - } - for i := 0; i < sz2; i++ { - v1[i] = uint64(rand.Int63n(limit)) - } - sort.Slice(u1, func(i, j int) bool { return u1[i] < u1[j] }) - sort.Slice(v1, func(i, j int) bool { return v1[i] < v1[j] }) - - u := &pb.List{Uids: u1} - v := &pb.List{Uids: v1} - dst1 := &pb.List{} - dst2 := &pb.List{} - compressedUids := codec.Encode(v1, 256) - - fmt.Printf("len: %d, compressed: %d, bytes/int: %f\n", - len(v1), compressedUids.Size(), float64(compressedUids.Size())/float64(len(v1))) - b.Run(fmt.Sprintf(":IntersectWith:ratio=%d:size=%d:overlap=%.2f:", r, sz, overlap), - func(b *testing.B) { - for k := 0; k < b.N; k++ { - IntersectWith(u, v, dst1) - } - }) - b.Run(fmt.Sprintf("compressed:IntersectWith:ratio=%d:size=%d:overlap=%.2f:", r, sz, overlap), - func(b *testing.B) { - for k := 0; k < b.N; k++ { - IntersectCompressedWith(compressedUids, 0, u, dst2) - } - }) - fmt.Println() - i := 0 - j := 0 - for i < len(dst1.Uids) { - if dst1.Uids[i] != dst2.Uids[j] { - b.Errorf("Unexpected error in intersection") - } - // Behaviour of bin intersect is not defined when duplicates are present - i = skipDuplicate(dst1.Uids, i) - j = skipDuplicate(dst2.Uids, j) - } - if j < len(dst2.Uids) { - b.Errorf("Unexpected error in intersection") - } - - codec.FreePack(compressedUids) - } - } - - randomTests(10, 0.01) - randomTests(100, 0.01) - randomTests(1000, 0.01) - randomTests(10000, 0.01) - randomTests(100000, 0.01) - randomTests(1000000, 0.01) -} - -func skipDuplicate(in []uint64, idx int) int { - i := idx + 1 - for i < len(in) && in[i] == in[idx] { - i++ - } - return i -} - -func sortUint64(nums []uint64) { - sort.Slice(nums, func(i, j int) bool { return nums[i] < nums[j] }) -} - -func fillNums(N1, N2 int) ([]uint64, []uint64, []uint64) { - rand.Seed(time.Now().UnixNano()) - - commonNums := make([]uint64, N1) - blockNums := make([]uint64, N1+N2) - otherNums := make([]uint64, N1+N2) - - for i := 0; i < N1; i++ { - val := rand.Uint64() - commonNums[i] = val - blockNums[i] = val - otherNums[i] = val - } - - for i := N1; i < N1+N2; i++ { - blockNums[i] = rand.Uint64() - otherNums[i] = rand.Uint64() - } - - sortUint64(commonNums) - sortUint64(blockNums) - sortUint64(otherNums) - - return commonNums, blockNums, otherNums -} - -func TestIntersectCompressedWithLinJump(t *testing.T) { - lengths := []int{0, 1, 3, 11, 100} - - for _, N1 := range lengths { - for _, N2 := range lengths { - // Intersection of blockNums and otherNums is commonNums. - commonNums, blockNums, otherNums := fillNums(N1, N2) - - enc := codec.Encoder{BlockSize: 10} - for _, num := range blockNums { - enc.Add(num) - } - - pack := enc.Done() - dec := codec.Decoder{Pack: pack} - dec.Seek(0, codec.SeekStart) - - actual := make([]uint64, 0) - IntersectCompressedWithLinJump(&dec, otherNums, &actual) - require.Equal(t, commonNums, actual) - codec.FreePack(pack) - } - } -} - -func TestIntersectCompressedWithBin(t *testing.T) { - lengths := []int{0, 1, 3, 11, 100} - - for _, N1 := range lengths { - for _, N2 := range lengths { - // Intersection of blockNums and otherNums is commonNums. - commonNums, blockNums, otherNums := fillNums(N1, N2) - - enc := codec.Encoder{BlockSize: 10} - for _, num := range blockNums { - enc.Add(num) - } - - pack := enc.Done() - dec := codec.Decoder{Pack: pack} - dec.Seek(0, codec.SeekStart) - - actual := make([]uint64, 0) - IntersectCompressedWithBin(&dec, otherNums, &actual) - require.Equal(t, commonNums, actual) - codec.FreePack(pack) - } - } -} - -func TestIntersectCompressedWithBinMissingSize(t *testing.T) { - lengths := []int{0, 1, 3, 11, 100} - - for _, N1 := range lengths { - for _, N2 := range lengths { - // Intersection of blockNums and otherNums is commonNums. - commonNums, blockNums, otherNums := fillNums(N1, N2) - - // Set the block size to 0 to verify that the method still works in this case. - enc := codec.Encoder{BlockSize: 0} - for _, num := range blockNums { - enc.Add(num) - } - - pack := enc.Done() - dec := codec.Decoder{Pack: pack} - dec.Seek(0, codec.SeekStart) - - actual := make([]uint64, 0) - IntersectCompressedWithBin(&dec, otherNums, &actual) - require.Equal(t, commonNums, actual) - codec.FreePack(pack) - } - } + require.Equal(t, []uint64{1, 3, 5}, codec.GetUids(u)) } diff --git a/backup/run.go b/backup/run.go new file mode 100644 index 00000000000..9404e6aa6c5 --- /dev/null +++ b/backup/run.go @@ -0,0 +1,300 @@ +/* + * Copyright 2021 Dgraph Labs, Inc. and Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package backup + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "net/url" + "os" + "path/filepath" + "time" + + bpb "github.com/dgraph-io/badger/v3/pb" + "github.com/golang/glog" + + "github.com/dgraph-io/dgraph/ee" + "github.com/dgraph-io/dgraph/posting" + "github.com/dgraph-io/dgraph/protos/pb" + "github.com/dgraph-io/dgraph/worker" + "github.com/dgraph-io/dgraph/x" + "github.com/dgraph-io/ristretto/z" + "github.com/pkg/errors" + "github.com/spf13/cobra" +) + +// LsBackup is the sub-command used to list the backups in a folder. +var LsBackup x.SubCommand + +var ExportBackup x.SubCommand + +var opt struct { + backupId string + badger string + location string + pdir string + zero string + key x.Sensitive + forceZero bool + destination string + format string + verbose bool + upgrade bool // used by export backup command. +} + +func init() { + initBackupLs() + initExportBackup() +} + +func initBackupLs() { + LsBackup.Cmd = &cobra.Command{ + Use: "lsbackup", + Short: "List info on backups in a given location", + Args: cobra.NoArgs, + Run: func(cmd *cobra.Command, args []string) { + defer x.StartProfile(LsBackup.Conf).Stop() + if err := runLsbackupCmd(); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + }, + Annotations: map[string]string{"group": "tool"}, + } + LsBackup.Cmd.SetHelpTemplate(x.NonRootTemplate) + flag := LsBackup.Cmd.Flags() + flag.StringVarP(&opt.location, "location", "l", "", + "Sets the source location URI (required).") + flag.BoolVar(&opt.verbose, "verbose", false, + "Outputs additional info in backup list.") + _ = LsBackup.Cmd.MarkFlagRequired("location") +} + +func runLsbackupCmd() error { + manifests, err := worker.ListBackupManifests(opt.location, nil) + if err != nil { + return errors.Wrapf(err, "while listing manifests") + } + + type backupEntry struct { + Path string `json:"path"` + Since uint64 `json:"since"` + ReadTs uint64 `json:"read_ts"` + BackupId string `json:"backup_id"` + BackupNum uint64 `json:"backup_num"` + Encrypted bool `json:"encrypted"` + Type string `json:"type"` + Groups map[uint32][]string `json:"groups,omitempty"` + DropOperations []*pb.DropOperation `json:"drop_operations,omitempty"` + } + + type backupOutput []backupEntry + + var output backupOutput + for _, manifest := range manifests { + + be := backupEntry{ + Path: manifest.Path, + Since: manifest.SinceTsDeprecated, + ReadTs: manifest.ReadTs, + BackupId: manifest.BackupId, + BackupNum: manifest.BackupNum, + Encrypted: manifest.Encrypted, + Type: manifest.Type, + } + if opt.verbose { + be.Groups = manifest.Groups + be.DropOperations = manifest.DropOperations + } + output = append(output, be) + } + b, err := json.MarshalIndent(output, "", "\t") + if err != nil { + fmt.Println("error:", err) + } + os.Stdout.Write(b) + fmt.Println() + return nil +} + +func initExportBackup() { + ExportBackup.Cmd = &cobra.Command{ + Use: "export_backup", + Short: "Export data inside single full or incremental backup", + Long: ``, + Args: cobra.NoArgs, + Run: func(cmd *cobra.Command, args []string) { + defer x.StartProfile(ExportBackup.Conf).Stop() + if err := runExportBackup(); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + }, + Annotations: map[string]string{"group": "tool"}, + } + + ExportBackup.Cmd.SetHelpTemplate(x.NonRootTemplate) + flag := ExportBackup.Cmd.Flags() + flag.StringVarP(&opt.location, "location", "l", "", + `Sets the location of the backup. Both file URIs and s3 are supported. + This command will take care of all the full + incremental backups present in the location.`) + flag.StringVarP(&opt.destination, "destination", "d", "", + "The folder to which export the backups.") + flag.StringVarP(&opt.format, "format", "f", "rdf", + "The format of the export output. Accepts a value of either rdf or json") + flag.BoolVar(&opt.upgrade, "upgrade", false, + `If true, retrieve the CORS from DB and append at the end of GraphQL schema. + It also deletes the deprecated types and predicates. + Use this option when exporting a backup of 20.11 for loading onto 21.03.`) + ee.RegisterEncFlag(flag) +} + +type bufWriter struct { + writers *worker.Writers + req *pb.ExportRequest +} + +func exportSchema(writers *worker.Writers, val []byte, pk x.ParsedKey) error { + kv := &bpb.KV{} + var err error + if pk.IsSchema() { + kv, err = worker.SchemaExportKv(pk.Attr, val, true) + if err != nil { + return err + } + } else { + kv, err = worker.TypeExportKv(pk.Attr, val) + if err != nil { + return err + } + } + return worker.WriteExport(writers, kv, "rdf") +} + +func (bw *bufWriter) Write(buf *z.Buffer) error { + kv := &bpb.KV{} + err := buf.SliceIterate(func(s []byte) error { + kv.Reset() + if err := kv.Unmarshal(s); err != nil { + return errors.Wrap(err, "processKvBuf failed to unmarshal kv") + } + pk, err := x.Parse(kv.Key) + if err != nil { + return errors.Wrap(err, "processKvBuf failed to parse key") + } + if pk.Attr == "_predicate_" { + return nil + } + if pk.IsSchema() || pk.IsType() { + return exportSchema(bw.writers, kv.Value, pk) + } + if pk.IsData() { + pl := &pb.PostingList{} + if err := pl.Unmarshal(kv.Value); err != nil { + return errors.Wrap(err, "ProcessKvBuf failed to Unmarshal pl") + } + l := posting.NewList(kv.Key, pl, kv.Version) + kvList, err := worker.ToExportKvList(pk, l, bw.req) + if err != nil { + return errors.Wrap(err, "processKvBuf failed to Export") + } + if len(kvList.Kv) == 0 { + return nil + } + exportKv := kvList.Kv[0] + return worker.WriteExport(bw.writers, exportKv, bw.req.Format) + } + return nil + }) + return errors.Wrap(err, "bufWriter failed to write") +} + +func runExportBackup() error { + keys, err := ee.GetKeys(ExportBackup.Conf) + if err != nil { + return err + } + opt.key = keys.EncKey + if opt.format != "json" && opt.format != "rdf" { + return errors.Errorf("invalid format %s", opt.format) + } + // Create exportDir and temporary folder to store the restored backup. + exportDir, err := filepath.Abs(opt.destination) + if err != nil { + return errors.Wrapf(err, "cannot convert path %s to absolute path", exportDir) + } + if err := os.MkdirAll(exportDir, 0755); err != nil { + return errors.Wrapf(err, "cannot create dir %s", exportDir) + } + + uri, err := url.Parse(opt.location) + if err != nil { + return errors.Wrapf(err, "runExportBackup") + } + handler, err := x.NewUriHandler(uri, nil) + if err != nil { + return errors.Wrapf(err, "runExportBackup") + } + latestManifest, err := worker.GetLatestManifest(handler, uri) + if err != nil { + return errors.Wrapf(err, "runExportBackup") + } + + mapDir, err := ioutil.TempDir(x.WorkerConfig.TmpDir, "restore-export") + x.Check(err) + defer os.RemoveAll(mapDir) + glog.Infof("Created temporary map directory: %s\n", mapDir) + + encFlag := z.NewSuperFlag(ExportBackup.Conf.GetString("encryption")). + MergeAndCheckDefault(ee.EncDefaults) + // TODO: Can probably make this procesing concurrent. + for gid := range latestManifest.Groups { + glog.Infof("Exporting group: %d", gid) + req := &pb.RestoreRequest{ + GroupId: gid, + Location: opt.location, + EncryptionKeyFile: encFlag.GetPath("key-file"), + RestoreTs: 1, + } + if _, err := worker.RunMapper(req, mapDir); err != nil { + return errors.Wrap(err, "Failed to map the backups") + } + + in := &pb.ExportRequest{ + GroupId: uint32(gid), + ReadTs: latestManifest.ValidReadTs(), + UnixTs: time.Now().Unix(), + Format: opt.format, + Destination: exportDir, + } + writers, err := worker.NewWriters(in) + defer writers.Close() + if err != nil { + return err + } + + w := &bufWriter{req: in, writers: writers} + if err := worker.RunReducer(w, mapDir); err != nil { + return errors.Wrap(err, "Failed to reduce the map") + } + if err := writers.Close(); err != nil { + return errors.Wrap(err, "Failed to finish write") + } + } + return nil +} diff --git a/chunker/chunk.go b/chunker/chunk.go index 5c1e56cf787..33557b230c5 100644 --- a/chunker/chunk.go +++ b/chunker/chunk.go @@ -352,7 +352,7 @@ func slurpQuoted(r *bufio.Reader, out *bytes.Buffer) error { // and decompressed automatically even without the gz extension. The key, if non-nil, // is used to decrypt the file. The caller is responsible for calling the returned cleanup // function when done with the reader. -func FileReader(file string, key x.SensitiveByteSlice) (rd *bufio.Reader, cleanup func()) { +func FileReader(file string, key x.Sensitive) (*bufio.Reader, func()) { var f *os.File var err error if file == "-" { @@ -363,6 +363,12 @@ func FileReader(file string, key x.SensitiveByteSlice) (rd *bufio.Reader, cleanu x.Check(err) + return StreamReader(file, key, f) +} + +// StreamReader returns a bufio given a ReadCloser. The file is passed just to check for .gz files +func StreamReader(file string, key x.Sensitive, f io.ReadCloser) ( + rd *bufio.Reader, cleanup func()) { cleanup = func() { _ = f.Close() } if filepath.Ext(file) == ".gz" { diff --git a/chunker/json_parser.go b/chunker/json_parser.go index 24db7de4276..4b468d51209 100644 --- a/chunker/json_parser.go +++ b/chunker/json_parser.go @@ -26,11 +26,12 @@ import ( "sync/atomic" "unicode" - "github.com/dgraph-io/dgo/v200/protos/api" + "github.com/dgraph-io/dgo/v210/protos/api" "github.com/dgraph-io/dgraph/protos/pb" "github.com/dgraph-io/dgraph/types" "github.com/dgraph-io/dgraph/types/facets" "github.com/dgraph-io/dgraph/x" + simdjson "github.com/dgraph-io/simdjson-go" "github.com/pkg/errors" geom "github.com/twpayne/go-geom" "github.com/twpayne/go-geom/encoding/geojson" @@ -82,6 +83,14 @@ func handleBasicFacetsType(key string, facetVal interface{}) (*api.Facet, error) jsonValue = jsonInt valueType = api.Facet_INT } + // these int64/float64 cases are needed for the FastParseJSON simdjson + // parser, which doesn't use json.Number + case int64: + jsonValue = v + valueType = api.Facet_INT + case float64: + jsonValue = v + valueType = api.Facet_FLOAT case bool: jsonValue = v valueType = api.Facet_BOOL @@ -180,8 +189,10 @@ func parseScalarFacets(m map[string]interface{}, prefix string) ([]*api.Facet, e // This is the response for a map[string]interface{} i.e. a struct. type mapResponse struct { - uid string // uid retrieved or allocated for the node. - fcts []*api.Facet // facets on the edge connecting this node to the source if any. + uid string // uid retrieved or allocated for the node. + namespace uint64 // namespace to which the node belongs. + fcts []*api.Facet // facets on the edge connecting this node to the source if any. + rawFacets map[string]interface{} } func handleBasicType(k string, v interface{}, op int, nq *api.NQuad) error { @@ -201,6 +212,14 @@ func handleBasicType(k string, v interface{}, op int, nq *api.NQuad) error { } nq.ObjectValue = &api.Value{Val: &api.Value_IntVal{IntVal: i}} + // this int64 case is needed for FastParseJSON, which doesn't use json.Number + case int64: + if v == 0 && op == DeleteNquads { + nq.ObjectValue = &api.Value{Val: &api.Value_IntVal{IntVal: v}} + return nil + } + nq.ObjectValue = &api.Value{Val: &api.Value_IntVal{IntVal: v}} + case string: // Default value is considered as S P * deletion. if v == "" && op == DeleteNquads { @@ -246,10 +265,11 @@ func handleBasicType(k string, v interface{}, op int, nq *api.NQuad) error { func (buf *NQuadBuffer) checkForDeletion(mr mapResponse, m map[string]interface{}, op int) { // Since uid is the only key, this must be S * * deletion. - if op == DeleteNquads && len(mr.uid) > 0 && len(m) == 1 { + if op == DeleteNquads && len(mr.uid) > 0 && len(m) == 1 && len(mr.rawFacets) == 0 { buf.Push(&api.NQuad{ Subject: mr.uid, Predicate: x.Star, + Namespace: mr.namespace, ObjectValue: &api.Value{Val: &api.Value_DefaultVal{DefaultVal: x.Star}}, }) } @@ -377,6 +397,16 @@ func getNextBlank() string { func (buf *NQuadBuffer) mapToNquads(m map[string]interface{}, op int, parentPred string) ( mapResponse, error) { var mr mapResponse + + // move all facets from global map to smaller mr.rawFacets map + mr.rawFacets = make(map[string]interface{}) + for k, v := range m { + if strings.Contains(k, x.FacetDelimeter) { + mr.rawFacets[k] = v + delete(m, k) + } + } + // Check field in map. if uidVal, ok := m["uid"]; ok { var uid uint64 @@ -389,9 +419,13 @@ func (buf *NQuadBuffer) mapToNquads(m map[string]interface{}, op int, parentPred } uid = uint64(ui) + // this int64 case is needed for FastParseJSON, which doesn't use json.Number + case int64: + uid = uint64(uidVal) + case string: s := stripSpaces(uidVal) - if len(uidVal) == 0 { + if uidVal == "" { uid = 0 } else if ok := strings.HasPrefix(uidVal, "_:"); ok { mr.uid = uidVal @@ -408,21 +442,46 @@ func (buf *NQuadBuffer) mapToNquads(m map[string]interface{}, op int, parentPred } } - if len(mr.uid) == 0 { + if mr.uid == "" { if op == DeleteNquads { // Delete operations with a non-nil value must have a uid specified. return mr, errors.Errorf("UID must be present and non-zero while deleting edges.") } - mr.uid = getNextBlank() } + namespace := x.GalaxyNamespace + if ns, ok := m["namespace"]; ok { + switch nsVal := ns.(type) { + case json.Number: + nsi, err := nsVal.Int64() + if err != nil { + return mr, err + } + namespace = uint64(nsi) + + // this int64 case is needed for FastParseJSON, which doesn't use json.Number + case int64: + namespace = uint64(nsVal) + case string: + s := stripSpaces(nsVal) + if s == "" { + namespace = 0 + } else if n, err := strconv.ParseUint(s, 0, 64); err == nil { + namespace = n + } else { + return mr, err + } + } + } + mr.namespace = namespace + for pred, v := range m { // We have already extracted the uid above so we skip that edge. // v can be nil if user didn't set a value and if omitEmpty was not supplied as JSON // option. // We also skip facets here because we parse them with the corresponding predicate. - if pred == "uid" || strings.Index(pred, x.FacetDelimeter) > 0 { + if pred == "uid" || pred == "namespace" { continue } @@ -432,6 +491,7 @@ func (buf *NQuadBuffer) mapToNquads(m map[string]interface{}, op int, parentPred nq := &api.NQuad{ Subject: mr.uid, Predicate: pred, + Namespace: namespace, ObjectValue: &api.Value{Val: &api.Value_DefaultVal{DefaultVal: x.Star}}, } // Here we split predicate and lang directive (ex: "name@en"), if needed. With JSON @@ -448,14 +508,12 @@ func (buf *NQuadBuffer) mapToNquads(m map[string]interface{}, op int, parentPred nq := api.NQuad{ Subject: mr.uid, Predicate: pred, + Namespace: namespace, } prefix := pred + x.FacetDelimeter - // TODO - Maybe do an initial pass and build facets for all predicates. Then we don't have - // to call parseFacets everytime. - // Only call parseBasicFacets when value type for the predicate is not list. if _, ok := v.([]interface{}); !ok { - fts, err := parseScalarFacets(m, prefix) + fts, err := parseScalarFacets(mr.rawFacets, prefix) if err != nil { return mr, err } @@ -467,6 +525,13 @@ func (buf *NQuadBuffer) mapToNquads(m map[string]interface{}, op int, parentPred nq.Predicate, nq.Lang = x.PredicateLang(nq.Predicate) switch v := v.(type) { + // these int64/float64 cases are needed for FastParseJSON, which doesn't use json.Number + case int64, float64: + if err := handleBasicType(pred, v, op, &nq); err != nil { + return mr, err + } + buf.Push(&nq) + buf.PushPredHint(pred, pb.Metadata_SINGLE) case string, json.Number, bool: if err := handleBasicType(pred, v, op, &nq); err != nil { return mr, err @@ -500,21 +565,32 @@ func (buf *NQuadBuffer) mapToNquads(m map[string]interface{}, op int, parentPred buf.PushPredHint(pred, pb.Metadata_SINGLE) case []interface{}: buf.PushPredHint(pred, pb.Metadata_LIST) - // TODO(Ashish): We need to call this only in case of scalarlist, for other lists - // this can be avoided. - facetsMapSlice, err := parseMapFacets(m, prefix) - if err != nil { - return mr, err - } + // NOTE: facetsMapSlice should be empty unless this is a scalar list + var facetsMapSlice []map[int]*api.Facet for idx, item := range v { + if idx == 0 { + // determine if this is a scalar list + switch item.(type) { + case string, float64, json.Number, int64: + var err error + facetsMapSlice, err = parseMapFacets(mr.rawFacets, prefix) + if err != nil { + return mr, err + } + default: + // not a scalar list, continue + } + } + nq := api.NQuad{ Subject: mr.uid, Predicate: pred, + Namespace: namespace, } switch iv := item.(type) { - case string, float64, json.Number: + case string, float64, json.Number, int64: if err := handleBasicType(pred, iv, op, &nq); err != nil { return mr, err } @@ -575,8 +651,9 @@ func (buf *NQuadBuffer) mapToNquads(m map[string]interface{}, op int, parentPred } } - fts, err := parseScalarFacets(m, parentPred+x.FacetDelimeter) + fts, err := parseScalarFacets(mr.rawFacets, parentPred+x.FacetDelimeter) mr.fcts = fts + return mr, err } @@ -588,6 +665,87 @@ const ( DeleteNquads ) +// FastParseJSON currently parses NQuads about 30% faster than ParseJSON. +// +// This function is very similar to buf.ParseJSON, but we just replace encoding/json with +// simdjson-go. +func (buf *NQuadBuffer) FastParseJSON(b []byte, op int) error { + if !simdjson.SupportedCPU() { + // default to slower / old parser + return buf.ParseJSON(b, op) + } + // parse the json into tape format + tape, err := simdjson.Parse(b, nil) + if err != nil { + return err + } + + // we only need the iter to get the first element, either an array or object + iter := tape.Iter() + + tmp := &simdjson.Iter{} + + // if root object, this will be filled + obj := &simdjson.Object{} + // if root array, this will be filled + arr := &simdjson.Array{} + + // grab the first element + typ := iter.Advance() + switch typ { + case simdjson.TypeRoot: + if typ, tmp, err = iter.Root(tmp); err != nil { + return err + } + if typ == simdjson.TypeObject { + // the root element is an object, so parse the object + if obj, err = tmp.Object(obj); err != nil { + return err + } + // attempt to convert to map[string]interface{} + m, err := obj.Map(nil) + if err != nil { + return err + } + // pass to next parsing stage + mr, err := buf.mapToNquads(m, op, "") + if err != nil { + return err + } + buf.checkForDeletion(mr, m, op) + } else if typ == simdjson.TypeArray { + // the root element is an array, so parse the array + if arr, err = tmp.Array(arr); err != nil { + return err + } + // attempt to convert to []interface{} + a, err := arr.Interface() + if err != nil { + return err + } + if len(a) > 0 { + // attempt to convert each array element to a + // map[string]interface{} for further parsing + var o interface{} + for _, o = range a { + if _, ok := o.(map[string]interface{}); !ok { + return errors.Errorf("only array of map allowed at root") + } + // pass to next parsing stage + mr, err := buf.mapToNquads(o.(map[string]interface{}), op, "") + if err != nil { + return err + } + buf.checkForDeletion(mr, o.(map[string]interface{}), op) + } + } + } + default: + return errors.Errorf("initial element not found in json") + } + return nil +} + // ParseJSON parses the given byte slice and pushes the parsed NQuads into the buffer. func (buf *NQuadBuffer) ParseJSON(b []byte, op int) error { buffer := bytes.NewBuffer(b) @@ -597,7 +755,6 @@ func (buf *NQuadBuffer) ParseJSON(b []byte, op int) error { var list []interface{} if err := dec.Decode(&ms); err != nil { // Couldn't parse as map, lets try to parse it as a list. - buffer.Reset() // The previous contents are used. Reset here. // Rewrite b into buffer, so it can be consumed. if _, err := buffer.Write(b); err != nil { @@ -607,11 +764,9 @@ func (buf *NQuadBuffer) ParseJSON(b []byte, op int) error { return err } } - if len(list) == 0 && len(ms) == 0 { return nil } - if len(list) > 0 { for _, obj := range list { if _, ok := obj.(map[string]interface{}); !ok { @@ -625,21 +780,22 @@ func (buf *NQuadBuffer) ParseJSON(b []byte, op int) error { } return nil } - mr, err := buf.mapToNquads(ms, op, "") + if err != nil { + return err + } buf.checkForDeletion(mr, ms, op) - return err + return nil } // ParseJSON is a convenience wrapper function to get all NQuads in one call. This can however, lead // to high memory usage. So be careful using this. func ParseJSON(b []byte, op int) ([]*api.NQuad, *pb.Metadata, error) { buf := NewNQuadBuffer(-1) - err := buf.ParseJSON(b, op) + err := buf.FastParseJSON(b, op) if err != nil { return nil, nil, err } - buf.Flush() nqs := <-buf.Ch() metadata := buf.Metadata() diff --git a/chunker/json_parser_test.go b/chunker/json_parser_test.go index 655acb7cc35..36ba208b2da 100644 --- a/chunker/json_parser_test.go +++ b/chunker/json_parser_test.go @@ -29,7 +29,7 @@ import ( "github.com/dgraph-io/dgraph/tok" "github.com/golang/glog" - "github.com/dgraph-io/dgo/v200/protos/api" + "github.com/dgraph-io/dgo/v210/protos/api" "github.com/dgraph-io/dgraph/types" "github.com/stretchr/testify/require" ) @@ -60,14 +60,15 @@ type address struct { } type Person struct { - Uid string `json:"uid,omitempty"` - Name string `json:"name,omitempty"` - Age int `json:"age,omitempty"` - Married *bool `json:"married,omitempty"` - Now *time.Time `json:"now,omitempty"` - Address address `json:"address,omitempty"` // geo value - Friends []Person `json:"friend,omitempty"` - School *School `json:"school,omitempty"` + Uid string `json:"uid,omitempty"` + Namespace string `json:"namespace,omitempty"` + Name string `json:"name,omitempty"` + Age int `json:"age,omitempty"` + Married *bool `json:"married,omitempty"` + Now *time.Time `json:"now,omitempty"` + Address address `json:"address,omitempty"` // geo value + Friends []Person `json:"friend,omitempty"` + School *School `json:"school,omitempty"` } func Parse(b []byte, op int) ([]*api.NQuad, error) { @@ -76,6 +77,13 @@ func Parse(b []byte, op int) ([]*api.NQuad, error) { return nqs.nquads, err } +// FastParse uses buf.FastParseJSON() simdjson parser. +func FastParse(b []byte, op int) ([]*api.NQuad, error) { + nqs := NewNQuadBuffer(1000) + err := nqs.FastParseJSON(b, op) + return nqs.nquads, err +} + func (exp *Experiment) verify() { // insert the data into dgraph dg, err := testutil.DgraphClientWithGroot(testutil.SockAddr) @@ -109,10 +117,12 @@ func TestNquadsFromJson1(t *testing.T) { tn := time.Now().UTC() m := true p := Person{ - Name: "Alice", - Age: 26, - Married: &m, - Now: &tn, + Uid: "1", + Namespace: "0x2", + Name: "Alice", + Age: 26, + Married: &m, + Now: &tn, Address: address{ Type: "Point", Coords: []float64{1.1, 2.0}, @@ -124,8 +134,12 @@ func TestNquadsFromJson1(t *testing.T) { nq, err := Parse(b, SetNquads) require.NoError(t, err) - require.Equal(t, 5, len(nq)) + + fastNQ, err := FastParse(b, SetNquads) + require.NoError(t, err) + require.Equal(t, 5, len(fastNQ)) + exp := &Experiment{ t: t, nqs: nq, @@ -144,6 +158,9 @@ address ]} `} exp.verify() + + exp.nqs = fastNQ + exp.verify() } func TestNquadsFromJson2(t *testing.T) { @@ -166,6 +183,11 @@ func TestNquadsFromJson2(t *testing.T) { nq, err := Parse(b, SetNquads) require.NoError(t, err) require.Equal(t, 6, len(nq)) + + fastNQ, err := FastParse(b, SetNquads) + require.NoError(t, err) + require.Equal(t, 6, len(fastNQ)) + exp := &Experiment{ t: t, nqs: nq, @@ -185,6 +207,9 @@ friend { }]}`, } exp.verify() + + exp.nqs = fastNQ + exp.verify() } func TestNquadsFromJson3(t *testing.T) { @@ -199,6 +224,10 @@ func TestNquadsFromJson3(t *testing.T) { require.NoError(t, err) nq, err := Parse(b, SetNquads) require.NoError(t, err) + + fastNQ, err := FastParse(b, SetNquads) + require.NoError(t, err) + exp := &Experiment{ t: t, nqs: nq, @@ -213,6 +242,9 @@ school {name} }]}`, } exp.verify() + + exp.nqs = fastNQ + exp.verify() } func TestNquadsFromJson4(t *testing.T) { @@ -220,6 +252,10 @@ func TestNquadsFromJson4(t *testing.T) { nq, err := Parse([]byte(json), SetNquads) require.NoError(t, err) + + fastNQ, err := FastParse([]byte(json), SetNquads) + require.NoError(t, err) + exp := &Experiment{ t: t, nqs: nq, @@ -234,6 +270,9 @@ weight expected: fmt.Sprintf(`{"alice":%s}`, json), } exp.verify() + + exp.nqs = fastNQ + exp.verify() } func TestNquadsFromJsonMap(t *testing.T) { @@ -245,6 +284,10 @@ func TestNquadsFromJsonMap(t *testing.T) { nq, err := Parse([]byte(json), SetNquads) require.NoError(t, err) + + fastNQ, err := FastParse([]byte(json), SetNquads) + require.NoError(t, err) + exp := &Experiment{ t: t, nqs: nq, @@ -257,6 +300,9 @@ friends {name} expected: fmt.Sprintf(`{"people":[%s]}`, json), } exp.verify() + + exp.nqs = fastNQ + exp.verify() } func TestNquadsFromMultipleJsonObjects(t *testing.T) { @@ -323,6 +369,10 @@ func TestNquadsFromMultipleJsonObjects(t *testing.T) { nq, err := Parse([]byte(json), SetNquads) require.NoError(t, err) + + fastNQ, err := FastParse([]byte(json), SetNquads) + require.NoError(t, err) + exp := &Experiment{ t: t, nqs: nq, @@ -335,6 +385,9 @@ friends expected: fmt.Sprintf(`{"people":%s}`, json), } exp.verify() + + exp.nqs = fastNQ + exp.verify() } func TestJsonNumberParsing(t *testing.T) { @@ -351,14 +404,28 @@ func TestJsonNumberParsing(t *testing.T) { {`{"uid": "1", "key": 0E-0}`, &api.Value{Val: &api.Value_DoubleVal{DoubleVal: 0}}}, } - for _, test := range tests { + for i, test := range tests { nqs, err := Parse([]byte(test.in), SetNquads) + if i == 2 { + fmt.Println(err) + } if test.out != nil { require.NoError(t, err, "%T", err) require.Equal(t, makeNquad("1", "key", test.out), nqs[0]) } else { require.Error(t, err) } + + fastNQ, err := FastParse([]byte(test.in), SetNquads) + if i == 2 { + fmt.Println(err) + } + if test.out != nil { + require.NoError(t, err, "%T", err) + require.Equal(t, makeNquad("1", "key", test.out), fastNQ[0]) + } else { + require.Error(t, err) + } } } @@ -367,6 +434,42 @@ func TestNquadsFromJson_UidOutofRangeError(t *testing.T) { _, err := Parse([]byte(json), SetNquads) require.Error(t, err) + + _, err = FastParse([]byte(json), SetNquads) + require.Error(t, err) +} + +func TestNquadsFromJsonArray(t *testing.T) { + json := `[ + { + "uid": "uid(Project10)", + "Ticket.row": { + "uid": "uid(x)" + } + }, + { + "Project.columns": [ + { + "uid": "uid(x)" + } + ], + "uid": "uid(Project3)" + }, + { + "Ticket.onColumn": { + "uid": "uid(x)" + }, + "uid": "uid(Ticket4)" + } + ]` + + nqs, err := Parse([]byte(json), SetNquads) + require.NoError(t, err) + require.Equal(t, 3, len(nqs)) + + nqs, err = FastParse([]byte(json), SetNquads) + require.NoError(t, err) + require.Equal(t, 3, len(nqs)) } func TestNquadsFromJson_NegativeUidError(t *testing.T) { @@ -374,6 +477,9 @@ func TestNquadsFromJson_NegativeUidError(t *testing.T) { _, err := Parse([]byte(json), SetNquads) require.Error(t, err) + + _, err = FastParse([]byte(json), SetNquads) + require.Error(t, err) } func TestNquadsFromJson_EmptyUid(t *testing.T) { @@ -382,6 +488,9 @@ func TestNquadsFromJson_EmptyUid(t *testing.T) { nq, err := Parse([]byte(json), SetNquads) require.NoError(t, err) + fastNQ, err := FastParse([]byte(json), SetNquads) + require.NoError(t, err) + exp := &Experiment{ t: t, nqs: nq, @@ -395,6 +504,9 @@ school { name} "name":"Crown Public School"}]}]}`, } exp.verify() + + exp.nqs = fastNQ + exp.verify() } func TestNquadsFromJson_BlankNodes(t *testing.T) { @@ -403,6 +515,9 @@ func TestNquadsFromJson_BlankNodes(t *testing.T) { nq, err := Parse([]byte(json), SetNquads) require.NoError(t, err) + fastNQ, err := FastParse([]byte(json), SetNquads) + require.NoError(t, err) + exp := &Experiment{ t: t, nqs: nq, @@ -416,6 +531,9 @@ school { name} "name":"Crown Public School"}]}]}`, } exp.verify() + + exp.nqs = fastNQ + exp.verify() } func TestNquadsDeleteEdges(t *testing.T) { @@ -423,6 +541,10 @@ func TestNquadsDeleteEdges(t *testing.T) { nq, err := Parse([]byte(json), DeleteNquads) require.NoError(t, err) require.Equal(t, 3, len(nq)) + + fastNQ, err := FastParse([]byte(json), DeleteNquads) + require.NoError(t, err) + require.Equal(t, 3, len(fastNQ)) } func checkCount(t *testing.T, nq []*api.NQuad, pred string, count int) { @@ -498,9 +620,16 @@ func TestNquadsFromJsonFacets1(t *testing.T) { require.NoError(t, err) require.Equal(t, 3, len(nq)) + fastNQ, err := FastParse([]byte(json), SetNquads) + require.NoError(t, err) + require.Equal(t, 3, len(fastNQ)) + for _, n := range nq { glog.Infof("%v", n) + } + for _, n := range fastNQ { + glog.Infof("%v", n) } checkFacets(t, nq, "mobile", []*api.Facet{ @@ -512,6 +641,15 @@ func TestNquadsFromJsonFacets1(t *testing.T) { }, }) + checkFacets(t, fastNQ, "mobile", []*api.Facet{ + { + Key: "operation", + Value: []byte(operation), + ValType: api.Facet_STRING, + Tokens: operationTokens, + }, + }) + checkFacets(t, nq, "car", []*api.Facet{ { Key: "first", @@ -534,6 +672,29 @@ func TestNquadsFromJsonFacets1(t *testing.T) { ValType: api.Facet_DATETIME, }, }) + + checkFacets(t, fastNQ, "car", []*api.Facet{ + { + Key: "first", + Value: []byte{1}, + ValType: api.Facet_BOOL, + }, + { + Key: "age", + Value: ageBytes[:], + ValType: api.Facet_INT, + }, + { + Key: "price", + Value: priceBytes[:], + ValType: api.Facet_FLOAT, + }, + { + Key: "since", + Value: timeBinary, + ValType: api.Facet_DATETIME, + }, + }) } func TestNquadsFromJsonFacets2(t *testing.T) { @@ -544,6 +705,11 @@ func TestNquadsFromJsonFacets2(t *testing.T) { require.NoError(t, err) require.Equal(t, 3, len(nq)) checkCount(t, nq, "friend", 1) + + fastNQ, err := FastParse([]byte(json), SetNquads) + require.NoError(t, err) + require.Equal(t, 3, len(fastNQ)) + checkCount(t, fastNQ, "friend", 1) } // Test valid facets json. @@ -580,6 +746,23 @@ func TestNquadsFromJsonFacets3(t *testing.T) { require.Equal(t, 2, len(nq.Facets)) } } + + fastNQ, err := FastParse([]byte(json), SetNquads) + require.NoError(t, err) + require.Equal(t, 4, len(fastNQ)) + for _, nq := range fastNQ { + predVal := nq.ObjectValue.GetStrVal() + switch predVal { + case "Alice": + require.Equal(t, 0, len(nq.Facets)) + case "Joshua": + require.Equal(t, 1, len(nq.Facets)) + case "David": + require.Equal(t, 1, len(nq.Facets)) + case "Josh": + require.Equal(t, 2, len(nq.Facets)) + } + } } // Test invalid facet format with scalar list predicate. @@ -726,9 +909,48 @@ func TestNquadsFromJsonFacets4(t *testing.T) { } else { require.NoError(t, err, "TestNquadsFromJsonFacets4-%s", input.Name) } + + _, err = FastParse([]byte(input.Json), SetNquads) + if input.ErrorOut { + require.Error(t, err, "TestNquadsFromJsonFacets4-%s", input.Name) + } else { + require.NoError(t, err, "TestNquadsFromJsonFacets4-%s", input.Name) + } } } +func TestNquadsFromJsonFacets5(t *testing.T) { + // Dave has uid facets which should go on the edge between Alice and Dave, + // AND Emily has uid facets which should go on the edge between Dave and Emily + json := `[ + { + "name": "Alice", + "friend": [ + { + "name": "Dave", + "friend|close": true, + "friend": [ + { + "name": "Emily", + "friend|close": true + } + ] + } + ] + } + ]` + + nq, err := Parse([]byte(json), SetNquads) + require.NoError(t, err) + require.Equal(t, 5, len(nq)) + checkCount(t, nq, "friend", 1) + + fastNQ, err := FastParse([]byte(json), SetNquads) + require.NoError(t, err) + require.Equal(t, 5, len(fastNQ)) + checkCount(t, fastNQ, "friend", 1) +} + func TestNquadsFromJsonError1(t *testing.T) { p := Person{ Name: "Alice", @@ -743,6 +965,10 @@ func TestNquadsFromJsonError1(t *testing.T) { _, err = Parse(b, DeleteNquads) require.Error(t, err) require.Contains(t, err.Error(), "UID must be present and non-zero while deleting edges.") + + _, err = FastParse(b, DeleteNquads) + require.Error(t, err) + require.Contains(t, err.Error(), "UID must be present and non-zero while deleting edges.") } func TestNquadsFromJsonList(t *testing.T) { @@ -751,6 +977,10 @@ func TestNquadsFromJsonList(t *testing.T) { nq, err := Parse([]byte(json), SetNquads) require.NoError(t, err) require.Equal(t, 6, len(nq)) + + fastNQ, err := FastParse([]byte(json), SetNquads) + require.NoError(t, err) + require.Equal(t, 6, len(fastNQ)) } func TestNquadsFromJsonDelete(t *testing.T) { @@ -759,6 +989,10 @@ func TestNquadsFromJsonDelete(t *testing.T) { nq, err := Parse([]byte(json), DeleteNquads) require.NoError(t, err) require.Equal(t, nq[0], makeNquadEdge("1000", "friend", "1001")) + + fastNQ, err := FastParse([]byte(json), DeleteNquads) + require.NoError(t, err) + require.Equal(t, fastNQ[0], makeNquadEdge("1000", "friend", "1001")) } func TestNquadsFromJsonDeleteStar(t *testing.T) { @@ -766,6 +1000,10 @@ func TestNquadsFromJsonDeleteStar(t *testing.T) { nq, err := Parse([]byte(json), DeleteNquads) require.NoError(t, err) + + fastNQ, err := FastParse([]byte(json), DeleteNquads) + require.NoError(t, err) + expected := &api.NQuad{ Subject: "1000", Predicate: "name", @@ -775,19 +1013,27 @@ func TestNquadsFromJsonDeleteStar(t *testing.T) { }, }, } + require.Equal(t, expected, nq[0]) + require.Equal(t, expected, fastNQ[0]) } func TestValInUpsert(t *testing.T) { json := `{"uid":1000, "name": "val(name)"}` nq, err := Parse([]byte(json), SetNquads) require.NoError(t, err) + + fastNQ, err := FastParse([]byte(json), SetNquads) + require.NoError(t, err) + expected := &api.NQuad{ Subject: "1000", Predicate: "name", ObjectId: "val(name)", } + require.Equal(t, expected, nq[0]) + require.Equal(t, expected, fastNQ[0]) } func TestNquadsFromJsonDeleteStarLang(t *testing.T) { @@ -795,6 +1041,10 @@ func TestNquadsFromJsonDeleteStarLang(t *testing.T) { nq, err := Parse([]byte(json), DeleteNquads) require.NoError(t, err) + + fastNQ, err := FastParse([]byte(json), DeleteNquads) + require.NoError(t, err) + expected := &api.NQuad{ Subject: "1000", Predicate: "name", @@ -805,7 +1055,9 @@ func TestNquadsFromJsonDeleteStarLang(t *testing.T) { }, Lang: "es", } + require.Equal(t, expected, nq[0]) + require.Equal(t, expected, fastNQ[0]) } func TestSetNquadNilValue(t *testing.T) { @@ -814,4 +1066,309 @@ func TestSetNquadNilValue(t *testing.T) { nq, err := Parse([]byte(json), SetNquads) require.NoError(t, err) require.Equal(t, 0, len(nq)) + + fastNQ, err := FastParse([]byte(json), SetNquads) + require.NoError(t, err) + require.Equal(t, 0, len(fastNQ)) +} + +// See PR #7737 to understand why this test exists. +func TestNquadsFromJsonEmptyFacet(t *testing.T) { + json := `{"uid":1000,"doesnt|exist":null}` + + // fast + buf := NewNQuadBuffer(-1) + require.Nil(t, buf.FastParseJSON([]byte(json), DeleteNquads)) + buf.Flush() + // needs to be empty, otherwise node gets deleted + require.Equal(t, 0, len(<-buf.Ch())) + + // old + buf = NewNQuadBuffer(-1) + require.Nil(t, buf.ParseJSON([]byte(json), DeleteNquads)) + buf.Flush() + // needs to be empty, otherwise node gets deleted + require.Equal(t, 0, len(<-buf.Ch())) +} + +func BenchmarkNoFacets(b *testing.B) { + json := []byte(`[ + { + "uid":123, + "flguid":123, + "is_validate":"xxxxxxxxxx", + "createDatetime":"xxxxxxxxxx", + "contains":{ + "createDatetime":"xxxxxxxxxx", + "final_individ":"xxxxxxxxxx", + "cm_bad_debt":"xxxxxxxxxx", + "cm_bill_address1":"xxxxxxxxxx", + "cm_bill_address2":"xxxxxxxxxx", + "cm_bill_city":"xxxxxxxxxx", + "cm_bill_state":"xxxxxxxxxx", + "cm_zip":"xxxxxxxxxx", + "zip5":"xxxxxxxxxx", + "cm_customer_id":"xxxxxxxxxx", + "final_gaid":"xxxxxxxxxx", + "final_hholdid":"xxxxxxxxxx", + "final_firstname":"xxxxxxxxxx", + "final_middlename":"xxxxxxxxxx", + "final_surname":"xxxxxxxxxx", + "final_gender":"xxxxxxxxxx", + "final_ace_prim_addr":"xxxxxxxxxx", + "final_ace_sec_addr":"xxxxxxxxxx", + "final_ace_urb":"xxxxxxxxxx", + "final_ace_city_llidx":"xxxxxxxxxx", + "final_ace_state":"xxxxxxxxxx", + "final_ace_postal_code":"xxxxxxxxxx", + "final_ace_zip4":"xxxxxxxxxx", + "final_ace_dpbc":"xxxxxxxxxx", + "final_ace_checkdigit":"xxxxxxxxxx", + "final_ace_iso_code":"xxxxxxxxxx", + "final_ace_cart":"xxxxxxxxxx", + "final_ace_lot":"xxxxxxxxxx", + "final_ace_lot_order":"xxxxxxxxxx", + "final_ace_rec_type":"xxxxxxxxxx", + "final_ace_remainder":"xxxxxxxxxx", + "final_ace_dpv_cmra":"xxxxxxxxxx", + "final_ace_dpv_ftnote":"xxxxxxxxxx", + "final_ace_dpv_status":"xxxxxxxxxx", + "final_ace_foreigncode":"xxxxxxxxxx", + "final_ace_match_5":"xxxxxxxxxx", + "final_ace_match_9":"xxxxxxxxxx", + "final_ace_match_un":"xxxxxxxxxx", + "final_ace_zip_move":"xxxxxxxxxx", + "final_ace_ziptype":"xxxxxxxxxx", + "final_ace_congress":"xxxxxxxxxx", + "final_ace_county":"xxxxxxxxxx", + "final_ace_countyname":"xxxxxxxxxx", + "final_ace_factype":"xxxxxxxxxx", + "final_ace_fipscode":"xxxxxxxxxx", + "final_ace_error_code":"xxxxxxxxxx", + "final_ace_stat_code":"xxxxxxxxxx", + "final_ace_geo_match":"xxxxxxxxxx", + "final_ace_geo_lat":"xxxxxxxxxx", + "final_ace_geo_lng":"xxxxxxxxxx", + "final_ace_ageo_pla":"xxxxxxxxxx", + "final_ace_geo_blk":"xxxxxxxxxx", + "final_ace_ageo_mcd":"xxxxxxxxxx", + "final_ace_cgeo_cbsa":"xxxxxxxxxx", + "final_ace_cgeo_msa":"xxxxxxxxxx", + "final_ace_ap_lacscode":"xxxxxxxxxx", + "final_dsf_businessflag":"xxxxxxxxxx", + "final_dsf_dropflag":"xxxxxxxxxx", + "final_dsf_throwbackflag":"xxxxxxxxxx", + "final_dsf_seasonalflag":"xxxxxxxxxx", + "final_dsf_vacantflag":"xxxxxxxxxx", + "final_dsf_deliverytype":"xxxxxxxxxx", + "final_dsf_dt_curbflag":"xxxxxxxxxx", + "final_dsf_dt_ndcbuflag":"xxxxxxxxxx", + "final_dsf_dt_centralflag":"xxxxxxxxxx", + "final_dsf_dt_doorslotflag":"xxxxxxxxxx", + "final_dsf_dropcount":"xxxxxxxxxx", + "final_dsf_nostatflag":"xxxxxxxxxx", + "final_dsf_educationalflag":"xxxxxxxxxx", + "final_dsf_rectyp":"xxxxxxxxxx", + "final_mailability_score":"xxxxxxxxxx", + "final_occupancy_score":"xxxxxxxxxx", + "final_multi_type":"xxxxxxxxxx", + "final_deceased_flag":"xxxxxxxxxx", + "final_dnm_flag":"xxxxxxxxxx", + "final_dnc_flag":"xxxxxxxxxx", + "final_dnf_flag":"xxxxxxxxxx", + "final_prison_flag":"xxxxxxxxxx", + "final_nursing_home_flag":"xxxxxxxxxx", + "final_date_of_birth":"xxxxxxxxxx", + "final_date_of_death":"xxxxxxxxxx", + "vip_number":"xxxxxxxxxx", + "vip_store_no":"xxxxxxxxxx", + "vip_division":"xxxxxxxxxx", + "vip_phone_number":"xxxxxxxxxx", + "vip_email_address":"xxxxxxxxxx", + "vip_first_name":"xxxxxxxxxx", + "vip_last_name":"xxxxxxxxxx", + "vip_gender":"xxxxxxxxxx", + "vip_status":"xxxxxxxxxx", + "vip_membership_date":"xxxxxxxxxx", + "vip_expiration_date":"xxxxxxxxxx", + "cm_date_addr_chng":"xxxxxxxxxx", + "cm_date_entered":"xxxxxxxxxx", + "cm_name":"xxxxxxxxxx", + "cm_opt_on_acct":"xxxxxxxxxx", + "cm_origin":"xxxxxxxxxx", + "cm_orig_acq_source":"xxxxxxxxxx", + "cm_phone_number":"xxxxxxxxxx", + "cm_phone_number2":"xxxxxxxxxx", + "cm_problem_cust":"xxxxxxxxxx", + "cm_rm_list":"xxxxxxxxxx", + "cm_rm_rented_list":"xxxxxxxxxx", + "cm_tax_code":"xxxxxxxxxx", + "email_address":"xxxxxxxxxx", + "esp_email_id":"xxxxxxxxxx", + "esp_sub_date":"xxxxxxxxxx", + "esp_unsub_date":"xxxxxxxxxx", + "cm_user_def_1":"xxxxxxxxxx", + "cm_user_def_7":"xxxxxxxxxx", + "do_not_phone":"xxxxxxxxxx", + "company_num":"xxxxxxxxxx", + "customer_id":"xxxxxxxxxx", + "load_date":"xxxxxxxxxx", + "activity_date":"xxxxxxxxxx", + "email_address_hashed":"xxxxxxxxxx", + "event_id":"", + "contains":{ + "uid": 123, + "flguid": 123, + "is_validate":"xxxxxxxxxx", + "createDatetime":"xxxxxxxxxx" + } + } + }]`) + + // we're parsing 125 nquads at a time, so the MB/s == MNquads/s + b.SetBytes(125) + for n := 0; n < b.N; n++ { + Parse([]byte(json), SetNquads) + } +} + +func BenchmarkNoFacetsFast(b *testing.B) { + json := []byte(`[ + { + "uid":123, + "flguid":123, + "is_validate":"xxxxxxxxxx", + "createDatetime":"xxxxxxxxxx", + "contains":{ + "createDatetime":"xxxxxxxxxx", + "final_individ":"xxxxxxxxxx", + "cm_bad_debt":"xxxxxxxxxx", + "cm_bill_address1":"xxxxxxxxxx", + "cm_bill_address2":"xxxxxxxxxx", + "cm_bill_city":"xxxxxxxxxx", + "cm_bill_state":"xxxxxxxxxx", + "cm_zip":"xxxxxxxxxx", + "zip5":"xxxxxxxxxx", + "cm_customer_id":"xxxxxxxxxx", + "final_gaid":"xxxxxxxxxx", + "final_hholdid":"xxxxxxxxxx", + "final_firstname":"xxxxxxxxxx", + "final_middlename":"xxxxxxxxxx", + "final_surname":"xxxxxxxxxx", + "final_gender":"xxxxxxxxxx", + "final_ace_prim_addr":"xxxxxxxxxx", + "final_ace_sec_addr":"xxxxxxxxxx", + "final_ace_urb":"xxxxxxxxxx", + "final_ace_city_llidx":"xxxxxxxxxx", + "final_ace_state":"xxxxxxxxxx", + "final_ace_postal_code":"xxxxxxxxxx", + "final_ace_zip4":"xxxxxxxxxx", + "final_ace_dpbc":"xxxxxxxxxx", + "final_ace_checkdigit":"xxxxxxxxxx", + "final_ace_iso_code":"xxxxxxxxxx", + "final_ace_cart":"xxxxxxxxxx", + "final_ace_lot":"xxxxxxxxxx", + "final_ace_lot_order":"xxxxxxxxxx", + "final_ace_rec_type":"xxxxxxxxxx", + "final_ace_remainder":"xxxxxxxxxx", + "final_ace_dpv_cmra":"xxxxxxxxxx", + "final_ace_dpv_ftnote":"xxxxxxxxxx", + "final_ace_dpv_status":"xxxxxxxxxx", + "final_ace_foreigncode":"xxxxxxxxxx", + "final_ace_match_5":"xxxxxxxxxx", + "final_ace_match_9":"xxxxxxxxxx", + "final_ace_match_un":"xxxxxxxxxx", + "final_ace_zip_move":"xxxxxxxxxx", + "final_ace_ziptype":"xxxxxxxxxx", + "final_ace_congress":"xxxxxxxxxx", + "final_ace_county":"xxxxxxxxxx", + "final_ace_countyname":"xxxxxxxxxx", + "final_ace_factype":"xxxxxxxxxx", + "final_ace_fipscode":"xxxxxxxxxx", + "final_ace_error_code":"xxxxxxxxxx", + "final_ace_stat_code":"xxxxxxxxxx", + "final_ace_geo_match":"xxxxxxxxxx", + "final_ace_geo_lat":"xxxxxxxxxx", + "final_ace_geo_lng":"xxxxxxxxxx", + "final_ace_ageo_pla":"xxxxxxxxxx", + "final_ace_geo_blk":"xxxxxxxxxx", + "final_ace_ageo_mcd":"xxxxxxxxxx", + "final_ace_cgeo_cbsa":"xxxxxxxxxx", + "final_ace_cgeo_msa":"xxxxxxxxxx", + "final_ace_ap_lacscode":"xxxxxxxxxx", + "final_dsf_businessflag":"xxxxxxxxxx", + "final_dsf_dropflag":"xxxxxxxxxx", + "final_dsf_throwbackflag":"xxxxxxxxxx", + "final_dsf_seasonalflag":"xxxxxxxxxx", + "final_dsf_vacantflag":"xxxxxxxxxx", + "final_dsf_deliverytype":"xxxxxxxxxx", + "final_dsf_dt_curbflag":"xxxxxxxxxx", + "final_dsf_dt_ndcbuflag":"xxxxxxxxxx", + "final_dsf_dt_centralflag":"xxxxxxxxxx", + "final_dsf_dt_doorslotflag":"xxxxxxxxxx", + "final_dsf_dropcount":"xxxxxxxxxx", + "final_dsf_nostatflag":"xxxxxxxxxx", + "final_dsf_educationalflag":"xxxxxxxxxx", + "final_dsf_rectyp":"xxxxxxxxxx", + "final_mailability_score":"xxxxxxxxxx", + "final_occupancy_score":"xxxxxxxxxx", + "final_multi_type":"xxxxxxxxxx", + "final_deceased_flag":"xxxxxxxxxx", + "final_dnm_flag":"xxxxxxxxxx", + "final_dnc_flag":"xxxxxxxxxx", + "final_dnf_flag":"xxxxxxxxxx", + "final_prison_flag":"xxxxxxxxxx", + "final_nursing_home_flag":"xxxxxxxxxx", + "final_date_of_birth":"xxxxxxxxxx", + "final_date_of_death":"xxxxxxxxxx", + "vip_number":"xxxxxxxxxx", + "vip_store_no":"xxxxxxxxxx", + "vip_division":"xxxxxxxxxx", + "vip_phone_number":"xxxxxxxxxx", + "vip_email_address":"xxxxxxxxxx", + "vip_first_name":"xxxxxxxxxx", + "vip_last_name":"xxxxxxxxxx", + "vip_gender":"xxxxxxxxxx", + "vip_status":"xxxxxxxxxx", + "vip_membership_date":"xxxxxxxxxx", + "vip_expiration_date":"xxxxxxxxxx", + "cm_date_addr_chng":"xxxxxxxxxx", + "cm_date_entered":"xxxxxxxxxx", + "cm_name":"xxxxxxxxxx", + "cm_opt_on_acct":"xxxxxxxxxx", + "cm_origin":"xxxxxxxxxx", + "cm_orig_acq_source":"xxxxxxxxxx", + "cm_phone_number":"xxxxxxxxxx", + "cm_phone_number2":"xxxxxxxxxx", + "cm_problem_cust":"xxxxxxxxxx", + "cm_rm_list":"xxxxxxxxxx", + "cm_rm_rented_list":"xxxxxxxxxx", + "cm_tax_code":"xxxxxxxxxx", + "email_address":"xxxxxxxxxx", + "esp_email_id":"xxxxxxxxxx", + "esp_sub_date":"xxxxxxxxxx", + "esp_unsub_date":"xxxxxxxxxx", + "cm_user_def_1":"xxxxxxxxxx", + "cm_user_def_7":"xxxxxxxxxx", + "do_not_phone":"xxxxxxxxxx", + "company_num":"xxxxxxxxxx", + "customer_id":"xxxxxxxxxx", + "load_date":"xxxxxxxxxx", + "activity_date":"xxxxxxxxxx", + "email_address_hashed":"xxxxxxxxxx", + "event_id":"", + "contains":{ + "uid": 123, + "flguid": 123, + "is_validate":"xxxxxxxxxx", + "createDatetime":"xxxxxxxxxx" + } + } + }]`) + + // we're parsing 125 nquads at a time, so the MB/s == MNquads/s + b.SetBytes(125) + for n := 0; n < b.N; n++ { + FastParse([]byte(json), SetNquads) + } } diff --git a/chunker/rdf_parser.go b/chunker/rdf_parser.go index 1f465095e36..ef904b86b70 100644 --- a/chunker/rdf_parser.go +++ b/chunker/rdf_parser.go @@ -22,7 +22,7 @@ import ( "strings" "unicode" - "github.com/dgraph-io/dgo/v200/protos/api" + "github.com/dgraph-io/dgo/v210/protos/api" "github.com/dgraph-io/dgraph/lex" "github.com/dgraph-io/dgraph/protos/pb" "github.com/dgraph-io/dgraph/types" @@ -36,9 +36,9 @@ var ( ErrEmpty = errors.New("RDF: harmless error, e.g. comment line") ) -// Function to do sanity check for subject, predicate, object and label strings. +// Function to do sanity check for subject, predicate and object strings. func sane(s string) bool { - // Label and ObjectId can be "", we already check that subject and predicate + // ObjectId can be "", we already check that subject and predicate // shouldn't be empty. if len(s) == 0 { return true @@ -194,7 +194,12 @@ L: break L case itemLabel: - rnq.Label = strings.TrimFunc(item.Val, isSpaceRune) + s := strings.TrimFunc(item.Val, isSpaceRune) + namespace, err := strconv.ParseUint(s, 0, 64) + if err != nil { + return rnq, errors.Errorf("Invalid namespace ID. Input: [%s]", line) + } + rnq.Namespace = namespace case itemLeftRound: it.Prev() // backup '(' @@ -221,8 +226,7 @@ L: if len(rnq.ObjectId) == 0 && rnq.ObjectValue == nil { return rnq, errors.Errorf("No Object in NQuad. Input: [%s]", line) } - if !sane(rnq.Subject) || !sane(rnq.Predicate) || - !sane(rnq.ObjectId) || !sane(rnq.Label) { + if !sane(rnq.Subject) || !sane(rnq.Predicate) || !sane(rnq.ObjectId) { return rnq, errors.Errorf("NQuad failed sanity check:%+v", rnq) } diff --git a/chunker/rdf_parser_test.go b/chunker/rdf_parser_test.go index 3aadbfec4b9..609674eb01f 100644 --- a/chunker/rdf_parser_test.go +++ b/chunker/rdf_parser_test.go @@ -19,7 +19,7 @@ package chunker import ( "testing" - "github.com/dgraph-io/dgo/v200/protos/api" + "github.com/dgraph-io/dgo/v210/protos/api" "github.com/dgraph-io/dgraph/lex" "github.com/dgraph-io/dgraph/types/facets" "github.com/dgraph-io/dgraph/x" @@ -346,40 +346,40 @@ var testNQuads = []struct { expectedErr: false, }, { - input: `_:alice "stuff"^^