diff --git a/content/riak/kv/2.2.6/_reference-links.md b/content/riak/kv/2.2.6/_reference-links.md new file mode 100644 index 0000000000..b5ceae4a02 --- /dev/null +++ b/content/riak/kv/2.2.6/_reference-links.md @@ -0,0 +1,248 @@ + +# Riak KV 2.2.6 Reference Links List + + +## Common + +[downloads]: {{}}riak/kv/2.2.6/downloads/ +[install index]: {{}}riak/kv/2.2.6/setup/installing +[upgrade index]: {{}}riak/kv/2.2.6/upgrading +[plan index]: {{}}riak/kv/2.2.6/planning +[config index]: {{}}riak/2.1.3/using/configuring/ +[config reference]: {{}}riak/kv/2.2.6/configuring/reference/ +[manage index]: {{}}riak/kv/2.2.6/using/managing +[performance index]: {{}}riak/kv/2.2.6/using/performance +[glossary vnode]: {{}}riak/kv/2.2.6/learn/glossary/#vnode +[contact basho]: https://www.tiot.jp/en/about-us/contact-us/ + + +## Planning + +[plan index]: {{}}riak/kv/2.2.6/setup/planning +[plan start]: {{}}riak/kv/2.2.6/setup/planning/start +[plan backend]: {{}}riak/kv/2.2.6/setup/planning/backend +[plan backend bitcask]: {{}}riak/kv/2.2.6/setup/planning/backend/bitcask +[plan backend leveldb]: {{}}riak/kv/2.2.6/setup/planning/backend/leveldb +[plan backend memory]: {{}}riak/kv/2.2.6/setup/planning/backend/memory +[plan backend multi]: {{}}riak/kv/2.2.6/setup/planning/backend/multi +[plan cluster capacity]: {{}}riak/kv/2.2.6/setup/planning/cluster-capacity +[plan bitcask capacity]: {{}}riak/kv/2.2.6/setup/planning/bitcask-capacity-calc +[plan best practices]: {{}}riak/kv/2.2.6/setup/planning/best-practices +[plan future]: {{}}riak/kv/2.2.6/setup/planning/future + + +## Installing + +[install index]: {{}}riak/kv/2.2.6/setup/installing +[install aws]: {{}}riak/kv/2.2.6/setup/installing/amazon-web-services +[install debian & ubuntu]: {{}}riak/kv/2.2.6/setup/installing/debian-ubuntu +[install freebsd]: {{}}riak/kv/2.2.6/setup/installing/freebsd +[install mac osx]: {{}}riak/kv/2.2.6/setup/installing/mac-osx +[install rhel & centos]: {{}}riak/kv/2.2.6/setup/installing/rhel-centos +[install smartos]: {{}}riak/kv/2.2.6/setup/installing/smartos +[install solaris]: {{}}riak/kv/2.2.6/setup/installing/solaris +[install suse]: {{}}riak/kv/2.2.6/setup/installing/suse +[install windows azure]: {{}}riak/kv/2.2.6/setup/installing/windows-azure + +[install source index]: {{}}riak/kv/2.2.6/setup/installing/source +[install source erlang]: {{}}riak/kv/2.2.6/setup/installing/source/erlang +[install source jvm]: {{}}riak/kv/2.2.6/setup/installing/source/jvm + +[install verify]: {{}}riak/kv/2.2.6/setup/installing/verify + + +## Upgrading + +[upgrade index]: {{}}riak/kv/2.2.6/setup/upgrading +[upgrade checklist]: {{}}riak/kv/2.2.6/setup/upgrading/checklist +[upgrade version]: {{}}riak/kv/2.2.6/setup/upgrading/version +[upgrade cluster]: {{}}riak/kv/2.2.6/setup/upgrading/cluster +[upgrade mdc]: {{}}riak/kv/2.2.6/setup/upgrading/multi-datacenter +[upgrade downgrade]: {{}}riak/kv/2.2.6/setup/downgrade + + +## Configuring + +[config index]: {{}}riak/kv/2.2.6/configuring +[config basic]: {{}}riak/kv/2.2.6/configuring/basic +[config backend]: {{}}riak/kv/2.2.6/configuring/backend +[config manage]: {{}}riak/kv/2.2.6/configuring/managing +[config reference]: {{}}riak/kv/2.2.6/configuring/reference/ +[config strong consistency]: {{}}riak/kv/2.2.6/configuring/strong-consistency +[config load balance]: {{}}riak/kv/2.2.6/configuring/load-balancing-proxy +[config mapreduce]: {{}}riak/kv/2.2.6/configuring/mapreduce +[config search]: {{}}riak/kv/2.2.6/configuring/search/ + +[config v3 mdc]: {{}}riak/kv/2.2.6/configuring/v3-multi-datacenter +[config v3 nat]: {{}}riak/kv/2.2.6/configuring/v3-multi-datacenter/nat +[config v3 quickstart]: {{}}riak/kv/2.2.6/configuring/v3-multi-datacenter/quick-start +[config v3 ssl]: {{}}riak/kv/2.2.6/configuring/v3-multi-datacenter/ssl + +[config v2 mdc]: {{}}riak/kv/2.2.6/configuring/v2-multi-datacenter +[config v2 nat]: {{}}riak/kv/2.2.6/configuring/v2-multi-datacenter/nat +[config v2 quickstart]: {{}}riak/kv/2.2.6/configuring/v2-multi-datacenter/quick-start +[config v2 ssl]: {{}}riak/kv/2.2.6/configuring/v2-multi-datacenter/ssl + + + +## Using + +[use index]: {{}}riak/kv/2.2.6/using/ +[use admin commands]: {{}}riak/kv/2.2.6/using/cluster-admin-commands +[use running cluster]: {{}}riak/kv/2.2.6/using/running-a-cluster + +### Reference + +[use ref custom code]: {{}}riak/kv/2.2.6/using/reference/custom-code +[use ref handoff]: {{}}riak/kv/2.2.6/using/reference/handoff +[use ref monitoring]: {{}}riak/kv/2.2.6/using/reference/statistics-monitoring +[use ref search]: {{}}riak/kv/2.2.6/using/reference/search +[use ref 2i]: {{}}riak/kv/2.2.6/using/reference/secondary-indexes +[use ref snmp]: {{}}riak/kv/2.2.6/using/reference/snmp +[use ref strong consistency]: {{}}riak/kv/2.2.6/using/reference/strong-consistency +[use ref jmx]: {{}}riak/kv/2.2.6/using/reference/jmx +[use ref obj del]: {{}}riak/kv/2.2.6/using/reference/object-deletion/ +[use ref v3 mdc]: {{}}riak/kv/2.2.6/using/reference/v3-multi-datacenter +[use ref v2 mdc]: {{}}riak/kv/2.2.6/using/reference/v2-multi-datacenter + +### Cluster Admin + +[use admin index]: {{}}riak/kv/2.2.6/using/admin/ +[use admin commands]: {{}}riak/kv/2.2.6/using/admin/commands/ +[use admin riak cli]: {{}}riak/kv/2.2.6/using/admin/riak-cli/ +[use admin riak-admin]: {{}}riak/kv/2.2.6/using/admin/riak-admin/ +[use admin riak control]: {{}}riak/kv/2.2.6/using/admin/riak-control/ + +### Cluster Operations + +[cluster ops add remove node]: {{}}riak/kv/2.2.6/using/cluster-operations/adding-removing-nodes +[cluster ops inspect node]: {{}}riak/kv/2.2.6/using/cluster-operations/inspecting-node +[cluster ops change info]: {{}}riak/kv/2.2.6/using/cluster-operations/changing-cluster-info +[cluster ops load balance]: {{}}riak/kv/2.2.6/configuring/load-balancing-proxy +[cluster ops bucket types]: {{}}riak/kv/2.2.6/using/cluster-operations/bucket-types +[cluster ops handoff]: {{}}riak/kv/2.2.6/using/cluster-operations/handoff +[cluster ops log]: {{}}riak/kv/2.2.6/using/cluster-operations/logging +[cluster ops obj del]: {{}}riak/kv/2.2.6/using/reference/object-deletion +[cluster ops backup]: {{}}riak/kv/2.2.6/using/cluster-operations/backing-up +[cluster ops mdc]: {{}}riak/kv/2.2.6/using/cluster-operations/v3-multi-datacenter +[cluster ops strong consistency]: {{}}riak/kv/2.2.6/using/cluster-operations/strong-consistency +[cluster ops 2i]: {{}}riak/kv/2.2.6/using/reference/secondary-indexes +[cluster ops v3 mdc]: {{}}riak/kv/2.2.6/using/cluster-operations/v3-multi-datacenter +[cluster ops v2 mdc]: {{}}riak/kv/2.2.6/using/cluster-operations/v2-multi-datacenter + +### Repair/Recover + +[repair recover index]: {{}}riak/kv/2.2.6/using/repair-recovery +[repair recover index]: {{}}riak/kv/2.2.6/using/repair-recovery/failure-recovery/ + +### Security + +[security index]: {{}}riak/kv/2.2.6/using/security/ +[security basics]: {{}}riak/kv/2.2.6/using/security/basics +[security managing]: {{}}riak/kv/2.2.6/using/security/managing-sources/ + +### Performance + +[perf index]: {{}}riak/kv/2.2.6/using/performance/ +[perf benchmark]: {{}}riak/kv/2.2.6/using/performance/benchmarking +[perf open files]: {{}}riak/kv/2.2.6/using/performance/open-files-limit/ +[perf erlang]: {{}}riak/kv/2.2.6/using/performance/erlang +[perf aws]: {{}}riak/kv/2.2.6/using/performance/amazon-web-services +[perf latency checklist]: {{}}riak/kv/2.2.6/using/performance/latency-reduction + +### Troubleshooting + +[troubleshoot http]: {{}}riak/kv/2.2.6/using/troubleshooting/http-204 + + +## Developing + +[dev index]: {{}}riak/kv/2.2.6/developing +[dev client libraries]: {{}}riak/kv/2.2.6/developing/client-libraries +[dev data model]: {{}}riak/kv/2.2.6/developing/data-modeling +[dev data types]: {{}}riak/kv/2.2.6/developing/data-types +[dev kv model]: {{}}riak/kv/2.2.6/developing/key-value-modeling + +### Getting Started + +[getting started]: {{}}riak/kv/2.2.6/developing/getting-started +[getting started java]: {{}}riak/kv/2.2.6/developing/getting-started/java +[getting started ruby]: {{}}riak/kv/2.2.6/developing/getting-started/ruby +[getting started python]: {{}}riak/kv/2.2.6/developing/getting-started/python +[getting started php]: {{}}riak/kv/2.2.6/developing/getting-started/php +[getting started csharp]: {{}}riak/kv/2.2.6/developing/getting-started/csharp +[getting started nodejs]: {{}}riak/kv/2.2.6/developing/getting-started/nodejs +[getting started erlang]: {{}}riak/kv/2.2.6/developing/getting-started/erlang +[getting started golang]: {{}}riak/kv/2.2.6/developing/getting-started/golang + +[obj model java]: {{}}riak/kv/2.2.6/developing/getting-started/java/object-modeling +[obj model ruby]: {{}}riak/kv/2.2.6/developing/getting-started/ruby/object-modeling +[obj model python]: {{}}riak/kv/2.2.6/developing/getting-started/python/object-modeling +[obj model csharp]: {{}}riak/kv/2.2.6/developing/getting-started/csharp/object-modeling +[obj model nodejs]: {{}}riak/kv/2.2.6/developing/getting-started/nodejs/object-modeling +[obj model erlang]: {{}}riak/kv/2.2.6/developing/getting-started/erlang/object-modeling +[obj model golang]: {{}}riak/kv/2.2.6/developing/getting-started/golang/object-modeling + +### Usage + +[usage index]: {{}}riak/kv/2.2.6/developing/usage +[usage bucket types]: {{}}riak/kv/2.2.6/developing/usage/bucket-types +[usage commit hooks]: {{}}riak/kv/2.2.6/developing/usage/commit-hooks +[usage conflict resolution]: {{}}riak/kv/2.2.6/developing/usage/conflict-resolution +[usage content types]: {{}}riak/kv/2.2.6/developing/usage/content-types +[usage create objects]: {{}}riak/kv/2.2.6/developing/usage/creating-objects +[usage custom extractors]: {{}}riak/kv/2.2.6/developing/usage/custom-extractors +[usage delete objects]: {{}}riak/kv/2.2.6/developing/usage/deleting-objects +[usage mapreduce]: {{}}riak/kv/2.2.6/developing/usage/mapreduce +[usage search]: {{}}riak/kv/2.2.6/developing/usage/search +[usage search schema]: {{}}riak/kv/2.2.6/developing/usage/search-schemas +[usage search data types]: {{}}riak/kv/2.2.6/developing/usage/searching-data-types +[usage 2i]: {{}}riak/kv/2.2.6/developing/usage/secondary-indexes +[usage update objects]: {{}}riak/kv/2.2.6/developing/usage/updating-objects + +### App Guide + +[apps mapreduce]: {{}}riak/kv/2.2.6/developing/app-guide/advanced-mapreduce +[apps replication properties]: {{}}riak/kv/2.2.6/developing/app-guide/replication-properties +[apps strong consistency]: {{}}riak/kv/2.2.6/developing/app-guide/strong-consistency + +### API + +[dev api backend]: {{}}riak/kv/2.2.6/developing/api/backend +[dev api http]: {{}}riak/kv/2.2.6/developing/api/http +[dev api http status]: {{}}riak/kv/2.2.6/developing/api/http/status +[dev api pbc]: {{}}riak/kv/2.2.6/developing/api/protocol-buffers/ + + +## Learn + +[learn new nosql]: {{}}riak/kv/learn/new-to-nosql +[learn use cases]: {{}}riak/kv/learn/use-cases +[learn why riak]: {{}}riak/kv/learn/why-riak-kv + +[glossary]: {{}}riak/kv/2.2.6/learn/glossary/ +[glossary aae]: {{}}riak/kv/2.2.6/learn/glossary/#active-anti-entropy-aae +[glossary read rep]: {{}}riak/kv/2.2.6/learn/glossary/#read-repair +[glossary vnode]: {{}}riak/kv/2.2.6/learn/glossary/#vnode + +[concept aae]: {{}}riak/kv/2.2.6/learn/concepts/active-anti-entropy/ +[concept buckets]: {{}}riak/kv/2.2.6/learn/concepts/buckets +[concept cap neg]: {{}}riak/kv/2.2.6/learn/concepts/capability-negotiation +[concept causal context]: {{}}riak/kv/2.2.6/learn/concepts/causal-context +[concept clusters]: {{}}riak/kv/2.2.6/learn/concepts/clusters/ +[concept crdts]: {{}}riak/kv/2.2.6/learn/concepts/crdts +[concept eventual consistency]: {{}}riak/kv/2.2.6/learn/concepts/eventual-consistency +[concept keys objects]: {{}}riak/kv/2.2.6/learn/concepts/keys-and-objects +[concept replication]: {{}}riak/kv/2.2.6/learn/concepts/replication +[concept strong consistency]: {{}}riak/kv/2.2.6/using/reference/strong-consistency +[concept vnodes]: {{}}riak/kv/2.2.6/learn/concepts/vnodes + + + +## Community + +[community]: {{}}community +[community projects]: {{}}community/projects +[reporting bugs]: {{}}community/reporting-bugs +[taishi]: {{}}community/taishi + diff --git a/content/riak/kv/2.2.6/add-ons.md b/content/riak/kv/2.2.6/add-ons.md new file mode 100644 index 0000000000..99d331224d --- /dev/null +++ b/content/riak/kv/2.2.6/add-ons.md @@ -0,0 +1,19 @@ +--- +title: "Add-ons" +description: "Add-on technology for Riak KV" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Add-ons" + identifier: "add-ons" + weight: 400 + pre: tools +toc: true +--- + + + +Here at Basho, we've developed integrations between Riak KV and other best-of-breed components in your application stack. Each integration, called an add-on, is explained in this section, from installation to feature-set. + +* [Riak Redis Add-on]({{}}riak/kv/2.2.6/add-ons/redis/) \ No newline at end of file diff --git a/content/riak/kv/2.2.6/add-ons/redis.md b/content/riak/kv/2.2.6/add-ons/redis.md new file mode 100644 index 0000000000..0eb9075350 --- /dev/null +++ b/content/riak/kv/2.2.6/add-ons/redis.md @@ -0,0 +1,56 @@ +--- +title: "Riak Redis Add-on" +description: "Redis Add-on for Riak KV" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Redis Add-on" + identifier: "add-ons_redis" + weight: 101 + parent: "add-ons" +toc: true +commercial_offering: true +--- + + +[addon redis develop]: ./developing-rra/ +[addon redis features]: ./redis-add-on-features/ +[addon redis setup]: ./set-up-rra/ +[addon redis use]: ./get-started-with-rra/ +[ee]: https://www.tiot.jp/en/about-us/contact-us/ + + +Riak Redis Add-on (RRA) is a distributed cache service that joins the power of Redis caching with the eventual consistency guarantees of Riak KV. + +RRA enables you to reduce latency for Riak KV reads through the use of a distributed cache layer. This type of caching is most effective for keys that are immutable or have an infrequent change rate. + +Whether you are looking to build out a session, shopping cart, advertisement or other dynamically-rendered copy, RRA helps reduce read pressure on your persistent store (Riak KV). + +## Compatibility + +RRA is supported on the following platforms: + +* RHEL/CentOS 6 +* RHEL/CentOS 7 +* Ubuntu 12.04 LTS "Precise Pangolin" +* Ubuntu 14.04 LTS "Trusty Tahr" +* Debian 7 "Wheezy" +* Debian 8 "Jessie" + +RRA is compatible with the following services: + +* Riak KV Enterprise (2.1.4+) +* Riak TS Enterprise (1.4.0+) +* Redis 2.x and 3.x (in 3.x, not supporting Redis Cluster) + * Redis Cluster and RRA's consistent hash are at odds, which surface as errors + such as MOVED, ASK, and CROSSSLOT messages from Redis, see (WIP): + https://github.com/antirez/redis-rb-cluster + +## Get Started + +* [Set up RRA.][addon redis setup] +* [Use RRA with various clients.][addon redis use] +* [Develop with RRA.][addon redis develop] +* [Learn about RRA's features.][addon redis features] + diff --git a/content/riak/kv/2.2.6/add-ons/redis/developing-rra.md b/content/riak/kv/2.2.6/add-ons/redis/developing-rra.md new file mode 100644 index 0000000000..008320a2c1 --- /dev/null +++ b/content/riak/kv/2.2.6/add-ons/redis/developing-rra.md @@ -0,0 +1,325 @@ +--- +title: "Developing with Riak Redis Add-on" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Develop with Redis Add-on" + identifier: "add-ons_redis_develop" + weight: 403 + parent: "add-ons_redis" +toc: true +commercial_offering: true +--- + +[redis-clients]: http://redis.io/clients +[usage bucket types]: {{}}riak/kv/2.2.6/developing/usage/bucket-types/ +[dev api http]: {{}}riak/kv/2.2.6/developing/api/http +[config-behaviors]: http://basho.com/posts/technical/riaks-config-behaviors-part-4/ +[apps replication properties]: {{}}riak/kv/2.2.6/developing/app-guide/replication-properties +[usage commit hooks]: {{}}riak/kv/2.2.6/developing/usage/commit-hooks/ +[concept causal context]: {{}}riak/kv/2.2.6/learn/concepts/causal-context +[ee]: https://www.tiot.jp/en/about-us/contact-us/ + +This page will walk you through setting up your environment for development with Riak Redis Add-on (RRA), as well as present examples and configuration parameters for basic development operations. + +## Overview + +Riak Redis Add-on (RRA) packages a cache proxy service. The cache proxy service provides accessibility to Riak KV, as a persistent data store, with Redis, as a cache through the various Redis client libraries and command-line interface tool `redis-cli`. + +As with Riak KV, the cache proxy service almost always performs best and most +predictably when you use the basic CRUD operations -- Create, Read, Update, +Delete -- that you'd find in any key/value store. Learning these operations +is a great place to start when beginning to develop applications that use +RRA. + +The set of clients (including recommendations) for Redis are listed at +[Redis clients][redis-clients]. For brevity sake, examples provided here are +in: + +* Erlang (Eredis) +* Javascript (node_redis) +* Python (redis-py) +* Ruby (redis-rb) +* Scala (lettuce) +* Java, see the Scala examples. The code intentionally uses as few Scala tricks as possible to focus on the use of the Redis client. + +## Riak KV Setup + +While you can use Riak Redis Add-on with Riak KV configured so either `last_write_wins` is set to 'true' or `allow_mult` is set to 'true', we recommend using the `allow_mult` setting in order to provide client sibling resolution in the event of a network partition. The examples and instructions on this page will assume `allow_mult` is set to 'true'. + +The cache proxy service is tested under both configurations. However, due to lack of support via the Redis protocol for returning multiple values for a single `GET`, effectively `last_write_wins` semantics apply. + +For a deeper explanation of Riak KV's configurable behaviors, see John Daily's +blog series [part 4][config-behaviors] . + +### Bucket Type Setup + +#### Create a Bucket Type + +If your application organizes data in a way that does not include bucket-type +and instead only uses bucket to organize its keyspace, the `default` bucket-type +can be used by omitting the bucket-type portion of the colon-delimited +hierarchical namespaced key. Otherwise said, `test:food` is equivalent to +`default:test:food` where the bucket-type is `default`, the bucket is `test`, +and the key is `food`. For examples here, we will use `rra:test:food` to clearly +use a bucket-type. + +If your application organizes data including a bucket-type, ensure that that +bucket-type is created in Riak without specifying the data type, so effectively +an opaque value, ie a `string`. The following command provides an example of +creating the bucket-type `rra`: + +```sh +if ! riak-admin bucket-type status rra >/dev/null 2>&1; then + riak-admin bucket-type create rra '{"props":{}}' + riak-admin bucket-type activate rra +fi +``` + +#### Set Bucket Props + +The following is an example, using Riak KV's default HTTP port, of setting `allow_mult` to 'true' and `last_write_wins` to 'false': + +```sh +curl -XPUT -H 'Content-Type: application/json' \ + -d '{"props": {"allow_mult": true, "last_write_wins": false}}' \ + 'http://127.0.0.1:8098/types/rra/buckets/test/props' +``` + +For additional configuration options see [bucket properties][dev api http]. + +## Object/Key Operations + +Riak KV organizes data into buckets, keys, and values, with +[bucket types][usage bucket types] acting as an additional namespace in Riak KV +versions 2.0 and greater. Values, which we'll refer to as objects, are identifiable by a unique key, and each key/value pair is stored in a bucket. + +Objects accessed via the cache proxy service in Riak Redis Add-on are restricted to plaintext format. This plaintext format may be a simple string, JSON, XML, or other plaintext representations that can be parsed in the client application (e.g. YAML). + +While buckets are a flat namespace in Riak KV and you can name them +whatever you'd like (`bucket` or `a90bf521c` or `___`), within the cache proxy +service, Redis bucket_type:bucket:key is mapped to Riak KV +bucket_type/bucket/key, so bucket type and bucket names should not contain +colon (`:`). When not specified, bucket type defaults to "default". + +Outside of the above restriction, bucket names have no intrinsic significance beyond allowing you to store objects with the same key in different buckets. + +The same goes for naming keys: many objects can have the same key as long as they're in different buckets. There is no restriction on key containing colon (`:`), and this practice of representing a nested namespace is common in applications using Redis. + +Riak KV [bucket types][usage bucket types] enable you to provide common +configurations for buckets (as many buckets as you wish). This means you can +easily enable buckets to share common configurations, i.e. identical +[replication properties][apps replication properties] or +[commit hooks][usage commit hooks]. + + +## Reading Objects + +Reads via the cache proxy service are analogous to a Redis `GET`, with the added benefit of reading-through to Riak KV which results in greater resilience through node outages and network partitions. + +To request a value at a bucket/key in Riak KV, issue the following: + +```erlang +{ok, RedisClientPid} = eredis:start_link("127.0.0.1", 22122). +{ok, Value} = eredis:q(RedisClientPid, ["GET", "rra:test:food"]). +``` + +```javascript +var redis = require("redis"), + client = redis.createClient(22122, "127.0.0.1"); + +client.get("rra:test:food", redis.print); +``` + +```python +import redis + +r = redis.StrictRedis(host="127.0.0.1", port=22122) + +r.get("rra:test:food") +``` + +```ruby +require "redis" + +redis = Redis.new(host: "127.0.0.1", port: 22122) + +redis.get("rra:test:food") +``` + +```scala +import com.lambdaworks.redis._ + +var client = RedisClient.create("redis://127.0.0.1:22122") +var connection = client.connect() + +var value = connection.get("rra:test:food") +``` + +### Get Configuration Parameters + +>**Note:** The cache proxy service read option (related to replication factor and +consistency concern) may optionally be set within the nutcracker.conf. This will result in an override of the setting value at the bucket-level in Riak KV. + +The following configuration parameters apply to `GET` and may be set within the +RRA configuration file `/etc/cache_proxy/cache_proxy_22122.yml`: + +|Parameter |Description |Default| +|----------------|-----------------|-------| +|`n_val` | The number of replicas for objects in a bucket. The `n_val` should be an integer greater than 0 and less than or equal to the number of nodes in the cluster.

**NOTE**: If you change the `n_val` after keys have been added to the bucket it may result in failed reads, as the new value may not be replicated to all of the appropriate partitions. | `3` | +|`pr` | How many vnodes must respond for a read to be deemed successful. | `0` | +|`r` | How many replicas need to agree when retrieving an existing object before responding. | `2` | +|`basic_quorum` | Whether to return early in some failure cases, e.g. when `r`=1 and you get 2 errors and a success. | `0` (false) | +|`sloppy_quorum` | Whether to treat vnodes holding values for another vnode as acceptable within the quorum determination. | `0` (false) | +|`notfound_ok` | Whether to treat notfounds as successful reads for the purpose of `r`. | 1 (true) | +|`timeout` | The number of milliseconds to await a response. | `0` (server specified) | + + +### Sibling Resolution + +As the Redis protocol does not provide a means to return multiple siblings, +the cache proxy service must provide server-side sibling resolution. At present, only last-write-wins sibling resolution is available. The result is an effective +last-write-wins configuration for access through the cache proxy service. + + +## Writing Objects + +Writes via the cache proxy service are analogous to a Redis `SET`, with the added +benefit of writing to Riak KV followed by a `PEXPIRE` to Redis, invalidating +cache. As with HTTP PUT, `SET` semantically covers both create and update +operations. + +To set a value at a bucket/key in Riak KV, issue the following: + +```erlang +{ok, RedisClientPid} = eredis:start_link("127.0.0.1", 22122). +{ok, KeysAffected} = eredis:q(RedisClientPid, ["SET", "rra:test:food", "apple"]). +``` + +```javascript +var redis = require("redis"), + client = redis.createClient(22122, "127.0.0.1"); + +client.set("rra:test:food", "apple", redis.print); +``` + +```python +import redis + +r = redis.StrictRedis(host="127.0.0.1", port=22122) + +r.set("rra:test:food", "apple") +``` + +```ruby +require "redis" + +redis = Redis.new(host: "127.0.0.1", port: 22122) + +redis.set("rra:test:food', 'apple") +``` + +```scala +import com.lambdaworks.redis._ + +var client = RedisClient.create("redis://127.0.0.1:22122") +var connection = client.connect() + +connection.set("rra:test:food", "apple") +``` + +### Set Configuration Parameters + +>**Note:** The cache proxy service write option (related to replication factor and +consistency concern) may optionally be set within the nutcracker.conf, resulting +in an override of the setting value at the bucket-level in Riak KV. + +The following configuration parameters apply to `SET` and may be set within the +RRA configuration file `/etc/cache_proxy/cache_proxy_22122.yml`: + +|Parameter |Description |Default| +|----------------|-----------------|-------| +|`n_val` | The number of replicas for objects in a bucket. The `n_val` should be an integer greater than 0 and less than or equal to the number of nodes in the cluster.

**NOTE**: If you change the `n_val` after keys have been added to the bucket it may result in failed reads, as the new value may not be replicated to all of the appropriate partitions. | `3` | +|`pw` | How many vnodes must respond for a write to be deemed successful. | `0` | +|`w` | How many replicas need to acknowledge the write before responding. | `2` | +|`sloppy_quorum` | Whether to treat vnodes holding values for another vnode as acceptable within the quorum determination. | `0` (false) | + + +### Sibling Explosion + +As noted in the section "Sibling Resolution" above, Riak KV provides for a line of +descendency (known as the [causal context][[concept causal context]]) for a value stored at a key. Clients +performing write operations provide this causal context by setting the vector +clock (VClock) that they last read. + +If a client does not provide the causal context, Riak KV makes no assumptions and treats the write as a new causal context, semantically equivalent to a +create. In the case that a value is already stored at the key, this would lead +to a sibling. + +Since the Redis protocol does not provide a means to pass a VClock, the cache +proxy service needs to perform a read-before-write to obtain the current VClock so the write can continue the causal context previously established and avoid +"sibling explosion". + +Despite these efforts, in the event of a network partition, siblings will still +be created as clients writing to nodes on either side of the network partition +can create divergent lines of descendency. Sibling resolution remains the means +to merge these lines of descent into a coherent causal context. + +## Deleting Objects + +Deletes via the cache proxy service are analogous to a Redis `DEL`, with the added +benefit of writing to Riak KV followed by a `PEXPIRE` to Redis, invalidating +cache. + +To delete a value at a bucket/key in Riak KV, issue the following: + +```erlang +{ok, RedisClientPid} = eredis:start_link("127.0.0.1", 22122). +{ok, KeysAffected} = eredis:q(RedisClientPid, ["DEL", "rra:test:food"]). +``` + +```javascript +var redis = require("redis"), + client = redis.createClient(22122, "127.0.0.1"); + +client.del("rra:test:food", redis.print); +``` + +```python +import redis + +r = redis.StrictRedis(host="127.0.0.1", port=22122) + +r.del("rra:test:food") +``` + +```ruby +require "redis" + +redis = Redis.new(host: "127.0.0.1", port: 22122) + +redis.del("rra:test:food") +``` + +```scala +import com.lambdaworks.redis._ + +var client = RedisClient.create("redis://127.0.0.1:22122") +var connection = client.connect() + +connection.del("rra:test:food") +``` + +### Delete Configuration Parameters + +The following configuration parameters apply to `DEL` and may be set within the +RRA configuration file `/etc/cache_proxy/cache_proxy_22122.yml`: + +|Parameter |Description |Default| +|----------------|-----------------|-------| +|`n_val` | The number of replicas for objects in a bucket. The `n_val` should be an integer greater than 0 and less than or equal to the number of nodes in the cluster.

**NOTE**: If you change the `n_val` after keys have been added to the bucket it may result in failed reads, as the new value may not be replicated to all of the appropriate partitions. | `3` | +|`pw` | How many vnodes must respond for a write to be deemed successful. | `0` | +|`w` | How many replicas need to acknowledge the write before responding. | `2` | +|`sloppy_quorum` | Whether to treat vnodes holding values for another vnode as acceptable within the quorum determination. | `0` (false) | diff --git a/content/riak/kv/2.2.6/add-ons/redis/redis-add-on-features.md b/content/riak/kv/2.2.6/add-ons/redis/redis-add-on-features.md new file mode 100644 index 0000000000..70c7efe157 --- /dev/null +++ b/content/riak/kv/2.2.6/add-ons/redis/redis-add-on-features.md @@ -0,0 +1,131 @@ +--- +title: "Riak Redis Add-on Features" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Redis Add-on Features" + identifier: "add-ons_redis_features" + weight: 504 + parent: "add-ons_redis" +toc: true +commercial_offering: true +--- + +[ee]: https://www.tiot.jp/en/about-us/contact-us/ +[GET-sequence]: {{}}images/redis/GET_seq.msc.png +[SET-sequence]: {{}}images/redis/SET_seq.msc.png +[DEL-sequence]: {{}}images/redis/DEL_seq.msc.png +[Object-lifetime]: {{}}images/redis/Object_lifetime.msc.png +[redis docs]: http://redis.io/commands +[twemproxy docs]: https://github.com/twitter/twemproxy/blob/master/notes/redis.md + +## Overview + +The cache proxy service in Riak Redis Add-on (RRA) provides pre-sharding and connection aggregation as a service, which reduces latency and increases addressable cache memory space with lower-cost hardware. + +On this page, you will find detailed descriptions of cache proxy service components, including what each component does and how you implement it. The following components are available: + +* [Pre-sharding](#pre-sharding) +* [Connection Aggregation](#connection-aggregation) +* [Command Pipelining](#command-pipelining) +* [Read-through Cache](#read-through-cache) +* [Write-around Cache](#write-around-cache) +* [Commands](#commands) +* [Object Lifetime](#object-lifetime) + +## Pre-sharding + +Pre-sharding with consistent hashing dispatches object reads and writes based +on a configurable hash function, spreading load across multiple cache servers. +The cache proxy service uses pre-sharding to extend the total addressable cache memory space based on the number of Redis servers. Request keys are hashed, then +requests are routed to the Redis server that handles that portion of the key +range. + +Redis with no persistence is used as the frontend cache proxy service, and +Redis as a data server holds all data in memory. The addressable memory of +cache proxy is limited. By employing pre-sharding, the total addressable cache +memory space is extended by the number of Redis servers. + +## Connection Aggregation + +Redis client connections are a limited resource. Using the cache proxy service, connections may be spread across multiple Riak Redis Add-on (RRA) servers. This reduces the total required connections to the Redis server for the same key. + +Redis clients in various languages support specifying multiple servers, as well +as implementing multiple methods of spreading load across those servers (i.e. +round-robin load balancing or consistent hashing). Since the cache proxy service is providing consistent hashing, any Redis client method of supporting multiple +servers will suffice. + +## Command Pipelining + +The cache proxy service increases performance by pipelining requests to Redis. While pipelining can be performed at the client, the cache proxy service is ideal due to connection aggregation. Pipelining reduces network roundtrips to Redis and +lowers CPU usage on Redis. + +## Read-Through Cache + +Implementing caching strategies in the cache proxy service reduces the cost of implementing cache strategies in client code in multiple applications and languages. The cache proxy service supports the read-through cache strategy, the most prevalent caching strategy used in distributed computing. + +The read-through cache strategy of the GET command is represented by the +following sequence diagram: + +![GET command sequence diagram]({{}}images/redis/GET_seq.msc.png) + + +The `CACHE_TTL` configuration option establishes how long the cache takes to +become consistent with the backend server during a write (DELETE or PUT) to the +backend server. + +A short `CACHE_TTL`, for example "15s", reduces a significant amount of read +pressure from Riak, increasing performance of the overall solution. + +## Write-Around Cache + +The read-through cache strategy requires a TTL to keep cache as coherent as possible given that writes to Riak KV can and will be issued without the cache proxy service being informed of the write. The effect is that the cache proxy service is eventually consistent with the underlying Riak KV data store, with the time to consistency equal to the TTL. + +The cache proxy service write-around cache strategy was introduced to provide a means to keep cache coherent with zero time to consistency with the underlying Riak KV data store for all writes that the cache proxy is informed of. For the Redis String (Value in KV) datatype, SET and DEL commands result in writes to the underlying Riak KV data store followed by a PEXPIRE to invalidate cache. + +Of the three write cache strategies, the write-around cache strategy is the least +prone to race condition, but least optimal for the read which immediately follows +the write. In the overwhelming majority of distributed application data access +patterns, the added certainty of cache coherency afforded by write-around over +write-through is well worth the single cache miss. By definition, a key that is +cached is expected to be accessed frequently, hence the single cache miss is +expected to be followed by several accurate cache hits. + +The write-around cache strategy of the SET command is represented by the +following sequence diagram: + +![SET command sequence diagram]({{}}images/redis/SET_seq.msc.png) + +The write-around cache strategy of the DEL command is represented by the +following sequence diagram: + +![DEL command sequence diagram]({{}}images/redis/DEL_seq.msc.png) + +## Commands + +For command details, refer to the Redis [documentation][redis docs]. + +The cache proxy service supports the following augmented Redis commands fully: + +* GET - get the value of a key from Redis or Riak KV utilizing the read-through + caching strategy with a TTL set at service configuration time. + +* SET - set the value of a key to Riak KV and invalidate cache, issue a PEXPIRE + to Redis. + +* DEL - delete the value of a key to Riak KV and invalidate cache, issue a + PEXPIRE to Redis. + +The cache proxy service also supports the set of Redis commands supported by Twemproxy, but only to the point of pre-sharding and command pipelining, issued only to Redis. Refer to the Twemproxy [documentation][twemproxy docs]. + +>**Important:** While the cache proxy service does support issuing DEL commands, PEXPIRE, with a small TTL, is suggested instead when the semantic intent is to remove an item from cache. With write-around, the DEL command will issue a delete to the Riak backend. + +## Object Lifetime + +With the combination of read-through and write-around cache strategies, the +full object lifetime for a key-value is represented by the following +sequence diagram: + +![Object lifetime sequence diagram]({{}}images/redis/Object_lifetime.msc.png) diff --git a/content/riak/kv/2.2.6/add-ons/redis/set-up-rra.md b/content/riak/kv/2.2.6/add-ons/redis/set-up-rra.md new file mode 100644 index 0000000000..c246f76417 --- /dev/null +++ b/content/riak/kv/2.2.6/add-ons/redis/set-up-rra.md @@ -0,0 +1,280 @@ +--- +title: "Setting Up Riak Redis Add-on" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Set Up Redis Add-on" + identifier: "add-ons_redis_setup" + weight: 201 + parent: "add-ons_redis" +toc: true +commercial_offering: true +--- + +[addon redis develop]: ../developing-rra/ +[addon redis use]: ../using-rra +[ee]: https://www.tiot.jp/en/about-us/contact-us/ +[install index]: {{}}riak/kv/2.2.6/setup/installing +[perf open files]: {{}}riak/kv/2.2.6/using/performance/open-files-limit/#changing-the-limit +[lab ansible]: https://github.com/paegun/ansible-cache-proxy + +This page will walk you through the process of installing Riak Redis Add-on (RRA) and configuring it to run in your environment. Check the [prerequisites](#prerequisites) before you get started to make sure you have everything you need in order to successfully install and use RRA. + +## Prerequisites + +Before you begin installing Riak Redis Add-on (RRA), you will need to ensure that you have root or sudo access on the nodes where you plan to install RRA. You will also need to have Riak KV already [installed][install index]. + +While this page assumes that Redis is not already installed, existing installations of Redis are supported. If you have an existing Redis installation, look for the *skip ahead* instructions as you go. + +This page assumes that Redis is (or will be) installed on separate hosts from Riak KV. You will need the list of Riak KV and Redis host:port combinations. RRA communicates with Riak KV via the protobuf port, and the host:port values are used +to configure the cache proxy. + +## In the Lab + +An ansible setup for the Riak Redis Add-on (RRA) was developed to provide a +runnable example of an installation, see [ansible cache proxy][lab ansible]. +The remainder of this setup guide lists the commands required to install and +configure RRA manually. + +## Installing + +1. On all Redis and Riak Redis Add-on hosts, change the [open-files limit][perf open files]. +2. On all Redis hosts, install Redis. **Skip ahead* if you already have Redis installed. +3. Install Riak Redis Add-on. + +### Change the open-files limit + +As with Riak KV, both the total open-files limit and the per-user open-files limit +must be high enough to allow Redis and Riak Redis Add-on (RRA) to function. + +For a complete guide on changing limit in Riak KV, see +[Changing the limit][perf open files]. + +#### Linux + +On most Linux distributions, the total limit for open files is controlled by `sysctl`. + +```bash +sudo sysctl fs.file-max fs.file-max=65536 +sudo sysctl -p +``` + +To change the per-user file limit, you need to edit `/etc/security/limits.conf`. + +#### CentOS + +On CentOS systems, set a proper limit for the user you're usually logging in with +to do any kind of work on the machine, including managing Riak KV, Redis, or RRA services. On CentOS, `sudo` properly inherits the values from the +executing user. + +#### Ubuntu + +On Ubuntu systems, the following settings are recommended: + +```config +»USERNAME« hard nofile 65536 +»USERNAME« soft nofile 65536 +root hard nofile 65536 +root soft nofile 65536 +``` + +>**Note:** You may need to log out of your shell and then log back in for these changes to take effect. + + +### Install Redis + +>**Note:** If you already have Redis installed, *skip ahead* to "Install Riak Redis Add-on". + +#### Install on Ubuntu + +If you are on Ubuntu, run the following to install Redis: + +```bash +# add the dotdeb repositories to your APT sources. +sudo bash -c "cat >> /etc/apt/sources.list.d/dotdeb.org.list" <**Notes:** ss is used here to support a minimal installed system, but netstat may be used as well. + +### Install Riak Redis Add-on (RRA) + +>**Note:** +>Riak Redis Add-on (RRA) is available to Enterprise customers for download in the usual Zendesk forums. + +If you are on CentOS, run the following to install RRA: + +```bash +sudo yum -y localinstall cache_proxy_ee_1.1.0_x86_64.rpm +``` + +If you are on Ubuntu, run the following to install RRA: + +```bash +sudo dpkg -i cache_proxy_ee_1.1.0_amd64.deb +``` + +## Configuring Riak Redis Add-on + +To configure Riak Redis Add-on (RRA), edit the configuration file: /etc/cache_proxy/cache_proxy_22122.yml. + +The RRA configuration file is in YAML format. An example configuration +file is provided in the install, and it contains all relevant configuration elements: + +```config +» XML node name« : + listen: 0.0.0.0:22122 + hash: fnv1a_64 + distribution: ketama + auto_eject_hosts: true + redis: true + server_retry_timeout: 2000 + server_failure_limit: 1 + server_ttl: 1h + servers: + - 127.0.0.1:6379:1 + backend_type: riak + backend_max_resend: 2 + backends: + - 127.0.0.1:8087 +``` + +Set the `listen` configuration value to set the RRA listen port. + +To set the time-to-live (TTL) for values stored in cache, set the `server_ttl` +configuration value. Human-readable time values can be specified, +with the most likely units being `s` for seconds or `ms` for milliseconds. + +Set the list of Redis servers by listing the servers, separated by `-`, under the `servers` configuration value in the format `»host«:»port«:»weight«` (weight is optional). + +Set the list of Riak KV servers by listing the servers, separated by `-`, under the `backends` configuration value in the format `»host«:»port«:»weight«` +(weight is optional). You will want to make sure to list the Riak KV protobuf (pb) port here. + +### Verify your configuration + +If you are on Ubuntu, run the following to start RRA: + +```bash +sudo service cache_proxy start +``` + +If you are on CentOS, run the following to restart Redis and ensure redis-server +is enabled to start on boot: + +```bash +systemctl start cache_proxy +``` + +To verify RRA is running and listening on the expected port, run the +following (using the loopback interface and the default RRA port 22122 +as an example): + +```bash +redis-cli -h 127.0.0.1 -p 22122 set test:redis-add-on SUCCESS +redis-cli -h 127.0.0.1 -p 22122 get test:redis-add-on SUCCESS +``` + +Redis should respond with `SUCCESS`. + +If RRA is responding with the expected output, run the following to +clean up and remove the test value: + +```bash +redis-cli -h 127.0.0.1 -p 22122 del test:redis-add-on +``` + +If you did not get the expected output, run the following +to verify that RRA is running on the expected port: + +```bash +ss -nlp |grep [n]utcracker +``` + +>**Note:** ss is used here to support a minimal installed system, but netstat may be used as well. + +## Next Steps + +Get started with some [basic usage][addon redis use] or checkout out more info on [setting up for development (with examples)][addon redis develop]. diff --git a/content/riak/kv/2.2.6/add-ons/redis/set-up-rra/deployment-models.md b/content/riak/kv/2.2.6/add-ons/redis/set-up-rra/deployment-models.md new file mode 100644 index 0000000000..f1761f2efb --- /dev/null +++ b/content/riak/kv/2.2.6/add-ons/redis/set-up-rra/deployment-models.md @@ -0,0 +1,138 @@ +--- +title: "Riak Redis Add-on Deployment Models" +description: "Explore the various models for deploying Riak Redis Add-on" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Redis Add-on Deployment Models" + identifier: "add-ons_redis_deployment" + weight: 201 + parent: "add-ons_redis_setup" +toc: true +commercial_offering: true +--- + +[Local-deployment]: {{}}images/redis/rra_deployment_local.png +[Colocated-deployment]: {{}}images/redis/rra_deployment_colocated.png +[Distributed-deployment]: {{}}images/redis/rra_deployment_distributed.png + +## Deployment Models + +### Local Cache Deployment + +In a local cache deployment, the RRA and Redis are deployed to the application +server. + +![Local-deployment]({{}}images/redis/rra_deployment_local.png) + +Connections: + +* RRA: The connections between Application Service instances to RRA Service + instance are local. +* Redis: The connection between the RRA Service instance and Redis Service + instance is local. +* Riak: The connections between Application Servers to Riak Nodes is distributed + and bounded to equal the number of Riak nodes _multiplied_ by the number of + Application Servers since they are aggregated at the RRA Service instance. + +Advantages: + +* Cache hits are extremely fast + +Disadvantages: + +* Cache writes on one application server are *not* observed on other application + servers, so cache hit rates are likely lower unless some form of consistent + routing to the application server exists within the solution. +* Redis competing for RAM with the application service may be problematic + +### Colocated Cache Deployment + +In a colocated cache deployment, the RRA may be deployed either to the +application server (suggested) or to the Riak servers and Redis is deployed to +the Riak servers. + +In the case of deploying the RRA to the application servers, the RRA features +of reducing connections from the relatively high number of application service +instances to the fewer Redis (cache) and Riak (persistent) data service +instances allows for the greatest scale at the expense of the deployment cost +of pushing a service and its configuration. + +In the case of deploying the RRA to the colocated Redis and Riak data servers, +the maximum scale for the solution is contrained by the number of network +connections from the application services while deployment costs remain a matter +of pushing a service and its configuration. In either case, deployment should +be automated, so are not multiplied by the number of servers. + +![Colocated-deployment]({{}}images/redis/rra_deployment_colocated.png) + +Connections: + +* RRA: The connections between Application Service instances to RRA Service + instance are distributed and bounded to equal the number of Riak nodes + _multiplied_ by the number of Application Service instances. +* Redis: The connection between the RRA Service instance and Redis Service + instance is local. +* Riak: The connections between RRA to Riak Nodes is distributed and bounded to + equal the number of Riak nodes _squared_. + +Advantages: + +* Increases the cache hit rate as a cache write from one application server + will lead to a cache hit by all other application servers. + +Disadvantages: + +* Typically increased distance between the application service and Redis and + Riak services, so slightly increased latency compared to local. +* Redis competing for RAM with Riak will likely be problematic. Redis should + be configured to ensure `maxmemory` and `maxmemory-policy` constrain Redis + to ensure Riak is allotted sufficient RAM to serve the more important + persistent data storage and retrieval services. See http://redis.io/topics/config +* This model may seem to provide data locality, but in the case of faults in + either Redis or Riak services, the fault tolerance mechanisms of RRA and + Riak will not match exactly as communicating the necessary information to + support such a lock-step fault tolerance would lead to greater mean latencies + and Riak provides superior 99th percentile latency performance in the face + of faults. + + +### Distributed Cache Deployment + +In a distributed cache deployment, the RRA is deployed to the application server +and Redis is deployed to standalone servers, separate from Riak cluster nodes. + +![Distributed-deployment]({{}}images/redis/rra_deployment_distributed.png) + +Connections: + +* RRA: The connections between Application Service instances to RRA Service + instance are local. +* Redis: The connection between the RRA Service instance and Redis Service + instance are distributed and bounded to equal the number of Application + Servers _multipled_ by the number of Redis Servers. +* Riak: The connections between RRA to Riak Nodes is distributed and bounded to + equal the number of Riak nodes _multiplied_ by the number of Application + Servers since they are aggregated at the RRA Service instance. + +Advantages: + +* Increases the cache hit rate as a cache write from one application server + will lead to a cache hit by all other application servers. +* Keeps RRA near the application, reducing network connections. +* Moves Redis to distinct servers, allowing the cache more RAM and not + constraining the RAM of either application or persistent data services. + +Disadvantages: + +* Typically increased distance between the application service and Redis and + Riak services, so increased latency compared to local. + +### Recommendation + +The relative advantages and disadvantages of the Distributed Cache Deployment, +most notably the increased cache hit rate and reduced connection overhead, +should make it the standout choice for applications requiring the scale and +operational simplicity of Riak. For this reason, we recommend the Distributed +Cache Deployment. diff --git a/content/riak/kv/2.2.6/add-ons/redis/using-rra.md b/content/riak/kv/2.2.6/add-ons/redis/using-rra.md new file mode 100644 index 0000000000..c2a9688469 --- /dev/null +++ b/content/riak/kv/2.2.6/add-ons/redis/using-rra.md @@ -0,0 +1,242 @@ +--- +title: "Using Riak Redis Add-on" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Using Redis Addon" + identifier: "add-ons_redis_getstarted" + weight: 302 + parent: "add-ons_redis" +toc: true +commercial_offering: true +aliases: + - /riak-docs/riak/kv/2.2.6/add-ons/redis/get-started-with-rra +--- + +[addon redis develop]: ../developing-rra/ +[addon redis setup]: ../set-up-rra/ +[dev api http]: {{}}riak/kv/2.2.6/developing/api/http/ +[ee]: https://www.tiot.jp/en/about-us/contact-us/ + + +Now that you’ve [set up Riak Redis Add-on (RRA)][addon redis setup], you're ready to use RRA with any Redis client which supports `GET`, `PUT` and `DEL` operations. + +This page will walk you through using RRA. + +## Prerequisites + +We assume that the Redis client (`redis-cli`) is installed, either alongside the Redis server or on a test machine. + +You will need the list of Riak KV and Riak Redis Add-on host:port combinations. For testing, Riak KV values are obtained via the [HTTP API][dev api http]. + +## Run the Read-Through Test + +Throughout this test example, the bucket "test" and key "foo" are used to +demonstrate how to address the hieararchical namespace support in Riak KV +through the flat Redis key. The bucket type is not specified in this example, +so is effectively the default bucket type, named "default". For additional +information regarding key namespace, see [develop Riak Redis Add-on (RRA)][addon redis develop]. + +The read-through test ensures that your configuration correctly tracks values obtained from Riak KV and Riak Redis Add-on (RRA). The main actions of the test are: + +* DELETE the Riak object at the `test` bucket with the key `foo`, which checks that there are no siblings. +* PUT a Riak object with the value 'bar' at the `test` bucket with the key `foo`. +* GET the Riak object at the `test` bucket with the key `foo`. +* GET the string-representation of the object from the cache proxy service using the key `test:foo`. (The cache proxy service should parse out the first portion of the Redis colon-separated key (namespace) to identify which Riak bucket to perform the backend read from.) +* Assert that the value obtained from the previous cache proxy GET is 'bar'. + +First, create a file named`read_through_test.sh` with the following content: + +```bash +# set test environment +RIAK_HTTP_IP="127.0.0.1" +RIAK_HTTP_PORT="8098" +CACHE_PROXY_IP="127.0.0.1" +CACHE_PROXY_PORT="22122" +CACHE_PROXY_STATISTICS_PORT="22123" +RIAK_TEST_BUCKET="test" +KEY="foo" +VALUE="bar" + +# DELETE Riak object, ensure no siblings +curl -s -X DELETE "http://$RIAK_HTTP_IP:$RIAK_HTTP_PORT/buckets/$RIAK_TEST_BUCKET/keys/$KEY" + +# PUT Riak object +curl -s -X PUT -d "$VALUE" "http://$RIAK_HTTP_IP:$RIAK_HTTP_PORT/buckets/$RIAK_TEST_BUCKET/keys/$KEY" + +# GET Riak object +RIAK_VALUE=$(curl -s -X GET "http://$RIAK_HTTP_IP:$RIAK_HTTP_PORT/buckets/$RIAK_TEST_BUCKET/keys/$KEY") + +# GET Cache Proxy value +CACHE_VALUE=$(redis-cli -h "$CACHE_PROXY_IP" -p "$CACHE_PROXY_PORT" "$RIAK_TEST_BUCKET:$KEY" + +# DELETE Riak object, cleanup +curl -s -X DELETE "http://$RIAK_HTTP_IP:$RIAK_HTTP_PORT/buckets/$RIAK_TEST_BUCKET/keys/$KEY" + +# Assert +if [[ "RIAK_VALUE" == "$CACHE_VALUE" ]]; then + RESULT="Success" +else + RESULT="FAIL" +fi +echo "$RESULT - read $RIAK_VALUE from Riak and $CACHE_VALUE from Cache Proxy." +``` + +Then, once you've created the file, run it as follows: + +```bash +./read_through_test.sh 22122 8098 test +``` + +### Exceptions + +If the test does not pass, verify that both Redis and RRA are running. You can do this by running: + +```bash +ps aux |grep [r]edis +ps aux |grep [n]utcracker +``` + +The result should list `redis` and `nutcracker` respectively. + +Also, verify that Riak KV is started and listening on the protocol buffer port specified: + +```bash +sudo riak config effective |grep proto +``` + +If RRA is misconfigured, [reconfigure][redis add-on setup] it, and restart the service with the following: + +```bash +sudo service cache_proxy restart +``` + +If RRA is configured correctly and all required services are running, you may want to restart each service from front to back as follows: + +1. Stop RRA. +2. Stop Redis. +3. *Optional* Restart Riak KV (This should only be necessary if Riak KV is not responding to protocol buffer requests.) +4. Start Redis. +5. Start RRA. + +```bash +sudo service cache_proxy stop +sudo service redis stop + +# optional +sudo riak restart + +sudo service redis start +sudo service cache_proxy start +``` + +## Using Riak Redis Add-on + +Once you've successfully configured Riak Redis Add-on (RRA) and established a Riak KV and Redis client in the language of your choosing, you're ready to start using RRA. + +For objects that should not be cached, interact with Riak KV as usual: issuing GET, PUT, and DELETE commands through the Riak client. + +For objects that should be cached, read from RRA: issuing GET, SET, and DEL commands through the Redis client. + +### Monitoring + +#### RRA + +Since RRA is installed as a service, the system service monitoring daemon will automatically restart a service with the correct configuration in the event that the service’s process was killed or terminated by other means. + +The log file for RRA is stored by default in /var/log/cache_proxy.log . RRA is logrotate friendly, responding to the signal to reopen the log file following a rotate. + +For additional monitoring, RRA provides statistics on service availability. The statistics provided are generally useful in monitoring the health of the RRA service. + +For example, running the following command (using the loopback interface and the default statistics port as an example): + +```bash +telnet 127.0.0.1 22123 +``` + +Returns statistic results: + +```json +{ + "bdp_cache_proxy": { + "192.168.50.2:6379": { + "in_queue": 0, + "in_queue_bytes": 0, + "out_queue": 0, + "out_queue_bytes": 0, + "request_bytes": 216, + "requests": 9, + "response_bytes": 39, + "responses": 4, + "server_connections": 1, + "server_ejected_at": 0, + "server_eof": 0, + "server_err": 0, + "server_timedout": 0 + }, + "192.168.50.3:6379": { + "in_queue": 0, + "in_queue_bytes": 0, + "out_queue": 0, + "out_queue_bytes": 0, + "request_bytes": 0, + "requests": 0, + "response_bytes": 0, + "responses": 0, + "server_connections": 0, + "server_ejected_at": 0, + "server_eof": 0, + "server_err": 0, + "server_timedout": 0 + }, + "192.168.50.4:6379": { + "in_queue": 0, + "in_queue_bytes": 0, + "out_queue": 0, + "out_queue_bytes": 0, + "request_bytes": 90, + "requests": 5, + "response_bytes": 258, + "responses": 2, + "server_connections": 0, + "server_ejected_at": 0, + "server_eof": 0, + "server_err": 0, + "server_timedout": 0 + }, + "client_connections": 0, + "client_eof": 6, + "client_err": 0, + "forward_error": 0, + "fragments": 0, + "server_ejects": 0 + }, + "curr_connections": 4, + "service": "nutcracker", + "source": "vagrant", + "timestamp": 1438301846, + "total_connections": 10, + "uptime": 7227, + "version": "0.4.0" +} +``` + +Using the above results, you should be able to determine metrics changes that would flag a change in service health. With this information you can implement monitoring to help guarantee the overall health of the cache proxy service in RRA and the custom software within your overall solution. + +While we do not endorse a specific monitoring solution, the open interface to statistics allows you to use the monitoring solution of your choice. The following is a brief listing of compatible monitoring solutions: + +* Custom - https://github.com/gfranxman/NutcrackerMonitor +* NewRelic - http://newrelic.com/plugins/schoology/245 +* Nagios - https://github.com/schoology/twemproxy_nagios + +#### Redis + +Various Redis monitoring solutions exist in the market and, like monitoring RRA, these monitoring solutions make underlying calls to obtain Redis statistics, typically via the `info` command alone. + +As with RRA, Redis statistics available on the Redis client port allow for monitoring via solutions such as the following: + +* Custom - http://volumelabs.net/redis_monitoring/ +* NewRelic - http://newrelic.com/plugins/poison-pen-llc/28 +* Nagios - https://exchange.nagios.org/directory/Plugins/Databases/check_redis-2Epl/details diff --git a/content/riak/kv/2.2.6/configuring.md b/content/riak/kv/2.2.6/configuring.md new file mode 100644 index 0000000000..91f5724427 --- /dev/null +++ b/content/riak/kv/2.2.6/configuring.md @@ -0,0 +1,82 @@ +--- +title: "Configuring Riak KV" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Configuring" + identifier: "configuring" + weight: 200 + pre: cog +toc: true +--- + +[config basic]: ../configuring/basic +[config backend]: ../configuring/backend +[config manage]: ../configuring/managing +[config reference]: ../configuring/reference +[config strong consistency]: ../configuring/strong-consistency +[config load balance]: ../configuring/load-balancing-proxy +[config mapreduce]: ../configuring/mapreduce +[config search]: ../configuring/search +[config v3 mdc]: ../configuring/v3-multi-datacenter +[config v2 mdc]: ../configuring/v2-multi-datacenter + +## In This Section + +#### [Basic Configuration][config basic] + +A guide covering commonly adjusted parameters when setting up a new cluster. + +[Learn More >>][config basic] + +#### [Backend Configuration][config backend] + +Information on backend-specific configuration parameters. + +[Learn More >>][config backend] + +#### [Managing Configuration][config manage] + +A small guide to retrieving, checking, and debugging your cluster configuration. + +[Learn More >>][config manage] + +#### [Configuration Reference][config reference] + +A detailed list of all possible configuration parameters. + +[Learn More >>][config reference] + +#### [Implementing Strong Consistency][config strong consistency] + +An article providing information on configuring and monitoring a Riak KV +cluster's optional strong consistency subsystem. + +[Learn More >>][config strong consistency] + +#### [Load Balancing & Proxy][config load balance] + +A brief guide on commonly used load-balancing and proxy solutions. + +[Learn More >>][config load balance] + +#### [MapReduce Settings][config mapreduce] + +Tutorial on configuring and tuning MapReduce for a cluster. + +[Learn More >>][config mapreduce] + +#### [Search Settings][config search] + +Information on configuring and using Riak KV Search from an operational perspective. + +[Learn More >>][config search] + +#### [V3 Multi-Datacenter][config v3 mdc] + +A guide on configuring Riak's V3 Multi-Datacenter Replication + +[Learn More >>][config v3 mdc] + diff --git a/content/riak/kv/2.2.6/configuring/backend.md b/content/riak/kv/2.2.6/configuring/backend.md new file mode 100644 index 0000000000..da82fab674 --- /dev/null +++ b/content/riak/kv/2.2.6/configuring/backend.md @@ -0,0 +1,559 @@ +--- +title: "Backend Configuration" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Backend Configuration" + identifier: "configuring_backend" + weight: 110 + parent: "configuring" +toc: true +--- + +[plan backend leveldb]: {{}}riak/kv/2.2.6/setup/planning/backend/leveldb +[plan backend bitcask]: {{}}riak/kv/2.2.6/setup/planning/backend/bitcask +[plan backend memory]: {{}}riak/kv/2.2.6/setup/planning/backend/memory +[plan backend multi]: {{}}riak/kv/2.2.6/setup/planning/backend/multi + +## LevelDB + +Configurable parameters for Riak's [LevelDB][plan backend leveldb] storage backend. + +> **Note on upgrading to 2.0** +> +> If you are upgrading to Riak 2.0+ from a 1.x version, using LevelDB, and +wish to use your old configuration files, i.e. `app.config` and +`vm.args`, please note that you must set the `total_leveldb_mem_percent` +setting in the `eleveldb` section of `app.config`. We recommend setting +it to `70`. If you do not set this parameter, it will default to 15, +which can lead to problems in some clusters. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ConfigDescriptionDefault
leveldb.block_cache_thresholdThis setting defines the limit past which block cache memory can no +longer be released in favor of the page cache. This setting has no +impact in favor of file cache. The value is set on a per-vnode basis. +32MB
leveldb.compaction.trigger.tombstone_countControls when a background compaction initiates solely due to the +number of delete tombstones within an individual .sst table +file. A value of off disables the feature.1000
leveldb.compressionEnabling this setting (on), which is the default, +saves disk space. Disabling it may reduce read latency but increase +overall disk activity. This option can be changed at any time, but it +will not impact data on disk until the next time a file requires +compaction.on
leveldb.compression.algorithmThis setting is used to select which compression algorithm + is selected when leveldb.compression is on. + In new riak.conf files, this is explicitly set to + lz4; however when this setting is not provided, + snappy will be used for backward-compatibility. +

+ When you determine that you will no longer need backward-compatibility, + setting this to lz4 will cause future compactions + to use the LZ4 algorithm for compression.
lz4 in new riak.conf files

+ snappy when not provided +
leveldb.data_rootThe directory in which LevelDB will store its data../data/leveldb
leveldb.fadvise_willneedOption to override LevelDB's use of fadvise(DONTNEED) +with fadvise(WILLNEED) instead. WILLNEED can +reduce disk activity on systems where physical memory exceeds the +database size.false
leveldb.maximum_memoryThis parameter defines the server memory (in bytes) to assign to +LevelDB. Also see leveldb.maximum_memory.percent to set +LevelDB memory as a percentage of system total.80
leveldb.maximum_memory.percentThis parameter defines the percentage of total server memory to +assign to LevelDB. LevelDB will dynamically adjust its internal cache +sizes to stay within this size. The memory size can alternately be +assigned as a byte count via leveldb.maximum_memory +instead.70
leveldb.threadsThe number of worker threads performing LevelDB operations.71
leveldb.verify_checksumsEnables or disables the verification of the data fetched from +LevelDB against internal checksums.on
leveldb.verify_compactionEnables or disables the verification of LevelDB data during +compaction.on
leveldb.block.size_stepsDefines the number of incremental adjustments to attempt between the +block.size value and the maximum block.size +for an .sst table file. A value of zero disables the +underlying dynamic block_size feature.16
leveldb.block.restart_intervalDefines the key count threshold for a new key entry in the key +index for a block. Most deployments should leave this parameter alone. +16
leveldb.block.sizeDefines the size threshold for a block/chunk of data within one +.sst table file. Each new block gets an index entry in the +.sst table file's master index.4KB
leveldb.bloomfilterEach database .sst table file can include an optional +"bloom filter" that is highly effective in shortcutting data queries +that are destined to not find the requested key. The Bloom filter +typically increases the size of an .sst table file by about +2%.on
leveldb.write_buffer_size_minEach vnode first stores new key/value data in a memory-based write +buffer. This write buffer is in parallel to the recovery log mentioned +in the sync parameter. Riak creates each vnode with a +randomly sized write buffer for performance reasons. The random size is +somewhere between write_buffer_size_min and +write_buffer_size_max.30MB
leveldb.write_buffer_size_maxSee leveldb.write_buffer_size_min directly above.60MB
leveldb.limited_developer_memThis is a Riak-specific option that is used when a developer is +testing a high number of vnodes and/or several VMs on a machine with +limited physical memory. Do not use this option if making +performance measurements. This option overwrites values given to +write_buffer_size_min and +write_buffer_size_max.off
leveldb.sync_on_writeWhether LevelDB will flush after every write.

+Note: If you are familiar with fsync, this is analogous +to calling fsync after every write.
off
leveldb.tieredThe level number at which LevelDB data switches from the faster to +the slower array. The default of off disables the +feature.off
leveldb.tiered.path.fastThe path prefix for .sst files below the level set by +leveldb.tiered.
leveldb.tiered.path.slowThe path prefix for .sst files below the level set by +leveldb.tiered.
+ +## Bitcask + +Configurable parameters for Riak's [Bitcask][plan backend bitcask] storage backend. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ConfigDescriptionDefault
bitcask.data_rootThe directory under which Bitcask will store its data../data/bitcask
bitcask.io_modeConfigure how Bitcask writes data to disk. If set to +erlang, writes are made via Erlang's built-in file API; if +set to nif, writes are made via direct calls to the POSIX C +API. The nif mode provides higher throughput for certain +workloads, but has the potential to negatively impact the Erlang VM, +leading to higher worst-case latencies and possible throughput collapse +erlang
bitcask.expiryBy default, Bitcask keeps all of your data around. If your data has +limited time value, or if you need to purge data for space reasons, you +can set the expiry option. For example, if you need to +purge data automatically after 1 day, set the value to 1d. +off disables automatic expirationoff
bitcask.expiry.grace_timeBy default, Bitcask will trigger a merge whenever a data file +contains an expired key. This may result in excessive merging under some +usage patterns. To prevent this you can set the +bitcask.expiry.grace_time option. Bitcask will defer +triggering a merge solely for key expiry by the configured number of +seconds. Setting this to 1h effectively limits each cask to +merging for expiry once per hour.0
bitcask.hintfile_checksumsWhether to allow the CRC to be present at the end of hintfiles. +Setting this to allow_missing runs Bitcask in a +backwards-compatible mode in which old hint files will still be accepted +without CRC signatures.strict
bitcask.fold.max_putsSee the description for the bitcask.fold.max_age +config directly below.0
bitcask.fold.max_ageFold keys thresholds will reuse the keydir if another fold was +started less than fold.max_age ago and there were fewer +than fold.max_puts updates. Otherwise, it will wait until +all current fold keys complete and then start. Set either option to +unlimited to disable.unlimited
bitcask.merge.thresholds.fragmentationDescribes which ratio of dead keys to total keys in a file will +cause it to be included in the merge. The value of this setting is a +percentage from 0 to 100. For example, if a data file contains 4 dead +keys and 6 live keys, it will be included in the merge at the default +ratio (which is 40). Increasing the value will cause fewer files to be +merged, decreasing the value will cause more files to be merged.40
bitcask.merge.thresholds.dead_bytesDescribes the minimum amount of data occupied by dead keys in a file +to cause it to be included in the merge. Increasing the value will cause +fewer files to be merged, whereas decreasing the value will cause more +files to be merged.128MB
bitcask.merge.thresholds.small_fileDescribes the minimum size a file must have to be excluded from the +merge. Files smaller than the threshold will be included. Increasing +the value will cause more files to be merged, whereas decreasing the +value will cause fewer files to be merged.10MB
bitcask.merge.triggers.dead_bytesDescribes how much data stored for dead keys in a single file will +trigger merging. If a file meets or exceeds the trigger value for dead +bytes, merge will be triggered. Increasing the value will cause merging +to occur less often, whereas decreasing the value will cause merging to +happen more often. When either of these constraints are met by any file +in the directory, Bitcask will attempt to merge files.512MB
bitcask.merge.triggers.fragmentationDescribes which ratio of dead keys to total keys in a file will +trigger merging. The value of this setting is a percentage from 0 to +100. For example, if a data file contains 6 dead keys and 4 live keys, +then merge will be triggered at the default setting. Increasing this +value will cause merging to occur less often, whereas decreasing the +value will cause merging to happen more often.60
bitcask.merge.window.endSee the description of the bitcask.merge.policy config +below.23
bitcask.merge.window.startSee the description of the bitcask.merge.policy config +below.0
bitcask.merge.policyLets you specify when during the day merge operations are allowed to +be triggered. Valid options are: always, meaning no +restrictions; never, meaning that merging will never be +attempted; and window, specifying the hours during which +merging is permitted, where bitcask.merge.window.start and +bitcask.merge.window.end are integers between 0 and 23. If +merging has a significant impact on performance of your cluster, or your +cluster has quiet periods in which little storage activity occurs, you +may want to change this setting from the default.always
bitcask.merge_check_intervalBitcask periodically runs checks to determine whether merges are +necessary. This parameter determines how often those checks take place. +Expressed as a time unit, e.g. `10s` for 10 seconds, `5m` for 5 minutes, +etc.3m
bitcask.merge_check_jitterIn order to prevent merge operations from taking place on different +nodes at the same time, Riak can apply random variance to merge times, +expressed as a percentage of bitcask.merge_check_interval. +30%
bitcask.max_merge_sizeMaximum amount of data to merge in one go in the Bitcask backend. +100GB
bitcask.max_file_sizeDescribes the maximum permitted size for any single data file in the +Bitcask directory. If a write causes the current file to exceed this +size threshold then that file is closed, and a new file is opened for +writes.2GB
bitcask.sync.intervalSee the description of the bitcask.sync.strategy +directly below.
bitcask.sync.strategyChanges the durability of writes by specifying when to synchronize +data to disk. The default setting protects against data loss in the +event of application failure (process death) but leaves open a small +window in which data could be lost in the event of complete system +failure (e.g. hardware, OS, or power). The default mode, +none, writes data into operating system buffers which will +be written to the disks when those buffers are flushed by the operating +system. If the system fails, e.g. due power loss or crash, that data is +lost before those buffers are flushed to stable storage. This is +prevented by the setting o_sync, which forces the operating +system to flush to stable storage at every write. The effect of flushing +each write is better durability, however write throughput will suffer as +each write will have to wait for the write to complete. Available sync +strategies: none, which will let the operating system +manage syncing writes; o_sync, which will uses the +O_SYNC flag to force syncs on every write; and +interval, by which will force Bitcask to sync every +bitcask.sync.interval seconds.none
bitcask.open_timeoutSpecifies the maximum time Bitcask will block on startup while +attempting to create or open the data directory. You generally need not +change this value. If for some reason the timeout is exceeded on open +you'll see a log message of the form Failed to start bitcask +backend: .... . Only then should you consider a longer timeout. +4s
+ +## Memory Backend + +Configurable parameters for Riak's [Memory][plan backend memory] backend. + + + + + + + + + + + + + + + + + + + + + + + + +
ConfigDescriptionDefault
memory_backend.ttlEach value written will be written with this "time to live." Once +that object's time is up, it will be deleted on the next read of its +key. Minimum: 1s.
memory_backend.max_memory_per_vnodeThe maximum amount of memory consumed per vnode by the memory +storage backend. Minimum: 1MB.
+ +## Multi Backend + +Configurable parameters for Riak's [Multi][plan backend multi] backend, which enables you to utilize multiple data backends in a single Riak cluster. + +If you are using multiple backends, you can configure the backends +individually by prepending the setting with `multi_backend.$name`, where +`$name` is the name of the backend. `$name` can be any valid +configuration word, like `customer_data`, `my_data`, `foo_bar_backend`, +etc. + +Below is the general form for setting multi-backend parameters: + +```riakconf +multi_backend.$name.(existing_setting) = +# or +multi_backend.$name.$backend_type.(backend_specific_setting) = +``` + +Below is a listing of the available parameters: + + + + + + + + + + + + + + + + + + + + + + + + +
ConfigDescriptionDefault
multi_backend.$name.storage_backendThis parameter specifies the Erlang module defining the storage +mechanism that will be used on this node.bitcask
multi_backend.defaultThe default name of a backend when one is not specified.
+ +To give an example, if you have a LevelDB backend named +`customer_backend` and wish to set the `data_root` parameter to +`$(platform_data_dir)/leveldb_backends/customer_backend/`, you would +do so as follows: + +```riakconf +multi_backend.customer_backend.storage_backend = leveldb +multi_backend.customer_backend.leveldb.data_root = $(platform_data_dir)/leveldb_backends/customer_backend +multi_backend.customer_backend.leveldb.maximum_memory.percent = 50 +``` diff --git a/content/riak/kv/2.2.6/configuring/basic.md b/content/riak/kv/2.2.6/configuring/basic.md new file mode 100644 index 0000000000..aaa4bd9738 --- /dev/null +++ b/content/riak/kv/2.2.6/configuring/basic.md @@ -0,0 +1,235 @@ +--- +title: "Basic Riak KV Configuration" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Basic Configuration" + identifier: "configuring_basic" + weight: 100 + parent: "configuring" +toc: true +aliases: + - /riak-docs/riak/2.2.6/ops/building/configuration/ + - /riak-docs/riak/kv/2.2.6/ops/building/configuration/ +--- + +[config reference]: {{}}riak/kv/2.2.6/configuring/reference +[use running cluster]: {{}}riak/kv/2.2.6/using/running-a-cluster +[use admin riak-admin#member-status]: {{}}riak/kv/2.2.6/using/admin/riak-admin/#member-status +[perf erlang]: {{}}riak/kv/2.2.6/using/performance/erlang +[plan start]: {{}}riak/kv/2.2.6/setup/planning/start +[plan best practices]: {{}}riak/kv/2.2.6/setup/planning/best-practices +[cluster ops backup]: {{}}riak/kv/2.2.6/using/cluster-operations/backing-up +[cluster ops add remove node]: {{}}riak/kv/2.2.6/using/cluster-operations/adding-removing-nodes +[plan backend]: {{}}riak/kv/2.2.6/setup/planning/backend +[plan backend multi]: {{}}riak/kv/2.2.6/setup/planning/backend/multi +[plan backend bitcask]: {{}}riak/kv/2.2.6/setup/planning/backend/bitcask +[usage bucket types]: {{}}riak/kv/2.2.6/developing/usage/bucket-types +[apps replication properties]: {{}}riak/kv/2.2.6/developing/app-guide/replication-properties +[concept buckets]: {{}}riak/kv/2.2.6/learn/concepts/buckets +[concept eventual consistency]: {{}}riak/kv/2.2.6/learn/concepts/eventual-consistency +[perf benchmark]: {{}}riak/kv/2.2.6/using/performance/benchmarking +[perf open files]: {{}}riak/kv/2.2.6/using/performance/open-files-limit +[perf index]: {{}}riak/kv/2.2.6/using/performance +[perf aws]: {{}}riak/kv/2.2.6/using/performance/amazon-web-services +[Cluster Capacity Planning]: {{}}riak/kv/2.2.6/setup/planning/cluster-capacity/#ring-size-number-of-partitions + +This document covers the parameters that are commonly adjusted when +setting up a new cluster. We recommend that you also review the detailed +[Configuration Files][config reference] document before moving a cluster into +production. + +All configuration values discussed here are managed via the +configuration file on each node, and a node must be restarted for any +changes to take effect. + +> **Note** +> +> If you are upgrading to Riak KV version 2.0 or later from an pre-2.0 +release, you can use either your old `app.config` configuration file or +the newer `riak.conf` if you wish. +> +> If you have installed Riak KV 2.0 directly, you should use only +`riak.conf`. +> +> More on configuring Riak KV can be found in the [configuration files][config reference] +doc. + +We advise that you make as many of the changes below as practical +_before_ joining the nodes together into a cluster. Once your +configuration has been set on each node, follow the steps in [Basic Cluster Setup][use running cluster] to complete the clustering process. + +Use [`riak-admin member-status`][use admin riak-admin#member-status] +to determine whether any given node is a member of a cluster. + +## Erlang VM Tunings + +Prior to building and starting a cluster, there are some +Erlang-VM-related changes that you should make to your configuration +files. If you are using the older, `vm.args`-based Erlang VM tunings, +you should set the following: + +```vmargs ++sfwi 500 ++scl false +``` + +If you are using the newer, `riak.conf`-based configuration system, we +recommend the following settings: + +```riakconf +erlang.schedulers.force_wakeup_interval = 500 +erlang.schedulers.compaction_of_load = false +``` + +More information can be found in [Erlang VM Tuning][perf erlang]. + +## Ring Size + +The ring size, in Riak parlance, is the number of data partitions that +comprise the cluster. This quantity impacts the scalability and +performance of a cluster and, importantly, **it should be established +before the cluster starts receiving data**. + +If the ring size is too large for the number of servers, disk I/O will +be negatively impacted by the excessive number of concurrent databases +running on each server; if the ring size is too small, the servers' other +resources (primarily CPU and RAM) will go underutilized. + +See [Cluster Capacity Planning] for more details on choosing a ring size. + +The steps involved in changing the ring size depend on whether the +servers (nodes) in the cluster have already been joined together. + +### Cluster joined, but no data needs to be preserved + +1. Change the ring creation size parameter by uncommenting it and then +setting it to the desired value, for example 64: + + ```riakconf + ring_size = 64 + ``` + + ```appconfig + %% In the riak_core section: + {ring_creation_size, 64} + ``` + +2. Stop all nodes +3. Remove the ring data file on each node (see [Backing up Riak][cluster ops backup] for the location of this file) +4. Start all nodes +5. Re-add each node to the cluster (see [Adding and Removing Nodes][cluster ops add remove node]) or finish reviewing this document and proceed to [Basic Cluster Setup][use running cluster] + +### New servers, have not yet joined a cluster + +1. Change the ring creation size parameter by uncommenting it and then +setting it to the desired value, for example 64: + + ```riakconf + ring_size = 64 + ``` + + ```appconfig + %% In the riak_core section: + {ring_creation_size, 64} + ``` + +2. Stop all nodes +3. Remove the ring data file on each node (see [Backing up Riak][cluster ops backup] for +the location of this file) +4. Finish reviewing this document and proceed to [Basic Cluster Setup][use running cluster] + +### Verifying ring size + +You can use the `riak-admin` command can verify the ring size: + +```bash +riak-admin status | grep ring +``` + +Console output: + +``` +ring_members : ['riak@10.160.13.252'] +ring_num_partitions : 8 +ring_ownership : <<"[{'riak@10.160.13.252',8}]">> +ring_creation_size : 8 +``` + +If `ring_num_partitions` and `ring_creation_size` do not agree, that +means that the `ring_creation_size` value was changed too late and that +the proper steps were not taken to start over with a new ring. + +**Note**: Riak will not allow two nodes with different ring sizes to be +joined into a cluster. + +## Backend + +Another critical decision to be made is the backend to use. The choice +of backend strongly influences the performance characteristics and +feature set for a Riak environment. + +See [Choosing a Backend][plan backend] for a list of supported backends. Each +referenced document includes the necessary configuration bits. + +As with ring size, changing the backend will result in all data being +effectively lost, so spend the necessary time up front to evaluate and +benchmark backends. + +If still in doubt, consider using the [Multi][plan backend multi] backend for future +flexibility. + +If you do change backends from the default ([Bitcask][plan backend bitcask]), make sure you change it across all nodes. It is possible but generally unwise to use different backends on different nodes, as this would limit the +effectiveness of backend-specific features. + +## Default Bucket Properties + +Bucket properties are also very important factors in Riak's performance +and general behavior. The properties for any individual bucket can be +configured dynamically [using bucket types][usage bucket types], but default values for those properties can be defined in your [configuration files][config reference]. + +Below is an example of setting `last_write_wins` to `true` and `r` to 3. + +```riakconf +buckets.default.last_write_wins = true +buckets.default.r = 3 +``` + +```appconfig +{default_bucket_props, [ + {last_write_wins,true}, + {r,3}, + ... + ]} +``` + +For more on bucket properties, we recommend reviewing our docs on +[buckets][concept buckets], [bucket types][usage bucket types], [replication properties][apps replication properties], and [eventual consistency][concept eventual consistency], as well as Basho's five-part blog series, "Understanding Riak's Configurable Behaviors." + +* [Part 1](http://basho.com/understanding-riaks-configurable-behaviors-part-1/) +* [Part 2](http://basho.com/riaks-config-behaviors-part-2/) +* [Part 3](http://basho.com/riaks-config-behaviors-part-3/) +* [Part 4](http://basho.com/riaks-config-behaviors-part-4/) +* [Epilogue](http://basho.com/riaks-config-behaviors-epilogue/) + +If the default bucket properties are modified in your configuration +files and the node is restarted, any existing buckets will **not** be +directly impacted, although the mechanism described in [HTTP Reset Bucket Properties]({{}}riak/kv/2.2.6/developing/api/http/reset-bucket-props) can be used to force them to pick up the new +defaults. + +## System tuning + +Please review the following documents before conducting any +[benchmarking][perf benchmark] and/or rolling out a live production +cluster. + +* [Open Files Limit][perf open files] +* [System Performance Tuning][perf index] +* [AWS Performance Tuning][perf aws] +* [Configuration Files][config reference] + +## Joining the nodes together + +Please see [Running A Cluster][use running cluster] for the cluster creation process. diff --git a/content/riak/kv/2.2.6/configuring/global-object-expiration.md b/content/riak/kv/2.2.6/configuring/global-object-expiration.md new file mode 100644 index 0000000000..c4a36281d7 --- /dev/null +++ b/content/riak/kv/2.2.6/configuring/global-object-expiration.md @@ -0,0 +1,86 @@ +--- +title: "Configure Global Object Expiration" +description: "Enabling and configuring global object expiration for Riak KV." +menu: + riak_kv-2.2.6: + name: "Global Object Expiration" + identifier: "config_expiry" + weight: 180 + parent: "configuring" +project: "riak_kv" +project_version: "2.2.6" +toc: true +canonical_link: "riak/kv/latest/configuring/global-object-expiration" +--- + +[ttl]: https://en.wikipedia.org/wiki/Time_to_live + +By default, LevelDB keeps all of your data. But Riak KV allows you to configure global object expiration (`expiry`) or [time to live (TTL)][ttl] for your data. + +Expiration is disabled by default, but enabling it lets you expire older objects to reclaim the space used or purge data with a limited time value. + +## Enabling Expiry + +To enable global object expiry, add the `leveldb.expiration` setting to your riak.conf file: + +```riak.conf +leveldb.expiration = on +``` + +{{% note %}} +Turning on global object expiration will not retroactively expire previous data. Only data created while expiration is on will be scheduled for expiration. +{{% /note %}} + +## Setting Retention Time + +The `retention_time` setting is used to specify the time until objects expire. +Durations are set using a combination of an integer and a shortcut for the supported units: + +- Milliseconds - `ms` +- Seconds - `s` +- Minutes - `m` +- Hours - `h` +- Days - `d` +- Weeks - `w` +- Fortnight - `f` + +The following example configures objects to expire after 5 hours: + +```riak.conf +leveldb.expiration = on +leveldb.expiration.retention_time = 5h +``` + +You can also combine durations. For example, let's say you wanted objects to expire after 8 days and 9 hours: + +```riak.conf +leveldb.expiration = on +leveldb.expiration.retention_time = 8d9h +``` + +## Expiry Modes + +Global expiration supports two modes: + +- `whole_file` - the whole sorted string table (`.sst`) file is deleted when all of its objects are expired. +- `normal` - individual objects are removed as part of the usual compaction process. + +We recommend using `whole_file` with time series data that has a similar lifespan, as it will be much more efficient. + +The following example configure objects to expire after 1 day: + +```riak.conf +leveldb.expiration = on +leveldb.expiration.retention_time = 1d +leveldb.expiration.mode = whole_file +``` + +## Disable Expiry + +To disable global object expiration, set `leveldb.expiration` to `off` in your riak.conf file. If expiration is disabled, the other 2 settings are ignored. For example: + +```riak.conf +leveldb.expiration = off +leveldb.expiration.retention_time = 1d +leveldb.expiration.mode = whole_file +``` diff --git a/content/riak/kv/2.2.6/configuring/load-balancing-proxy.md b/content/riak/kv/2.2.6/configuring/load-balancing-proxy.md new file mode 100644 index 0000000000..f6e78ce063 --- /dev/null +++ b/content/riak/kv/2.2.6/configuring/load-balancing-proxy.md @@ -0,0 +1,271 @@ +--- +title: "Load Balancing and Proxy Configuration" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Load Balancing & Proxy" + identifier: "configuring_load_balance" + weight: 150 + parent: "configuring" +toc: true +aliases: + - /riak-docs/riak/2.2.6/ops/advanced/configs/load-balanacing-proxy/ + - /riak-docs/riak/kv/2.2.6/ops/advanced/configs/load-balanacing-proxy/ +--- + +[perf open files]: {{}}riak/kv/2.2.6/using/performance/open-files-limit + +The recommended best practice for operating Riak in production is to +place Riak behind a load-balancing or proxy solution, either hardware- +or software- based, while never directly exposing Riak to public network +interfaces. + +Riak users have reported success in using Riak with a variety of load- +balancing and proxy solutions. Common solutions include proprietary +hardware-based load balancers, cloud-based load balancing options, such +as Amazon's Elastic Load Balancer, and open-source software based +projects like HAProxy and Nginx. + +This guide briefly explores the commonly used open-source software-based +solutions HAProxy and Nginx, and provides some configuration and +operational tips gathered from community users and operations oriented +engineers at Basho. + +While it is by no means an exhaustive overview of the topic, this guide +should provide a starting point for choosing and implementing your own +solution. + +## HAProxy + +[HAProxy](http://haproxy.1wt.eu/) is a fast and reliable open-source +solution for load balancing and proxying of HTTP- and TCP-based +application traffic. + +Users have reported success in using HAProxy in combination with Riak in +a number of configurations and scenarios. Much of the information and +example configuration for this section is drawn from experiences of +users in the Riak community in addition to suggestions from Basho +engineering. + +### Example Configuration + +The following is an example starting-point configuration for HAProxy to +act as a load balancer. The example cluster has 4 nodes and will be +accessed by Riak clients using both the Protocol Buffers and HTTP +interfaces. + +> **Note on open files limits** +> +> The operating system's open files limits need to be greater than 256000 +for the example configuration that follows. Consult the [Open Files Limit][perf open files] documentation for details on configuring the value for different operating systems. + +```config +global + log 127.0.0.1 local0 + log 127.0.0.1 local1 notice + maxconn 256000 + chroot /var/lib/haproxy + user haproxy + group haproxy + spread-checks 5 + daemon + quiet + +defaults + log global + option dontlognull + option redispatch + option allbackups + maxconn 256000 + timeout connect 5000 + +backend riak_rest_backend + mode http + balance roundrobin + option httpchk GET /ping + option httplog + server riak1 riak1.:8098 weight 1 maxconn 1024 check + server riak2 riak2.:8098 weight 1 maxconn 1024 check + server riak3 riak3.:8098 weight 1 maxconn 1024 check + server riak4 riak4.:8098 weight 1 maxconn 1024 check + +frontend riak_rest + bind 127.0.0.1:8098 + # Example bind for SSL termination + # bind 127.0.0.1:8443 ssl crt /opt/local/haproxy/etc/data.pem + mode http + option contstats + default_backend riak_rest_backend + + +backend riak_protocol_buffer_backend + balance leastconn + mode tcp + option tcpka + option srvtcpka + server riak1 riak1.:8087 weight 1 maxconn 1024 check + server riak2 riak2.:8087 weight 1 maxconn 1024 check + server riak3 riak3.:8087 weight 1 maxconn 1024 check + server riak4 riak4.:8087 weight 1 maxconn 1024 check + + +frontend riak_protocol_buffer + bind 127.0.0.1:8087 + mode tcp + option tcplog + option contstats + mode tcp + option tcpka + option srvtcpka + default_backend riak_protocol_buffer_backend +``` + +A specific configuration detail worth noting from the example is the +commented option for SSL termination. HAProxy supports SSL directly as +of version 1.5. Provided that your HAProxy instance was built with +OpenSSL support, you can enable it by uncommenting the example line and +modifying it to suit your environment. More information is available in +the [HAProxy +documentation](http://cbonte.github.io/haproxy-dconv/configuration-1.5.html#5-ssl). + +Also note that the above example is considered a starting point and is a +work in progress based upon [this +example](https://gist.github.com/1507077). You should carefully examine +the configuration and change it according to your specific environment. + +### Maintaining Nodes Behind HAProxy + +When using HAProxy with Riak, you can instruct HAProxy to ping each node +in the cluster and automatically remove nodes that do not respond. + +You can also specify a round-robin configuration in HAProxy and have +your application handle connection failures by retrying after a timeout, +thereby reaching a functioning node upon retrying the connection +attempt. + +HAPproxy also has a standby system you can use to remove a node from +rotation while allowing existing requests to finish. You can remove +nodes from HAProxy directly from the command line by interacting with +the HAProxy stats socket with a utility such as +[socat](http://www.dest-unreach.org/socat/): + +```bash +echo "disable server /" | socat stdio /etc/haproxy/haproxysock +``` + +At this point, you can perform maintenance on the node, down the node, +and so on. When you've finished working with the node and it is again +available for requests, you can re-enable it: + +```bash +echo "enable server /" | socat stdio /etc/haproxy/haproxysock +``` + +Consult the following HAProxy documentation resources for more +information on configuring HAProxy in your environment: + +* [HAProxy Documentation](http://developers.google.com/s/results/?q=haproxy) +* [HAProxy Architecture](http://haproxy.1wt.eu/download/1.2/doc/architecture.txt) + +## Nginx + +Some users have reported success in using the [Nginx](http://nginx.org/) +HTTP server to proxy requests for Riak clusters. An example that +provides access to a Riak cluster *through GET requests only* is +provided here for reference. + +### Example Configuration + +The following is an example starting point configuration for Nginx to +act as a front-end proxy to a 5-node Riak cluster. + +This example forwards all GET requests to Riak nodes while rejecting all +other HTTP operations. + +{{% note title="Nginx version notes" %}} +This example configuration was verified on **Nginx version 1.2.3**. Please be +aware that earlier versions of Nginx did not support any HTTP 1.1 semantics +for upstream communication to backends. You should carefully examine this +configuration and make changes appropriate to your specific environment before +attempting to use it +{{% /note %}} + +Here is an example `nginx.conf` file: + +```config +upstream riak_hosts { + # server 10.0.1.10:8098; + # server 10.0.1.11:8098; + # server 10.0.1.12:8098; + # server 10.0.1.13:8098; + # server 10.0.1.14:8098; +} + +server { + listen 80; + server_name _; + access_log /var/log/nginx/riak.access.log; + + # your standard Nginx config for your site here... + location / { + root /var/www/nginx-default; + } + + # Expose the /riak endpoint and allow queries for keys only + location /riak/ { + proxy_set_header Host $host; + proxy_redirect off; + + client_max_body_size 10m; + client_body_buffer_size 128k; + + proxy_connect_timeout 90; + proxy_send_timeout 90; + proxy_read_timeout 90; + + proxy_buffer_size 64k; # If set to a smaller value, + # nginx can complain with an + # "too large headers" error + proxy_buffers 4 64k; + proxy_busy_buffers_size 64k; + proxy_temp_file_write_size 64k; + + if ($request_method != GET) { + return 405; + } + + # Disallow any link with the MapReduce query format "bucket,tag,_" + if ($uri ~ "/riak/[^/]*/[^/]*/[^,]+,[^,]+," ) { + return 405; + } + + if ($request_method = GET) { + proxy_pass http://riak_hosts; + } + } +} +``` + +{{% note title="Note on access controls" %}} +Even when filtering and limiting requests to GETs only as done in the example, +you should strongly consider additional access controls beyond what Nginx can +provide directly, such as specific firewall rules to limit inbound connections +to trusted sources. +{{% /note %}} + +### Querying Secondary Indexes Over HTTP + +When accessing Riak over HTTP and issuing Secondary Index queries, you +can encounter an issue due to the default Nginx handling of HTTP header +names containing underscore (`_`) characters. + +By default, Nginx will issue errors for such queries, but you can +instruct Nginx to handle such header names when doing Secondary Index +queries over HTTP by adding the following directive to the appropriate +`server` section of `nginx.conf`: + +``` +underscores_in_headers on; +``` diff --git a/content/riak/kv/2.2.6/configuring/managing.md b/content/riak/kv/2.2.6/configuring/managing.md new file mode 100644 index 0000000000..1a17178be0 --- /dev/null +++ b/content/riak/kv/2.2.6/configuring/managing.md @@ -0,0 +1,116 @@ +--- +title: "Managing Your Configuration" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Managing Configuration" + identifier: "configuring_managing" + weight: 130 + parent: "configuring" +toc: true +--- + +[use admin riak cli]: {{}}riak/kv/2.2.6/using/admin/riak-cli +[use admin riak cli#chkconfig]: {{}}riak/kv/2.2.6/using/admin/riak-cli/#chkconfig +[config reference#search]: {{}}riak/kv/2.2.6/configuring/reference/#search + +## Retrieving a Configuration Listing + +At any time, you can get a snapshot of currently applied configurations +through the command line. For a listing of *all* of the configs +currently applied in the node: + +```bash +riak config effective +``` + +This will output a long list of the following form: + +``` +anti_entropy = active +anti_entropy.bloomfilter = on +anti_entropy.concurrency_limit = 2 +# and so on +``` + +For detailed information about a particular configuration variable, use +the `config describe ` command. This command will output a +description of what the parameter configures, which datatype you should +use to set the parameter (integer, string, enum, etc.), the default +value of the parameter, the currently set value in the node, and the +name of the parameter in `app.config` in older versions of Riak (if +applicable). + +For in-depth information about the `ring_size` variable, for example: + +```bash +riak config describe ring_size +``` + +This will output the following: + +``` +Documentation for ring_size +Number of partitions in the cluster (only valid when first +creating the cluster). Must be a power of 2, minimum 8 and maximum +1024. + + Datatype : [integer] + Default Value: 64 + Set Value : undefined + app.config : riak_core.ring_creation_size +``` + +## Checking Your Configuration + +The [`riak`][use admin riak cli] command line tool has a +[`chkconfig`][use admin riak cli#chkconfig] command that enables you to +determine whether the syntax in your configuration files is correct. + +```bash +riak chkconfig +``` + +If your configuration files are syntactically sound, you should see the +output `config is OK` followed by a listing of files that were checked. +You can safely ignore this listing. If, however, something is +syntactically awry, you'll see an error output that provides details +about what is wrong. To give an example, the `search.solr.jmx_port` +setting (in the [Search][config reference#search] section below) +must be set as an integer. Imagine that we set it to something else: + +```riakconf +search.solr.jmx_port = banana +``` + +If we run `riak chkconfig` now, we'll get an error: + +``` +[error] Error generating configuration in phase transform_datatypes +[error] Error transforming datatype for: search.solr.jmx_port +[error] "banana" can't be converted to an integer +``` + +The error message will specify which configurable parameters are +syntactically unsound and attempt to provide an explanation why. + +Please note that the `chkconfig` command only checks for syntax. It will +_not_ be able to discern if your configuration is otherwise unsound, +e.g. if your configuration will cause problems on your operating system +or doesn't activate subsystems that you would like to use. + +## Debugging Your Configuration + +If there is a problem with your configuration but you're having trouble +identifying the problem, there is a command that you can use to debug +your configuration: + +```bash +riak config generate -l debug +``` + +If there are issues with your configuration, you will see detailed +output that might provide a better sense of what has gone wrong in the +config generation process. diff --git a/content/riak/kv/2.2.6/configuring/mapreduce.md b/content/riak/kv/2.2.6/configuring/mapreduce.md new file mode 100644 index 0000000000..962fd892b3 --- /dev/null +++ b/content/riak/kv/2.2.6/configuring/mapreduce.md @@ -0,0 +1,196 @@ +--- +title: "MapReduce Settings" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "MapReduce Settings" + identifier: "configuring_mapreduce" + weight: 170 + parent: "configuring" +toc: true +aliases: + - /riak-docs/riak/2.2.6/ops/advanced/configs/mapreduce/ + - /riak-docs/riak/kv/2.2.6/ops/advanced/configs/mapreduce/ +--- + +[usage mapreduce]: {{}}riak/kv/2.2.6/developing/usage/mapreduce +[config reference#appconfig]: {{}}riak/kv/2.2.6/configuring/reference/#app-config +[usage secondary-indexes]: {{}}riak/kv/2.2.6/developing/usage/secondary-indexes + +## Configuring MapReduce + +[MapReduce (M/R)][usage mapreduce] is always enabled, but configurable +through the [app.config][config reference#appconfig] file as +follows under `riak_kv` + +```erlang +{riak_kv, [ +``` + +`mapred_name` is the URL directory used to submit M/R requests to Riak. +By default `mapred`, making the command path, for example: +`http://localhost:8098/mapred` + +```erlang + {mapred_name, "mapred"}, +``` + +`mapred_2i_pipe` indicates whether [2i][usage secondary-indexes] +MapReduce inputs are queued in parallel in their own pipe (`true`), or +serially through a helper process (`false` or undefined). + +> **Note**: Set to `false` or leave undefined during an upgrade from 1.0. + +```erlang + {mapred_2i_pipe, true}, +``` + +Each of these entries control how many Javascript virtual machines are +available for executing map, reduce, pre- and post-commit hook +functions. + +This is largely relevant only if you are writing JavaScript M/R jobs. + +```erlang + {map_js_vm_count, 8 }, + {reduce_js_vm_count, 6 }, + {hook_js_vm_count, 2 }, +``` + +`js_max_vm_mem` is the maximum amount of memory, in megabytes, allocated +to the Javascript VMs. If unset, the default is 8MB. + +This is largely relevant only if you are writing JavaScript M/R jobs. + +```erlang + {js_max_vm_mem, 8}, +``` + +`js_thread_stack` is the maximum amount of thread stack, in megabytes, +allocated to the Javascript VMs. If unset, the default is 16MB. + +> **Note**: This is not the same as the C thread stack. + +```erlang + {js_thread_stack, 16}, +``` + +`js_source_dir` should point to a directory containing Javascript source +files which will be loaded when Riak initializes Javascript VMs. + +```erlang + %{js_source_dir, "/tmp/js_source"}, +``` + + + +## Configuration Tuning for Javascript + +If you load larger JSON objects in your buckets there is a possibility you might encounter an error like the following: + +```json + {"lineno":465,"message":"InternalError: script stack space quota is exhausted","source":"unknown"} +``` + + +You can increase the amount of memory allocated to the Javascript VM stack by editing your app.config. The following will increase the stack size from 8MB to 32MB: + +```erlang +{js_thread_stack, 8} +``` + +becomes + +```erlang +{js_thread_stack, 32}, +``` + +In addition to increasing the amount of memory allocated to the stack you can increase the heap size as well by increasing the `js_max_vm_mem` from the default of 8MB. If you are collecting a large amount of results in a reduce phase you may need to increase this setting. + +## Configuration for Riak 1.0 + +Riak 1.0 is the first release including the new MapReduce subsystem known as Riak Pipe. By default, new Riak clusters will use Riak Pipe to power their MapReduce queries. Existing Riak clusters that are upgraded to Riak 1.0 will continue to use the legacy MapReduce system unless the following line is added to the riak_kv section of each node's app.config: + +```erlang +%% Use Riak Pipe to power MapReduce queries +{mapred_system, pipe}, +``` + +> **Warning:** +> +> Do not enable Riak Pipe for MapReduce processing until all nodes in the cluster are running Riak 1.0. + +Other than speed and stability of the cluster, the choice of MapReduce subsystem (Riak Pipe or legacy) should be invisible to your client. All queries should have the same syntax and return the same results on Riak 1.0 with Riak Pipe as they did on earlier versions with the legacy subsystem. If you should find a case where this is not true, you may revert to using the legacy subsystem by either removing the aforementioned line in your app.config or by changing it to read like this: + +```erlang +%% Use the legacy MapReduce system +{mapred_system, legacy}, +``` + +## Configuration Tuning for Reduce Phases + +If you are using Riak 1.0 and the Riak Pipe subsystem for MapReduce queries, you have additional options for tuning your reduce phases. + +### Batch Size + +By default, Riak will evaluate a reduce function every time its phase receives 20 new inputs. If your reduce phases would run more efficiently with more or fewer new inputs, you may change this default by adding the following to the riak_kv section of your app.config: + +```erlang +%% Run reduce functions after 100 new inputs are received +{mapred_reduce_phase_batch_size, 100}, +``` + +You may also control this batching behavior on a per-query basis by using the static argument of the phase specification. When specifying phases over HTTP, the JSON configuration for evaluating the function after 150 new inputs looks like this: + +```json +{"reduce": + {...language, etc. as usual... + "arg":{"reduce_phase_batch_size":150}}} +``` + +In Erlang, you may either specify a similar mochijson2 structure for the phase argument, or use the simpler proplist form: + +```erlang +{reduce, FunSpec, [{reduce_phase_batch_size, 150}], Keep} +``` + +Finally, if you want your reduce function to be evaluated only once, after all inputs are received, use this argument instead: + +```json +{"reduce": + {...language, etc. as usual... + "arg":{"reduce_phase_only_1":true}}} +``` + +Similarly, in Erlang: + +```erlang +{reduce, FunSpec, [reduce_phase_only_1], Keep} +``` + +> **Warning:** +> +> A known bug in Riak 1.0.0 means that it is possible a reduce function may run more often than specified if handoff happens while the phase is accumulating inputs. This bug was fixed in 1.0.1. + +### Pre-Reduce + +If your reduce functions can benefit from parallel execution, it is possible to request that the outputs of a preceding map phase be reduced local to the partition that produced them, before being sent, as usual, to the final aggregate reduce. + +Pre-reduce is disabled by default. To enable it for all reduce phases by default, add the following to the riak_kv section of your app.config: + +```erlang +%% Always pre-reduce between map and reduce phases +{mapred_always_prereduce, true} +``` + +Pre-reduce may also be enabled or disabled on a per-phase basis via the Erlang API for map phases implemented in Erlang. To enable pre-reduce, for any map phase followed by a reduce phase, pass a proplist as its static phase argument and include the following flag: + +```erlang +{map, FunSpec, [do_prereduce], Keep} +``` + +> **Warning:** +> +>A known bug in Riak 1.0.0 prevents per-phase pre-reduce from being enabled over HTTP. This bug also prevents per-phase pre-reduce from being enabled for Javascript phases. Use the global app.config flag for these cases. This bug was fixed in 1.0.1. diff --git a/content/riak/kv/2.2.6/configuring/reference.md b/content/riak/kv/2.2.6/configuring/reference.md new file mode 100644 index 0000000000..0752c53774 --- /dev/null +++ b/content/riak/kv/2.2.6/configuring/reference.md @@ -0,0 +1,1984 @@ +--- +title: "Riak KV Configuration Reference" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Configuration Reference" + identifier: "configuring_reference" + weight: 140 + parent: "configuring" +toc: true +aliases: + - /riak-docs/riak/2.2.6/ops/advanced/configs/configuration-files/ + - /riak-docs/riak/kv/2.2.6/ops/advanced/configs/configuration-files/ +--- + +[concept clusters]: ../../learn/concepts/clusters +[plan backend bitcask]: ../../setup/planning/backend/bitcask +[config backend bitcask]: ../../setup/planning/backend/bitcask/#configuring-bitcask +[plan backend leveldb]: ../../setup/planning/backend/leveldb +[config backend leveldb]: ../../setup/planning/backend/leveldb/#configuring-eleveldb +[plan backend memory]: ../../setup/planning/backend/memory +[config backend memory]: ../../setup/planning/backend/memory/#configuring-the-memory-backend +[plan backend multi]: ../../setup/planning/backend/multi +[config backend multi]: ../../setup/planning/backend/multi/#configuring-multiple-backends-1 +[use admin riak cli]: ../../using/admin/riak-cli +[use admin riak-admin]: ../../using/admin/riak-admin +[glossary aae]: ../../learn/glossary/#active-anti-entropy-aae +[use ref search 2i]: ../../using/reference/secondary-indexes +[cluster ops bucket types]: ../../using/cluster-operations/bucket-types +[usage conflict resolution]: ../../developing/usage/conflict-resolution +[concept causal context]: ../../learn/concepts/causal-context +[usage mapreduce]: ../../developing/usage/mapreduce +[security index]: ../../using/security/ +[cluster ops strong consistency]: ../../using/cluster-operations/strong-consistency +[glossary vnode]: ../../learn/glossary/#vnode +[cluster ops handoff]: ../../using/cluster-operations/handoff +[Search Settings]: ../search#search-config-settings + +Riak has a `riak.conf` configuration file located in `/etc` if you are +using a source install or in `/etc/riak` or `/usr/local/etc` if you used +a binary install. + +The `riak.conf` file is used to set a wide variety of attributes for the +node, from the storage backend that the node will use to store data to +the location of SSL-related files to sibling resolution parameters and +beyond. + +> **Note on upgrades to 2.0** +> +> If your cluster is currently running a version of Riak prior to 2.0 and +you'd like to upgrade to version 2.0 or later, you may continue to use +your old `app.config` and `vm.args` files. You may also use the newer +`riak.conf` alongside them, but please be aware that any settings in +`app.config` or `vm.args` will override settings in `riak.conf`. + +## The advanced.config file + +For most Riak installations, the `riak.conf` file should be sufficient +for configuration management. But some installations, particularly those +upgrading from an earlier version of Riak to version 2.0 or later, may +need to make use of an `advanced.config` file to control some settings +available only in versions prior to 2.0. If this applies to your +installation, please see the [Advanced Configuration](#advanced-configuration) section below. + +## Node Metadata + +Every Riak node has a name and a cookie used to facilitate inter-node +communication. The following parameters enable you to customize the name +and cookie. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ConfigDescriptionDefault
distributed_cookieCookie for distributed node communication within a Riak cluster. +All nodes in the same cluster should use the same cookie or they will +not be able to communicate.riak
nodenameThe name of the Riak node.riak@127.0.0.1
ring_sizeNumber of partitions in the cluster (only valid when first creating +the cluster). Must be a power of 2. The minimum is 8 and the maximum is +1024.64
+ +## Ring + +Configurable parameters for your cluster's [ring][concept clusters]. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ConfigDescriptionDefault
ring.state_dirDefault location of ringstate../data/ring
ring_sizeNumber of partitions in the cluster (only valid when first creating +the cluster). Must be a power of 2. The minimum is 8 and the maximum is +1024.64
transfer_limitNumber of concurrent node-to-node transfers allowed.2
+ +## Storage Backend + +Riak enables you to choose from the following storage backends: + +* [Bitcask][plan backend bitcask] --- [configuration][config backend bitcask] +* [LevelDB][plan backend leveldb] --- [configuration][config backend leveldb] +* [Memory][plan backend memory] --- [configuration][config backend memory] +* [Multi][plan backend multi] --- [configuration][config backend multi] + + + + + + + + + + + + + + + + + + +
ConfigDescriptionDefault
storage_backendSpecifies the storage engine used for Riak's key-value data and +secondary indexes (if supported).

The available options are +bitcask (the default), leveldb, +memory, and multi.
bitcask
+ +## Directories + +The directories in which Riak stores data, logs, dependencies, +executables, and configuration files can be configured using the +parameters below. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ConfigDescriptionDefault
platform_bin_dirThe directory in which the riak-admin, +riak-debug, and now-deprecated search-cmd +executables are stored../bin
platform_data_dirThe directory in which Riak stores its storage backend data, as well +as active anti-entropy data, and cluster metadata../data
platform_etc_dirThe directory in which Riak's configuration files are stored../etc
platform_lib_dirThe directory in which Riak's dependencies are housed../lib
platform_log_dirThe directory in which Riak's log files are stored, e.g. +console.log, erlang.log, and +crash.log files../log
+ +Each of these directory parameters can be used to construct values for +other parameters by placing it within a `$(...)`. Thus, +`platform_log_dir` becomes `$(platform_log_dir)` and so on. + +To give an example, you can select the directory used by Riak's [active anti-entropy](#active-anti-entropy) system using the +`anti_entropy.data_dir` parameter. When setting that parameter, you can +specify an absolute directory, as below: + +```riakconf +anti_entropy.data_dir = /path/to/anti_entropy +``` + +Or you can use the value of `platform_data_dir`: + +```riakconf +anti_entropy.data_dir = $(platform_data_dir)/anti_entropy +``` + +## Search + +Configuration parameters for [Riak KV Search][use ref search 2i]. For a more detailed description of these parameters, check out [Search Settings]. + +Field | Default | Valid values | +:-----|:--------|:-------------| +`search` | `off` | `on` or `off` +`search.anti_entropy.data_dir` | `./data/yz_anti_entropy` | Directory +`search.anti_entropy.throttle.$tier.delay` | No default | Non-negative integer +`search.anti_entropy.throttle.$tier.solrq_queue_length` | No default | Non-negative integer +`search.dist_query` | `on` | `on` or `off` +`search.index.error_threshold.failure_count` | `3` | Integer +`search.index.error_threshold.failure_interval` | `5000` | Milliseconds +`search.index.error_threshold.reset_interval` | `30000` | Milliseconds +`search.queue.batch.flush_interval` | `1000` | `ms`, `s`, `m`, `h` +`search.queue.batch.maximum`| `100` | Integer +`search.queue.batch.minimum` | `1` | Integer +`search.queue.high_watermark` | `10000` | Integer +`search.queue.high_watermark.purge_strategy` | `purge_one` | `purge_one`, `purge_index`, or `off` +`search.root_dir` | `./data/yz` | Directory +`search.solr.jvm_options` | `-d64 -Xms1g -Xmx1g -XX:+UseStringCache -XX:+UseCompressedOops` | Java command-line arguments +`search.solr.jmx_port` | `8985` | Integer +`search.solr.jmx_port` | `8985` | Integer +`search.solr.port` | `8093` | Integer +`search.solr.start_timeout` | `30s` | Integer with time units (eg. 2m) +`yokozuna.aae_throttle_enabled` | `on` | `on` or `off` + + +## Riak Control + +Riak Control is a web-based administrative console for inspecting and +manipulating Riak clusters. The configurable parameters below enable you +to turn the Riak Control subsystem on and off and to configure console +authorization. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ConfigDescriptionDefault
riak_controlSet to off to disable the admin panel.off
riak_control.auth.modeAuthentication mode used for access to the admin panel. Options are +off (which is the default) or userlist.off
riak_control.auth.user.$username.passwordIf Riak Control's authentication mode +(riak_control.auth.mode) is set to userlist, +this is the list of usernames and passwords for access to the admin +panel.
+ +## Runtime Health + +Configurable parameters for interaction between Riak and the underlying +operating system. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ConfigDescriptionDefault
runtime_health.triggers.distribution_portWhether distribution ports with full input buffers will be counted +as busy. Distribution ports connect Riak nodes within a single cluster. +on
runtime_health.triggers.portWhether ports with full input buffers will be counted as busy. +Ports can represent open files or network sockets.on
runtime_health.triggers.process.heap_sizeA process will become busy when its heap exceeds this size +(in bytes).160444000
runtime_health.triggers.process.garbage_collectionA process will become busy when it exceeds this amount of time doing +garbage collection. Set as an integer plus time unit, e.g. `50ms` for 50 +milliseconds, `5s` for 5 seconds, etc.Note: Enabling +this setting can cause performance problems on multi-core systems.off
runtime_health.triggers.process.long_scheduleA process will become busy when it exceeds this amount of time +during a single process scheduling and execution cycle. Set as an integer +plus time unit, e.g. `50ms` for 50 milliseconds, `5s` for 5 seconds, +etc.off
runtime_health.thresholds.busy_portsThe threshold at which a warning will be triggered about the number +of ports that are overly busy. Ports with full input buffers count +toward this threshold.2
runtime_health.thresholds.busy_processesThe threshold at which to warn a warning will be triggered about the +number of processes that are overly busy. Processes with large heaps or +that take a long time to garbage collect will count toward this +threshold.30
+ +## Default Bucket Properties + +When configuring buckets [using bucket types][cluster ops bucket types], the table below lists the bucket properties that are used when no bucket type is specified. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ConfigDescriptionDefault
buckets.default.allow_multWhether or not siblings are allowed +

+Note: See +Conflict Resolution for a discussion of siblings.
false
buckets.default.basic_quorumWhether not-founds will invoke the "basic quorum" optimization. +This setting will short-circuit fetches where the majority of replicas +report that the key is not found. Only used when +notfound_ok is set to false.false
buckets.default.dwThe number of replicas which must reply to a write request +indicating that the write was committed to durable storage for the write +to be deemed successful.quorum
buckets.default.last_write_winsWhether conflicting writes resolve via timestamp.false
buckets.default.merge_strategyThe strategy used when merging objects that potentially have +conflicts. The default is 2 in Riak 2.0 for typed buckets +and 1 for non-typed buckets. This setting reduces sibling +creation through additional metadata on each sibling (also known as Dotted +Version Vectors). Setting this to 1 is the default for +Riak 1.4 and earlier, and may duplicate siblings that originated in the +same write.1
buckets.default.n_valThe number of replicas stored in **non-typed** buckets. For typed buckets, the default is 3 unless changed explicitly for that bucket type. +

+Note: See +Replication Properties +for further discussion.
3
buckets.default.notfound_okWhether not-founds will count toward a quorum of reads.true
buckets.default.postcommitA space-delimited list of functions that will be run after a value +is stored. Only Erlang functions are allowed, using the +module:function format.
buckets.default.precommitA space-delimited list of functions that will be run before a value +is stored, and that can abort the write. Only Erlang functions are +allowed, using the module:function format.
buckets.default.prThe number of primary, non-fallback replicas that must reply to a +read request.0
buckets.default.pwThe number of primary, non-fallback replicas which must reply to a +write request.0
buckets.default.rThe number of replicas which must reply to a read request.quorum
buckets.default.wThe number of replicas which must reply to a write request, +indicating that the write was received.quorum
buckets.default.rwThe number of replicas which must reply to a delete request.quorum
+ +## Object Settings + +Configurable parameters for [conflict resolution][usage conflict resolution] and dealing with [sibling explosion][concept causal context]. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ConfigDescriptionDefault
object.formatControls which binary representation of a riak value is stored on +disk. Options are 0, which will use the original +erlang:term_to_binary format but has a higher space +overhead, or 1, which will tell Riak to utilize a new +format for more compact storage of small values.1
object.siblings.maximumWriting an object with more than this number of siblings will send +a failure to the client.100
object.siblings.warning_thresholdWriting an object with more than this number of siblings will +generate a warning in the logs.25
object.size.maximumWriting an object larger than this will send a failure to the +client.50MB
object.size.warning_thresholdReading or writing objects larger than this size will write a +warning in the logs.5MB
+ +## Erlang VM + +In the older configuration system, the Erlang VM in which Riak runs was +configured using a `vm.args` file. In the new, `riak.conf`-based +system, the Erlang VM can be configured using the parameters in the +table below. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ConfigDescriptionDefault
erlang.async_threadsThe number of threads in the Erlang VM's asynchronous thread pool. +The valid range is 0-1024. If thread support is not available, this +parameter will have no impact; if thread support is available, the +default value is 64. This is the equivalent of the +A flag. +More information can be found here. +64 (if thread support is available)
erlang.async_threads.stack_sizeIf thread support is available in your Erlang VM, this parameter +sets the amount of memory allocated to each asynchronous thread, which +you can set as KB, MB, GB, etc. The valid range is 16-8192 kilowords, +which translates to 64-32768 KB on 32-bit architectures. Although there +is no default, we suggest a stack size of 16 kilowords, which translates +to 64 KB. This small default size has been chosen because the number of +asynchronous threads, set using the erlang.async_threads +parameter explained above, might be quite large. The 64 KB default is +enough for drivers delivered with Erlang/OTP but might not be large +enough to accommodate drivers that use the driver_async() +functionality, documented here.
erlang.distribution.net_ticktimeThe net kernel is an Erlang system process that provides various +forms of network monitoring. In a Riak cluster, one of the functions of +the net kernel is to periodically check node liveness. Tick +time is the frequency with which those checks happen. This +parameter determines that frequency for every N. If you set +this parameter to 10, for example, the tick will occur once +every 10 seconds.
erlang.distribution.port_range.minimumFor ease of firewall configuration, the Erlang distribution can be +bound to a limited range of TCP ports. If this parameter is set, and +erlang.distribution.port_range.maximum is not set, only +this port will be used. If the minimum is unset, no restriction will be +made on the port range. Instead, Erlang will listen on a random +high-numbered port. More information here and here.
erlang.distribution.port_range.maximumSee the description for +erlang.distribution.port_range.minimum directly above. +
erlang.schedulers.force_wakeup_intervalSet the scheduler forced wakeup interval. All run queues will be +scanned each time period specified (in milliseconds). While there are +sleeping schedulers in the system, one scheduler will be woken for each +non-empty run queue found. An interval of zero disables this feature, +which is the default. This feature is a workaround for lengthy executing +native code, and native code that does not properly bump reductions. +More information here.
erlang.schedulers.compaction_of_loadEnables or disables the Erlang scheduler's compaction of load. When +enabled (which is the default), load balancing will strive to establish +a load distribution that causes as many scheduler threads as possible to +be fully loaded, i.e. not to run out of scheduled work. This is +accomplished by migrating load, such as running processes, into a +smaller set of schedulers when schedulers frequently run out of work. +When disabled, the frequency at which schedulers run out of work will +not be taken into account by the load balancing logic.true (enabled)
erlang.schedulers.utilization_balancingEnables or disables the Erlang scheduler's balancing of load. By +default, scheduler utilization of balancing is disabled while scheduler +compaction of load is enabled, i.e. +erlang.schedulers.compaction_of_load is set to +true. In this state, the Erlang VM will strive for a load +distribution which causes as many scheduler threads as possible to be +fully loaded, i.e. to not run out of work. When load balancing is +enabled using this setting, the system will attempt to equally scheduler +utilization between schedulers.false (disabled)
erlang.distribution_buffer_sizeFor nodes with many busy_dist_port events, Basho +recommends raising the sender-side network distribution buffer size. +32MB may not be sufficient for some workloads and is a suggested +starting point. Erlangers may know this as +zdbbl. See more +here +.32MB
erlang.process_limitRaises the default Erlang process limit256000
erlang.max_ets_tablesRaises the ETS table limit256000
erlang.crash_dumpSets the location of crash dumps./log/erl_crash.dump
erlang.fullsweep_afterA non-negative integer which indicates how many times generational +garbage collections can be done without forcing a fullsweep collection. +In low-memory systems (especially without virtual memory), setting the +value to 0 can help to conserve memory. More information here. +0
erlang.max_portsThe number of concurrent ports/sockets. The valid range is 1024 to +134217727.65536
erlang.KEnables or disables the kernel poll functionality if the emulator +supports it. If the emulator does not support kernel poll, and the +K flag is passed to the emulator, a warning is issued at +startup. Similar information here.on
erlang.schedulers.totalSets the number of scheduler threads to create and scheduler +threads to set online when erlang.smp support has been +enabled. The maximum for both values is 1024. If the Erlang runtime +system is able to determine the amount of logical processors configured +and logical processors available, schedulers.total will +default to logical processors configured, and +schedulers.online will default to the number of logical +processors available. Otherwise, the default values will be 1. +Schedulers may be omitted if schedulers.online is not and +vice versa. If schedulers.total or +schedulers.online is specified as a negative number, the +value is subtracted from the default number of logical processors +configured or logical processors available, respectively. Specifying +the value 0 for Schedulers or +SchedulersOnline resets the number of scheduler threads or +scheduler threads online respective to its default value. This option +is ignored if the emulator doesn't have SMP support enabled (see the +erlang.smp flag). More information +here. +
erlang.schedulers.onlineSee the description for erlang.schedulers.total +directly above.
erlang.WSets the mapping of warning messages for error_logger. +Messages sent to the error logger using one of the warning routines can +be mapped either to errors, warnings (w, +which is the default), or info reports (i).w
erlang.smpStarts the Erlang runtime system with SMP support enabled. This may +fail if no runtime system with SMP support is available. The +auto setting starts the Erlang runtime system with SMP +support enabled if it is available and more than one logical processor +is detected. A value of disable starts a runtime system +without SMP support. Note: The runtime system with SMP +support will not be available on all supported platforms. See also the +erlang.schedulers settings. Some native extensions (NIFs) +require use of the SMP emulator. More information here.enable
erlang.shutdown_timeLimits how long the Erlang VM spends shutting down. After the +specified duration elapses, all existing processes are killed.10s
+ +## JavaScript MapReduce + +Configurable parameters for Riak's now-deprecated JavaScript +[MapReduce][usage mapreduce] system. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ConfigDescriptionDefault
javascript.source_dirA directory containing the Javascript source files which will be +loaded by Riak when it initializes Javascript VMs.
javascript.maximum_stack_sizeThe maximum amount of thread stack memory to allocate to each +JavaScript virtual machine.16MB
javascript.maximum_heap_sizeThe maximum amount of memory allocated to each JavaScript virtual +machine.8MB
javascript.hook_pool_sizeThe number of JavaScript virtual machines available for executing +pre-commit hook functions.2
javascript.reduce_pool_sizeThe number of JavaScript virtual machines available for executing +reduce functions.6
javascript.map_pool_sizeThe number of JavaScript virtual machines available for executing +map functions.8
+ +## Security + +Configurable parameters for [Riak KV Security][security index]. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ConfigDescriptionDefault
ssl.cacertfileThe default signing authority location for HTTPS.#(platform_etc_dir)/cacertfile.pem
ssl.keyfileDefault key location for HTTPS.#(platform_etc_dir)/key.pem
ssl.certfileDefault cert location for HTTPS.#(platform_etc_dir)/cert.pem
secure_referer_checkMeasures were added to Riak 1.2 to counteract cross-site scripting +and request-forgery attacks. Some reverse proxies cannot remove the +Referer header and make serving data directly from Riak +impossible. Turning this setting to off disables this +security check.on
check_crlWhether to check the certificate +revocation list (CRL) of a client certificate. This defaults to +on but some CAs may not maintain or define a CRL, so this +can be disabled if no CRL is available.on
tls_protocols.sslv3Determine which SSL/TLS versions are allowed. By default, only TLS +1.2 is allowed, but other versions can be enabled if clients don't +support the latest TLS standard. It is strongly recommended that SSLv3 +not be enabled unless absolutely necessary. More than one protocol can +be enabled at once. The tls_protocols parameters below can +be used to turn different versions on and off.off
tls_protocols.tlsv1.2on
tls_protocols.tlsv1.1off
tls_protocols.tlsv1off
honor_cipher_orderWhether to prefer the order in which the server lists its ciphers. +When set to off, the client's preferred cipher order +dictates which cipher is chosen.on
+ +## Client Interfaces + +Configurable parameters for clients connecting to Riak either through +Riak's Protocol Buffers or HTTP API. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ConfigDescriptionDefault
protobuf.nagleTurns off Nagle's algorithm for Protocol Buffers connections. This +is equivalent to setting the TCP_NODELAY option on the +socket.off
protobuf.backlogThe maximum length to which the queue of pending connections may +grow. If set, it must be an integer greater than zero. If you +anticipate a huge number of connections being initialized +simultaneously, set this number higher.128
listener.protobuf.$nameThis is the IP address and TCP port to which the Riak Protocol +Buffers interface will bind.{"127.0.0.1",8087}
listener.http.$nameThis is the IP address and TCP port to which the Riak HTTP +interface will bind.{"127.0.0.1",8098}
listener.https.$nameThis is the IP address and TCP port to which the Riak HTTPS +interface will bind.
+ +## Logging + +Configurable parameters for [lager](https://github.com/basho/lager), +Riak's logging system. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ConfigDescriptionDefault
log.consoleWhere to emit the default log messages (typically at +info severity). Possible values: off, which +disables console log messages; file, which specifies that +log messages will be output to the file specified by +log.console.file; console, which outputs +messages to standard output (seen when using riak +attach-direct); or both, which outputs messages both +to the file specified in log.console.file and to standard +out.file
log.console.fileWhen log.console is set to file or +both, this parameter determines the path of the file to +which console messages will be logged../log/console.log
log.console.levelThe severity level of the console log. Possible +values: +
    +
  • debug
  • +
  • info
  • +
  • warning
  • +
  • error
  • +
info
log.crashWhether to enable the crash logon
log.crash.fileIf the crash log is enabled, the file where its messages will be +written./log/crash.log
log.crash.maximum_message_sizeMaximum size of individual messages in the crash log64KB
log.crash.rotationThe schedule on which to rotate the crash log. More information here. +$D0
log.crash.rotation.keepThe number of rotated crash logs to keep. When set to +current, only the current open log file is kept. +Otherwise, an integer can be specified.5
log.crash.sizeMaximum size of the crash log before it is rotated10MB
log.error.fileThe file where error messages will be logged../log/error.log
log.error.messages_per_secondMaximum number of error_logger messages to handle per +second100
log.error.redirectWhether to redirect error_logger messages into +lageron
log.syslogWhen set to on, enables log output to syslogoff
log.syslog.facilitySets the facility +level of syslog output if log.syslog is set to +on. Possible values: +
  • auth
  • authpriv
  • +
  • clock
  • cron
  • +
  • daemon
  • ftp
  • +
  • kern
  • lpr
  • +
  • mail
  • news
  • +
  • syslog
  • user
  • +
  • uucp
+In addition to these settings, you may also select local0 +through local7.
daemon
log.syslog.identIf log.syslog is set to on, this setting +determines the prefix appended to each syslog message.riak
log.syslog.levelIf log.syslog is set to on, this setting +determines the log level of syslog output. Possible values: +
  • alert
  • critical
  • +
  • debug
  • emergency
  • +
  • error
  • info
  • none
  • notice
  • +
  • warning
info
saslWhether to enable sasl, Erlang's +built-in error loggeroff
+ +## Active Anti-Entropy + +Configurable parameters for Riak's active anti-entropy subsystem. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ConfigDescriptionDefault
anti_entropyHow Riak will repair out-of-sync keys. If set to +active, out-of-sync keys will be repaired in the +background; if set to passive, out-of-sync keys are only +repaired on read; and if set to active-debug, verbose +debugging information will be output.active
search.anti_entropy.throttleWhether the distributed throttle for Active Anti-Entropy is +enabled.on
search.anti_entropy.throttle.$tier.solrq_queue_lengthSets the throttling tiers for Active Anti-Entropy. Each tier is a +minimum vnode mailbox size and a time-delay that the throttle should +observe at that size and above. For example, +anti_entropy.throttle.tier1.mailbox_size = 0, +anti_entropy.throttle.tier1.delay = 0ms, +anti_entropy.throttle.tier2.mailbox_size = 40, +anti_entropy.throttle.tier2.delay = 5ms, etc. If +configured, there must be a tier which includes a mailbox size of 0. +Both .mailbox_size and .delay must be set for +each tier.
search.anti_entropy.throttle.$tier.delaySee the description for +anti_entropy.throttle.$tier.mailbox_size above.
anti_entropy.bloomfilterBloom filters are highly effective in shortcutting data queries +that are destined to not find the requested key, though they tend to +entail a small performance cost.on
anti_entropy.max_open_files20
anti_entropy.write_buffer_sizeThe LevelDB options used by Active Anti-Entropy to generate the +LevelDB-backed on-disk hashtrees.4MB
anti_entropy.data_dirThe directory where AAE hash trees are stored../data/anti_entropy
anti_entropy.trigger_intervalThe tick determines how often the Active Anti-Entropy manager looks +for work to do (building/expiring trees, triggering exchanges, etc). +Lowering this value will speed up the rate at which all replicas are +synced across the cluster. Increasing the value is not recommended. +15s
anti_entropy.concurrency_limitLimit how many Active Anti-Entropy exchanges or builds can happen +concurrently.2
anti_entropy.tree.expiryDetermines how often hash trees are expired after being built. +Periodically expiring a hash tree ensures that the on-disk hash tree +data stays consistent with the actual K/V backend data. It also helps +Riak identify silent disk failures and bit rot. However, expiration is +not needed for normal active anti-entropy operations and should be +infrequent for performance reasons. The time is specified in +milliseconds.1w
anti_entropy.tree.build_limit.per_timespan1h
anti_entropy.tree.build_limit.numberRestrict how fast AAE can build hash trees. Building the tree for a +given partition requires a full scan over that partition's data. Once +built, trees stay built until they are expired. .number is +the number of builds; .per_timespan is the amount of time +in which that number of builds occurs.1
anti_entropy.use_background_managerWhether AAE is to use a background process to limit AAE tree +rebuilds. If set to on, this will help to prevent system +response degradation under times of heavy load from multiple background +tasks that contend for the same system resources; setting this parameter +to off can cut down on system resource usage. +off
+ +## Intra-Cluster Handoff + +Configurable parameters for intra-cluster, i.e. inter-node, [handoff][cluster ops handoff]. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ConfigDescriptionDefault
handoff.max_rejectsThe maximum number of times that a secondary system within Riak, +such as Riak Search, can block handoff +of primary key/value data. The approximate maximum duration that a vnode +can be blocked can be determined by multiplying this setting by +vnode_management_timer. If you want to prevent handoff from +ever being blocked by a secondary system, set this parameter to +0.6
handoff.inboundWhether inbound handoff is enabled on the node. Possible values are +on or off.on
handoff.outboundWhether outbound handoff is enabled on the node. Possible values are +on or off.on
handoff.portSpecifies the TCP port that Riak uses for intra-cluster data +handoff.8099
handoff.ssl.certfileTo encrypt riak_core intra-cluster data handoff +traffic, uncomment this line and edit its path to an appropriate +certfile and keyfile.
handoff.ssl.keyfileThe keyfile paired with the certfile specified in +.certfile.
handoff.use_background_managerWhether Riak will use a background manager to limit K/V handoff. +This can help to prevent system response degradation during times of +heavy load caused by multiple background tasks that contend for the same +system resources; setting this parameter to off can cut +down on system resource usage.off
+ +## Riak Data Types + + + + + + + + + + + + + + + + + + + +
ConfigDescriptionDefault
datatypes.compression_levelWhether serialized Data Types will use compression and at what +level. When set to an integer, the parameter refers to the +aggressiveness of compression, on a scale from 0 to 9. on +is equivalent to 6, whereas off is equivalent to 0. Higher +values for compression tend to be more CPU intensive.1
+ +## SNMP + +Owing to lack of usage, SNMP support has been removed from Riak KV 2.2.6 and higher. + +## JMX + +Owing to lack of usage, JMX support has also been removed from Riak KV 2.2.6 and higher. + +## Strong Consistency + +> **Please Note:** +> +> Riak KV's strong consistency is an experimental feature and may be removed from the product in the future. Strong consistency is not commercially supported or production-ready. Strong consistency is incompatible with Multi-Datacenter Replication, Riak Search, Bitcask Expiration, LevelDB Secondary Indexes, Riak Data Types and Commit Hooks. It suffers from known issues and we do not recommend its usage in any production environment. + +Riak's strong consistency feature has a variety of tunable parameters +that allow you to enable and disable strong consistency, modify the +behavior of leaders and followers, set various timeouts, and more. More +detailed information from an operations perspective can be found in our +documentation on [managing strong consistency][cluster ops strong consistency]. + +Strong consistency is disabled by default. The `strong_consistency` +parameter enables you to turn it on. This setting is available in each +node's `riak.conf` file. + + + + + + + + + + + + + + + + + + +
ConfigDescriptionDefault
strong_consistencyEnables the consensus subsystem used for strongly consistent Riak +operations if set to on.off
+ +Unlike the `strong_consistency` setting, the settings listed below are +available only in `advanced.config`, in the `riak_ensemble` section of +that file. That section looks like this: + +```advancedconfig +{riak_ensemble, [ + {parameter1, value}, + {parameter2, value}, + %% Other setting + ]} +``` + +Further instructions on setting parameters in `advanced.config` can be +found in the [advanced configuration](#advanced-configuration) section below. + +Using these settings properly demands a firm understanding of the basic +architecture of Riak's implementation of strong consistency. We highly +recommend reading our documentation on the [implementation details][cluster ops strong consistency] behind +strong consistency before changing the defaults on these parameters. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ConfigDescriptionDefault
ensemble_tickThe rate at which leaders perform their periodic duties, including +refreshing the leader lease, in milliseconds. This setting must be lower +than both the lease_duration and +follower_timeout settings (both listed below). Lower values +mean that leaders perform their duties more frequently, which can allow +for faster convergence if a leader goes offline and then returns to the +ensemble; higher values mean that leaders perform their duties less +frequently, which can reduce network overhead.500
lease_durationDetermines how long a leader lease remains valid without being +refreshed (in milliseconds). This should be set higher than the +ensemble_tick setting (listed above) so that leaders have +time to refresh their leases before they time out, and it must be set +lower than the follower_timeout setting (listed below). +ensemble_tick * 3/2
follower_timeoutDetermines how long a follower waits to hear from a leader before it +abandons the leader (in milliseconds). This must be set greater than the +lease_duration setting.lease_duration * 4
alive_tokensDetermines the number of ticks the leader will wait to hear from its +associated vnode before assuming that the vnode +is unhealthy and stepping down as leader. If the vnode does not respond +to the leader before ensemble_tick * +alive_tokens milliseconds have elapsed, the leader will +give up leadership. It may be necessary to raise this setting if your +Riak vnodes are frequently stalling out on slow backend reads/writes. If +this setting is too low, it may cause slow requests to time out earlier +than the request timeout.2
storage_delayDetermines how long the consensus subsystem delays syncing to disk +when performing certain metadata operations (in milliseconds). This +delay allows multiple operations to be coalesced into a single disk +write. We do not recommend that you change this setting.50
storage_tickDetermines how often the consensus subsystem writes data to disk +that was requested to be written asynchronously (in milliseconds). We do +not recommend that you change this setting.5000
trust_leaseDetermines whether leader leases are used to optimize reads. When +set to true, a leader with a valid lease will handle the +read directly without contacting any followers; when set to +false, the leader will always contact followers. For more +information, see our internal documentation on + +leader leases.true
peer_get_timeoutDetermines the timeout used internally for reading consistent data, +in milliseconds. This setting must be greater than the highest request +timeout used by your application.60000 (1 minute)
peer_put_timeoutDetermines the timeout, in milliseconds, used internally for writing +consistent data. This setting must be greater than the highest request +timeout used by your application.60000 (1 minute)
peer_workersThe number of concurrent workers used by the leader to service +requests. Increasing this setting may boost performance depending on the +workload.1
tree_validationDetermines whether Riak considers peer Merkle trees to be trusted +after a node restart. When validation is enabled (the default), Riak +does not trust peer trees after a restart, instead requiring the peer to +sync with a trusted majority. This is the safest option, as it protects +Riak against undetected corruption of the Merkle tree. However, this +mode reduces Riak availability since it can sometimes require more than +a simple majority of nodes to be online and reachable.true
synchronous_tree_updatesDetermines whether the metadata updates to follower Merkle trees are +handled synchronously or not. When set to true, Riak +requires two quorum round trips to occur before replying back to the +client, the first quorum request to write the actual object and the +second to write the Merkle tree data. When set to false, +Riak will respond back to the client after the first round trip, letting +the metadata update happen asynchronously.

It's important to +note that the leader always updates its local Merkle tree +before responding to the client. This setting only affects the metadata +writes sent to followers.

In principle, asynchronous updates +are unsafe. If the leader crashes before sending the metadata updates +and all followers that had acknowledged the object write somehow revert +to the object value immediately prior to a write request, a future read +could return the immediately preceding value without realizing that it +was incorrect. Given that this scenario is unlikely, this setting +defaults to false in the name of improved performance.
false
+ + +## Miscellaneous + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ConfigDescriptionDefault
metadata_cache_sizeThis setting controls the size of the metadata cache for each vnode. +The cache can be disabled by setting it to off (this is the +default). Enabling the cache should not be necessary in disk-based +backends (i.e. LevelDB and Bitcask) but it can help performance in the +Memory backend. Note that this setting adjusts the size of the ETS table +rather than the actual data. Thus, more space may be used than the +simple size * number-of-vnodes calculation would imply. +

+Caution: This setting should not be changed without +extensive benchmarking.
off
max_concurrent_requestsThe maximum number of concurrent requests of each type (GET or PUT) +that is allowed. Setting this value to infinite disables +overload protection. The erlang.process_limit should be at +least 3 times this setting.50000
dtraceWhether DTrace is enabled. +Do not enable unless your Erlang/OTP runtime is compiled to support +DTrace, which is available in R15B01 (supported by the official source +package) and in R14B04 via a custom repository and branch.off
vnode_management_timerSets the frequency with which vnodes attempt to trigger handoff between +this node and other nodes in the cluster.10s (10 seconds)
retry_put_coordinator_failureWhen a PUT (i.e. write) request fails, Riak will retry the operation +if this setting is set to on, which is the default. Setting +it to off will speed response times on PUT requests in +general, but at the risk of potentially increasing the likelihood of +write failure.on
background_managerRiak's background manager is a subsystem that coordinates access to +shared resources from other Riak subsystems. The background manager can +help to prevent system response degradation under times of heavy load +caused by multiple background tasks.on
+ +## Advanced Configuration + +The `advanced.config` file takes the same format as the `app.config` +file familiar to users of versions of Riak prior to 2.0. Here is an +example: + +```advancedconfig +[ + {riak_core, + [ + {cluster_mgr, {"127.0.0.1", 8098 } }, + %% more riak_core configs + ]}, + + {riak_repl, + [ + {data_root, "/var/db/riak/riak_repl/"}, + %% more riak_repl configs + ] + } +]. +``` + +The following settings are available in the `advanced.config` file: + +#### `riak_repl` settings + +Most settings that are configurable through `advanced.config` are +related to Riak's `riak_repl` subsystem. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ConfigDescriptionDefault
data_rootPath (relative or absolute) to the working directory for the +replication process./var/db/riak/riak_repl/
max_fssource_clusterThe hard limit of fullsync workers that will be running on the +source side of a cluster across all nodes on that cluster for a fullsync +to a sink cluster. This means that if you have configured fullsync for +two different clusters, both with a max_fssource_cluster of +5, 10 fullsync workers can be in progress. This only affects nodes on +the source cluster on which this parameter is defined, either via the +configuration file or command line.5
max_fssource_nodeThis setting limits the number of fullsync workers that will be +running on each individual node in a source cluster. This is a hard +limit for all fullsyncs enabled; additional fullsync configurations will +not increase the number of fullsync workers allowed to run on any node. +This only affects nodes on the source cluster on which this parameter is +defined, either via the configuration file or command line. +1
max_fssink_nodeThis setting limits the number of fullsync workers allowed to run on +each individual node in a sink cluster. This is a hard limit for all +fullsyncs enabled; additional fullsync configurations will not increase +the number of fullsync workers allowed to run on any node. This only +affects nodes on the source cluster on which this parameter is defined, +either via the configuration file or command line.1
fullsync_on_connectWhether to initiate a fullsync on initial connection from the sink +cluster.true
fullsync_intervalA single-integer value representing the duration to wait, in +minutes, between fullsyncs, or a list of {clustername, +time_in_minutes} pairs for each sink participating in fullsync +replication.30
rtq_max_bytesThe maximum size, in bytes, to which the realtime replication queue +can grow before new objects are dropped. Dropped objects will need to be +replicated with a fullsync.104857600
proxy_getWhether to enable Riak CS proxy_get and block +filter.disabled
rt_heartbeat_intervalA heartbeat message is sent from the source to the sink every +rt_heartbeat_interval seconds. Setting +rt_heartbeat_interval to undefined disables +the realtime heartbeat. This feature is available only in Riak KV +Enterprise Edition 1.3.2 - 2.2.3 and then from Riak KV 2.2.6 onwards.15
rt_heartbeat_timeoutIf a heartbeat response is not received within the time period +specified by this setting (in seconds), the source connection exits and +will be re-established. This feature is available only in Riak KV +Enterprise Edition 1.3.2 - 2.2.3 and then from Riak KV 2.2.6 onwards.15
realtime_connection_rebalance_max_delay_secsShould a server on the source cluster be restarted, this is +the amount of time (in seconds), before the realtime connections are +rebalanced by a change in the number of source nodes.300
fullsync_use_background_managerBy default, fullsync replication will attempt to coordinate with +other Riak subsystems that may be contending for the same resources. +This will help to prevent system response degradations during times of +heavy load from multiple background tasks. To disable background +coordination, set this parameter to `false`. This feature is available +only in Riak KV Enterprise Edition 2.0 and later as well as Riak KV 2.2.6 onwards.true
+ +#### Upgrading Riak Search with `advanced.config` + +If you are upgrading to Riak 2.x and wish to upgrade to the new [Riak Search][use ref search]\(codename Yokozuna), you will need to enable +legacy Search while the upgrade is underway. You can add the following +snippet to your `advanced.config` configuration to do so: + +```advancedconfig +[ + %% Other configs + + {riak_search, [ {enabled, true} ]}, + {merge_index, [ + {data_root, "/var/lib/riak/merge_index"}, + {buffer_rollover_size, 1048576}, + {max_compact_segments, 20} + ]}, + + %% Other configs +]. +``` + +#### Other settings + +There are three non-`riak_repl` settings available in +`advanced.config`. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ConfigSectionDescriptionDefault
add_pathsriak_kvIf you are installing +custom code for Riak, e.g. for the purpose of running MapReduce jobs or commit hooks, this setting specifies +the paths to any compiled .beam files that you wish to use. +This is expressed as a list of absolute paths on the node's filesystem, +e.g. [ "/tmp", "/other" ].
cluster_mgrriak_coreThe cluster manager listens for connections from remote clusters on +the specified IP and port. Every node runs one cluster manager, but only +the cluster manager running on the cluster leader will service requests. +This can change as nodes enter and leave the cluster.{"127.0.0.1", 9080}
delete_moderiak_kvSpecifies how Riak behaves after objects are marked for deletion +with a tombstone. There are three possible settings: keep +disables tombstone removal altogether; immediate removes +objects' tombstones as soon as the delete request is received; and +setting delete_mode to an integer value specifies the +number of milliseconds to wait before removing tombstones. More +information can be found in Object +Deletion.3000 (3 seconds)
target_n_valriak_coreThe highest n_val that you generally intend to use. +This setting affects how partitions are distributed within the cluster, +helping to ensure that "hot spots" don't occur, i.e. that data is never +stored more than once on the same physical node. You will need to change +this setting only in rare circumstances. Assuming that +ring_size is a power of 2, the ideal value for this setting +is both (a) greater than or equal to the largest n_val for +any bucket type and (b) an even divisor of the number of partitions in +the ring, i.e. ring_size. The default is 4, +and the number of physical nodes in your cluster must be greater than +target_n_val for this setting to be effective at preventing +hot spots.4
+ +## Cluster Job Controls + +{{% note title="Warning" %}} +Before changing `cluster.job` controls in a production environment, test your application to ensure it does not have any hidden dependencies on them. +{{% /note %}} + +The `cluster.job` switches control whether classes of jobs are enabled or disabled through the HTTP(S) and Protobuf interfaces. All jobs are enabled by default. + +Field | Default | Valid values | +:-----|:--------|:-------------| +`cluster.job.riak_kv.list_buckets`|`enabled`|`enabled` or `disabled` +`cluster.job.riak_kv.stream_list_buckets`|`enabled`|`enabled` or `disabled` +`cluster.job.riak_kv.list_keys`|`enabled`|`enabled` or `disabled` +`cluster.job.riak_kv.stream_list_keys`|`enabled`|`enabled` or `disabled` +`cluster.job.riak_kv.map_reduce`|`enabled`|`enabled` or `disabled` +`cluster.job.riak_kv.map_reduce_js`|`enabled`|`enabled` or `disabled` +`cluster.job.riak_kv.secondary_index`|`enabled`|`enabled` or `disabled` +`cluster.job.riak_search.query`|`enabled`|`enabled` or `disabled` +`cluster.job.yokozuna.query`|`enabled`|`enabled` or `disabled` diff --git a/content/riak/kv/2.2.6/configuring/search.md b/content/riak/kv/2.2.6/configuring/search.md new file mode 100644 index 0000000000..4b5be7407f --- /dev/null +++ b/content/riak/kv/2.2.6/configuring/search.md @@ -0,0 +1,274 @@ +--- +title: "Riak Search Settings" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Riak Search Settings" + identifier: "configuring_search" + weight: 160 + parent: "configuring" +toc: true +aliases: + - /riak-docs/riak/2.2.6/ops/advanced/configs/search/ + - /riak-docs/riak/kv/2.2.6/ops/advanced/configs/search/ +--- + +[usage search]: {{}}riak/kv/2.2.6/developing/usage/search +[usage search schema]: {{}}riak/kv/2.2.6/developing/usage/search-schemas +[usage search data types]: {{}}riak/kv/2.2.6/developing/usage/searching-data-types +[usage custom extractors]: {{}}riak/kv/2.2.6/developing/usage/custom-extractors +[cluster-ops aae throttle]: {{}}riak/kv/2.2.6/using/cluster-operations/active-anti-entropy/#throttling +[config reference]: {{}}riak/kv/2.2.6/configuring/reference +[config reference#search]: {{}}riak/kv/2.2.6/configuring/reference/#search +[glossary aae]: {{}}riak/kv/2.2.6/learn/glossary/#active-anti-entropy-aae +[security index]: {{}}riak/kv/2.2.6/using/security/ + +[java se downloads]: http://www.oracle.com/technetwork/java/javase/downloads +[java se docs]: http://www.oracle.com/technetwork/java/javase/documentation + +This page covers how to use Riak Search (with +[Solr](http://lucene.apache.org/solr/) integration). + +For a simple reference of the available configs and their defaults, see the [configuration reference][config reference#search]. + +If you are looking to develop on or with Riak Search, take a look at: + +* [Using Search][usage search] +* [Search Schema][usage search schema] +* [Custom Search Extractors][usage custom extractors] +* [Riak KV Data Types and Search][usage search data types] + +## Overview + +We'll be walking through: + +1. [Prequisites](#prerequisites) +2. [Enable Riak Search](#enabling-riak-search) +3. [Search Configuration Settings](#search-config-settings) +4. [Additional Solr Information](#more-on-solr) + +## Prerequisites + +Because Solr is a Java application, you will need to install **Java 7 +or later** on every node. Installation packages can be found on the [Java SE Downloads +page][java se downloads] and instructions in the [Java SE documentation site][java se docs]. + + +## Enabling Riak Search + +Riak Search is not enabled by default, so you must enable it in every +node's [configuration file][config reference] as follows: + +```riak.conf +search = on +``` + + +## Search Config Settings + +You will find all the Riak Search configuration settings in riak.conf. Setting `search` to `on` is required, but other search settings are optional. A handy reference list of these parameters can be found in our [configuration files][config reference#search] documentation. + +### `search` + +Enable or disable search; defaults to `off`. + +Valid values: `on` or `off` + +### `search.anti_entropy.data_dir` + +The directory in which Riak Search stores files related to [active anti-entropy][glossary aae]; defaults to `./data/yz_anti_entropy`. + +Valid values: a directory + +### `search.anti_entropy.throttle` + +Whether the throttle for Yokozuna active anti-entropy is enabled; defaults to `on`. + +Valid values: `on` or `off` + +You can read more about throttling [here][cluster-ops aae throttle]. + +### `search.anti_entropy.throttle.$tier.delay` + +Set the throttling tiers delay for [active anti-entropy][glossary aae]; no default. + +Each tier is a [minimum Solrq queue size](#search-anti-entropy-throttle-tier-solrq-queue-length) and a time-delay that the throttle should observe at that size and above. + +For example: + +``` +search.anti_entropy.throttle.tier1.solrq_queue_length = 0 +search.anti_entropy.throttle.tier1.delay = 0ms +search.anti_entropy.throttle.tier2.solrq_queue_length = 40 +search.anti_entropy.throttle.tier2.delay = 5ms +``` +will introduce a 5 millisecond sleep for any queues of length 40 or higher. If configured, there must be a tier which includes a mailbox size of 0. Both [`.solrq_queue_length`](#search-anti-entropy-throttle-tier-solrq-queue-length) and `.delay` must be set for each tier. There is no limit to the number of tiers that may be specified. See [`search.anti_entropy.throttle`](#search-anti-entropy-throttle). + +Valid values: Non-negative integer + +### `search.anti_entropy.throttle.$tier.solrq_queue_length` + +Set the throttling tiers for [active anti-entropy][glossary aae]; no default. + +Each tier is a minimum Solrq queue size and a [time-delay](#search-anti-entropy-throttle-tier-delay) that the throttle +should observe at that size and above. + +For example: + +``` +search.anti_entropy.throttle.tier1.solrq_queue_length = 0 +search.anti_entropy.throttle.tier1.delay = 0ms +search.anti_entropy.throttle.tier2.solrq_queue_length = 40 +search.anti_entropy.throttle.tier2.delay = 5ms +``` +will introduce a 5 millisecond sleep for any queues of length 40 or higher. If configured, there must be a tier which includes a mailbox size of 0. Both `.solrq_queue_length` and [`.delay`](#search-anti-entropy-throttle-tier-delay) must be set for each tier. There is no limit to the number of tiers that may be specified. See [`search.anti_entropy.throttle`](#search-anti-entropy-throttle). + +Valid values: Non-negative integer + +### `search.dist_query` + +Enable this node in distributed query plans; defaults to `on`. + +If enabled, this node will participate in distributed Solr queries. If disabled, the node will be excluded from Riak search cover plans, and will therefore never be consulted in a distributed query. Note that this node may still be used to execute a query. Use this flag if you have a long running administrative operation (e.g. reindexing) which requires that the node be removed from query plans, and which would otherwise result in inconsistent search results. + +This setting can also be changed via `riak-admin` by issuing one of the following commands: + +``` +riak-admin set search.dist_query=off +``` + or + +``` +riak-admin set search.dist_query=on +``` + +Setting this value in riak.conf is useful when you are restarting a node which was removed from search queries with the `riak-admin` feature. Setting `search.dis_query` in riak.conf will prevent the node from being included in search queries until it is fully spun up. + +Valid values: `on` or `off` + +### `search.index.error_threshold.failure_count` + +The number of failures encountered while updating a search index within [`search.queue.error_threshold.failure_interval`](#search-queue-error-threshold-failure-interval) before Riak KV will skip updates to that index; defaults to `3`. + +Valid values: Integer + +### `search.index.error_threshold.failure_interval` + +The window of time during which `search.queue.error_threshold.failure_count` failures will cause Riak KV to skip updates to a search index; defaults to `5000`. + +If [`search.queue.error_threshold.failure_count`](#search-queue-error-threshold-failure-count) errors have occurred within this interval on a given search index, then Riak will skip updates to that index until the [`search.queue.error_threshold.reset_interval`](search-queue-error-threshold-reset-interval) has passed. + +Valid values: Milliseconds + +### `search.index.error_threshold.reset_interval` + +The amount of time it takes for updates to a given search index to resume/refresh once Riak KV has started skipping update operations; defaults to `30000`. + +Valid values: Milliseconds + +### `search.queue.batch.flush_interval` + +The maximum delay between notification to flush batches to Solr; defaults to `1000` (milliseconds). + +This setting is used to increase or decrease the frequency of batch delivery into Solr, specifically for relatively low-volume input into Riak KV. This setting ensures that data will be delivered into Solr in accordance with the `search.queue.batch.minimum` and `search.queue.batch.maximum` settings within the specified interval. Batches that are smaller than `search.queue.batch.minimum` will be delivered to Solr within this interval. This setting will generally have no effect on heavily loaded systems. You may use any time unit; the default is in milliseconds. + +Valid values: `ms`, `s`, `m`, or `h` + +### `search.queue.batch.maximum` + +The maximum batch size, in number of Riak objects; defaults to `500`. + +Any batches that are larger than this amount will be split, where the first `search.queue.batch.maximum` objects will be flushed to Solr and the remaining objects enqueued for that index will be retained until the next batch is delivered. This parameter ensures that at most `search.queue.batch.maximum` objects will be delivered into Solr in any given request. + +Valid values: Integer + +### `search.queue.batch.minimum` + +The minimum batch size, in number of Riak objects; defaults to `10`. + +Any batches that are smaller than this amount will not be immediately flushed to Solr, but are guaranteed to be flushed within the `search.queue.batch.flush_interval`. + +Valid valus: Integer + +### `search.queue.high_watermark` + +The queue high water mark; defaults to `1000`. + +If the total number of queued messages in a Solrq worker instance exceed this limit, then the calling vnode will be blocked until the total number falls below this limit. This parameter exercises flow control between Riak KV and the Riak Search batching subsystem, if writes into Solr start to fall behind. + +Valid values: Integer + +### `search.queue.high_watermark.purge_strategy` + +The strategy for how purging is handled when the `search.queue.high_watermark` is hit; defaults to `purge_one`. + +Valid values: `purge_one`, `purge_index`, or `off` + +* `purge_one` removes the oldest item on the queue from an erroring (references to fuses blown in the code) index in order to get below the [`search.queue.high_watermark`](#search-queue-high-watermark) +* `purge_index` removes all items associated with one random erroring (references to fuses blown in the code) index in order to get below the [`search.queue.high_watermark`](#search-queue-high-watermark) +* `off` disables purging + +### `search.root_dir` + +The root directory in which index data and configuration is stored; defaults to `./data/yz`. + +Valid values: a directory + +### `search.solr.jvm_options` + +The options to pass to the Solr JVM; defaults to `-d64 -Xms1g -Xmx1g -XX:+UseStringCache -XX:+UseCompressedOops`. + +Non-standard options (e.g. `-XX`) may not be portable across JVM implementations. + +Valid values: Java command-line arguments + +### `search.solr.jmx_port` + +The port number to which Solr JMX binds (note: binds on every interface); defaults to `8985`. + +Valid values: Integer + +NB JMX ceased being a Riak feature in Riak KV 2.2.6. This setting is left here for reference but no longer affects anything. + +### `search.solr.port` + +The port number to which Solr binds (note: binds on every interface); defaults to `8093`. + +Valid values: Integer + +### `search.solr.start_timeout` + +How long Riak KV will wait for Solr to start (attempts twice before shutdown); defaults to `30s`. + +Values lower than 1s will be rounded up to 1s. + +Valid values: Integer with time units (e.g. 2m) + + +## More on Solr +### Solr JVM and Ports + +Riak Search runs one Solr process per node to manage its indexing and +search functionality. While the underlying project manages +index distribution, node coverage for queries, active anti-entropy +(AAE), and JVM process management, you should provide plenty of RAM and diskspace for running both Riak and the JVM running Solr. We recommend a minimum of 6GB of RAM per node. + +Concerning ports, be sure to take the necessary [security][security index] precautions to prevent exposing the extra Solr ports +to the outside world. + +### Solr for Operators + +For further information on Solr monitoring, tuning, and performance, we +recommend the following documents for getting started: + +* [Solr Monitoring](https://wiki.apache.org/solr/SolrMonitoring) +* [Solr Performance + Factors](https://wiki.apache.org/solr/SolrPerformanceFactors) +* [Solr Performance + Problems](https://wiki.apache.org/solr/SolrPerformanceProblems) +* [JConsole](http://docs.oracle.com/javase/7/docs/technotes/guides/management/jconsole.html) + +A wide variety of other documentation is available from the Solr OSS +community. diff --git a/content/riak/kv/2.2.6/configuring/strong-consistency.md b/content/riak/kv/2.2.6/configuring/strong-consistency.md new file mode 100644 index 0000000000..6c0414124a --- /dev/null +++ b/content/riak/kv/2.2.6/configuring/strong-consistency.md @@ -0,0 +1,666 @@ +--- +title: "Implementing Strong Consistency" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Implementing Strong Consistency" + identifier: "configuring_strong_consistency" + weight: 190 + parent: "configuring" +toc: true +--- + +[apps strong consistency]: {{}}riak/kv/2.2.6/developing/app-guide/strong-consistency +[concept strong consistency]: {{}}riak/kv/2.2.6/using/reference/strong-consistency +[cluster ops add remove node]: {{}}riak/kv/2.2.6/using/cluster-operations/adding-removing-nodes +[config reference#strong-cons]: {{}}riak/kv/2.2.6/configuring/reference/#strong-consistency +[use admin riak cli]: {{}}riak/kv/2.2.6/using/admin/riak-cli +[concept eventual consistency]: {{}}riak/kv/2.2.6/learn/concepts/eventual-consistency +[plan backend bitcask]: {{}}riak/kv/2.2.6/setup/planning/backend/bitcask +[glossary vnode]: {{}}riak/kv/2.2.6/learn/glossary/#vnode +[concept buckets]: {{}}riak/kv/2.2.6/learn/concepts/buckets +[cluster ops bucket types]: {{}}riak/kv/2.2.6/using/cluster-operations/bucket-types +[use admin riak-admin#ensemble]: {{}}riak/kv/2.2.6/using/admin/riak-admin/#riak-admin-ensemble-status +[use admin riak-admin]: {{}}riak/kv/2.2.6/using/admin/riak-admin +[config reference#advanced]: {{}}riak/kv/2.2.6/configuring/reference/#advanced-configuration +[plan cluster capacity]: {{}}riak/kv/2.2.6/setup/planning/cluster-capacity +[cluster ops strong consistency]: {{}}riak/kv/2.2.6/using/cluster-operations/strong-consistency +[apps replication properties]: {{}}riak/kv/2.2.6/developing/app-guide/replication-properties +[concept causal context]: {{}}riak/kv/2.2.6/learn/concepts/causal-context +[dev data types]: {{}}riak/kv/2.2.6/developing/data-types +[glossary aae]: {{}}riak/kv/2.2.6/learn/glossary/#active-anti-entropy-aae +[cluster ops 2i]: {{}}riak/kv/2.2.6/using/reference/secondary-indexes +[usage commit hooks]: {{}}riak/kv/2.2.6/developing/usage/commit-hooks +[cluster ops obj del]: {{}}riak/kv/2.2.6/using/reference/object-deletion +[dev client libraries]: {{}}riak/kv/2.2.6/developing/client-libraries + +> **Please Note:** +> +> Riak KV's strong consistency is an experimental feature and may be removed from the product in the future. Strong consistency is not commercially supported or production-ready. Strong consistency is incompatible with Multi-Datacenter Replication, Riak Search, Bitcask Expiration, LevelDB Secondary Indexes, Riak Data Types and Commit Hooks. We do not recommend its usage in any production environment. + +This document provides information on configuring and monitoring a Riak +cluster's optional strong consistency subsystem. Documentation for +developers building applications using Riak's strong consistency feature +can be found in [Using Strong Consistency][apps strong consistency], while a more theoretical +treatment can be found in [Strong Consistency][concept strong consistency]. + +## Minimum Cluster Size + +In order to use strong consistency in Riak, **your cluster must consist +of at least three nodes**. If it does not, all strongly consistent +operations will fail. If your cluster is smaller than three nodes, you +will need to [add more nodes][cluster ops add remove node] and make sure +that strong consistency is [enabled](#enabling-strong-consistency) on all of them. + +Strongly consistent operations on a given key may also fail if a +majority of object replicas in a given ensemble are unavailable, whether +due to slowness, crashes, or network partitions. This means that you may +see strongly consistent operations fail even if the minimum cluster size +requirement has been met. More information on ensembles can be found in +[Implementation Details](#implementation-details). + +While strong consistency requires at least three nodes, we have a +variety of recommendations regarding cluster size, which can be found in +[Fault Tolerance](#fault-tolerance). + +## Enabling Strong Consistency + +Strong consistency in Riak is disabled by default. You can enable it in +each node's [configuration files][config reference#strong-cons]. + +```riakconf +strong_consistency = on +``` + +```appconfig +%% In the older, app.config-based system, the strong consistency +%% parameter is enable_consensus: + +{riak_core, [ + % ... + {enable_consensus, true}, + % ... + ]} +``` + +Remember that you must [restart your node][use admin riak cli] for +configuration changes to take effect. + +For strong consistency requirements to be applied to specific keys, +those keys must be in [buckets][concept buckets] bearing a bucket type with the +`consistent` property set to `true`. More information can be found in +[Using Bucket Types][cluster ops bucket types]. + +If you enable strong consistency on all nodes in a cluster with fewer +than three nodes, strong consistency will be **enabled** but not yet +**active**. Strongly consistent operations are not possible in this +state. Once at least three nodes with strong consistency enabled are +detected in the cluster, the system will be activated and ready for use. +You can check on the status of the strong consistency subsystem using +the [`riak-admin ensemble-status`][use admin riak-admin#ensemble] command. + +## Fault Tolerance + +Strongly consistent operations in Riak are necessarily less highly +available than [eventually consistent][concept eventual consistency] operations +because strongly consistent operations can only succeed if a **quorum** +of object replicas are currently reachable. A quorum can be expressed as +N / 2 + 1 (or `n_val` / 2 + 1), meaning that 3 replicas constitutes a +quorum if N=5, 4 replicas if N=7, etc. If N=7 and 4 replicas are +unavailable, for example, no strongly consistent operations on that +object can succeed. + +While Riak uses N=3 by default, bear in mind that **higher values of N +will allow for more fault tolerance**. The table below shows the number +of allowable missing replicas for assorted values of N: + +Replicas | Allowable missing replicas +:--------|:-------------------------- +3 | 1 +5 | 2 +7 | 3 +9 | 4 +15 | 7 + +Thus, we recommend setting `n_val` higher than the default of 3 for +strongly consistent operations. More on `n_val` in the section below. + +### n_val Recommendations + +Due to the quorum requirements explained above, we recommend that you +use _at least_ N=5 for strongly consistent data. You can set the value +of N, i.e. `n_val`, for buckets +[using bucket types][cluster ops bucket types]. For example, you +can create and activate a bucket type with N set to 5 and strong +consistency enabled---we'll call the bucket type +`consistent_and_fault_tolerant`---using the following series of +[commands][use admin riak-admin]: + +```bash +riak-admin bucket-type create consistent_and_fault_tolerant \ + '{"props": {"consistent":true,"n_val":5}}' +riak-admin bucket-type activate consistent_and_fault_tolerant +``` + +If the `activate` command outputs `consistent_and_fault_tolerant has +been activated`, the bucket type is now ready to provide strong +consistency guarantees. + +#### Setting the target_n_val parameter + +The `target_n_val` parameter sets the highest `n_val` that you intend to +use in an entire cluster. The purpose of this parameter is to ensure +that so-called "hot spots" don't occur, i.e. that data is never stored +more than once on the same physical node. This can happen when: + +* `target_n_val` is greater than the number of physical nodes, or +* the `n_val` for a bucket is greater than `target_n_val`. + +A problem to be aware of if you're using strong consistency is that the +default for `target_n_val` is 4, while our suggested minimum `n_val` for +strongly consistent bucket types is 5. This means that you will need to +raise `target_n_val` if you intend to use an `n_val` over 4 for _any_ +bucket type in your cluster. If you anticipate using an `n_val` of 7 as +the largest `n_val` within your cluster, for example, you will need to +set `target_n_val` to 7. + +This setting is not contained in `riak.conf`, and must instead be set in +the `advanced.config` file. For more information, see our documentation +on [advanced configuration][config reference#advanced]. + +If you are using strong consistency in a cluster that has already been +created with a `target_n_val` that is too low (remember that the default +is too low), you will need to raise it to the desired higher value and +restart each node. + +#### Note on Bucket Properties + +The `consistent` bucket property is one of two bucket properties, +alongside [`datatype`][cluster ops bucket types], that cannot be changed once a +bucket type has been created. + +Furthermore, if `consistent` is set to `true` for a bucket type, you +cannot change the `n_val` for the bucket type once it's been created. If +you attempt to do so, you'll see the following error: + +``` +Error updating bucket : +n_val cannot be modified for existing consistent type +``` + +If you've created a bucket type with a specific `n_val` and wish to +change it, you will need to create a new bucket type with the +appropriate `n_val` and use the new bucket type instead. + +### Fault Tolerance and Cluster Size + +From the standpoint of strongly consistent operations, larger clusters +tend to be more fault tolerant. Spreading ensembles across more nodes will decrease the number of ensembles active on each node and thus decrease the number of quorums affected when a node goes down. + +Imagine a 3-node cluster in which all ensembles are N=3 ensembles. If +two nodes go down, _all_ ensembles will lose quorum and will be unable +to function. Strongly consistent operations on the entire keyspace will +fail until at least one node is brought back online. And even when that +one node is brought back online, a significant portion of the keyspace +will continue to be unavailable for strongly consistent operations. + +For the sake of contrast, imagine a 50-node cluster in which all +ensembles are N=5 (i.e. all objects are replicated to five nodes). In +this cluster, each node is involved in only 10% of the total ensembles; +if a single node fails, that failure will thus impact only 10% of +ensembles. In addition, because N is set to 5, that will not impact +quorum for _any_ ensemble in the cluster; two additional node failures +would need to occur for quorum to be lost for _any_ ensemble. And even +in the case of three nodes failing, it is highly unlikely that that +failure would impact the same ensembles; if it did, only those ensembles +would become unavailable, affecting only 10% of the key space, as +opposed to 100% in the example of a 3-node cluster consisting of N=3 +ensembles. + +These examples illustrate why we recommend higher values for N---again, +at least N=5---as well as clusters with many nodes. The 50-node cluster +example above is used only to illustrate why larger clusters are more +fault tolerant. The definition of "many" nodes will vary according to your needs. +For recommendations regarding cluster size, see [Cluster Capacity Planning][plan cluster capacity]. + +### Offline Node Recommendations + +In general, strongly consistent Riak is more sensitive to the number of +nodes in the cluster than eventually consistent Riak, due to the quorum +requirements described above. While Riak is designed to withstand a +variety of failure scenarios that make nodes in the cluster unreachable, +such as hardware or network failure, **we nonetheless recommend that you +limit the number of nodes that you intentionally down or reboot**. +Having multiple nodes leave the cluster at once can threaten quorum and +thus affect the viability of some or all strongly consistent operations, +depending on the size of the cluster. + +If you're using strong consistency and you do need to reboot multiple +nodes, we recommend rebooting them very carefully. Rebooting nodes too +quickly in succession can force the cluster to lose quorum and thus be +unable to service strongly consistent operations. The best strategy is +to reboot nodes one at a time and wait for each node to rejoin existing +[ensembles][cluster ops strong consistency] before +continuing to the next node. At any point in time, the state of +currently existing ensembles can be checked using [`riak-admin ensemble-status`][admin riak-admin#ensemble]. + +## Performance + +If you run into performance issues, bear in mind that the key space in a +Riak cluster is spread across multiple [consensus groups][cluster ops strong consistency], each of which manages a portion of +that key space. Larger [ring sizes][concept clusters] allow more +independent consensus groups to exist in a cluster, which can provide +for more concurrency and higher throughput, and thus better performance. +The ideal ring size, however, will also depend on the number of nodes in +the cluster. General recommendations can be found in [Cluster Capacity Planning][plan cluster capacity]. + +Adding nodes to your cluster is another means of enhancing the +performance of strongly consistent operations. Instructions on doing so +can be found in [Adding and Removing Nodes][cluster ops add remove node]. + +Your cluster's configuration can also affect strong consistency +performance. See the section on [configuration][config reference#strong-cons] below. + +## riak-admin ensemble-status + +The [`riak-admin`][use admin riak-admin] interface +used for general node/cluster management has an `ensemble-status` +command that provides insight into the current status of the consensus +subsystem undergirding strong consistency. + +Running the command by itself will provide the current state of the +subsystem: + +```bash +riak-admin ensemble-status +``` + +If strong consistency is not currently enabled, you will see `Note: The +consensus subsystem is not enabled.` in the output of the command; if +strong consistency is enabled, you will see output like this: + +``` +============================== Consensus System =============================== +Enabled: true +Active: true +Ring Ready: true +Validation: strong (trusted majority required) +Metadata: best-effort replication (asynchronous) + +================================== Ensembles ================================== + Ensemble Quorum Nodes Leader +------------------------------------------------------------------------------- + root 4 / 4 4 / 4 riak@riak1 + 2 3 / 3 3 / 3 riak@riak2 + 3 3 / 3 3 / 3 riak@riak4 + 4 3 / 3 3 / 3 riak@riak1 + 5 3 / 3 3 / 3 riak@riak2 + 6 3 / 3 3 / 3 riak@riak2 + 7 3 / 3 3 / 3 riak@riak4 + 8 3 / 3 3 / 3 riak@riak4 +``` + +### Interpreting ensemble-status Output + +The following table provides a guide to `ensemble-status` output: + +Item | Meaning +:----|:------- +`Enabled` | Whether the consensus subsystem is enabled on the current node, i.e. whether the `strong_consistency` parameter in [`riak.conf`][config reference#strong-cons] has been set to `on`. If this reads `off` and you wish to enable strong consistency, see our documentation on enabling strong consistency. +`Active` | Whether the consensus subsystem is active, i.e. whether there are enough nodes in the cluster to use strong consistency, which requires at least three nodes. +`Ring Ready` | If `true`, then all of the [vnodes][glossary vnode] in the cluster have seen the current ring, which means that the strong consistency subsystem can be used; if `false`, then the system is not yet ready. If you have recently added or removed one or more nodes to/from the cluster, it may take some time for `Ring Ready` to change. +`Validation` | This will display `strong` if the `tree_validation` setting in riak.conf has been set to `on` and `weak` if set to `off`. +`Metadata` | This depends on the value of the `synchronous_tree_updates` setting in riak.conf, which determines whether strong consistency-related Merkle trees are updated synchronously or asynchronously. If `best-effort replication (asynchronous)`, then `synchronous_tree_updates` is set to `false`; if `guaranteed replication (synchronous)` then `synchronous_tree_updates` is set to `true`. +`Ensembles` | This displays a list of all of the currently existing ensembles active in the cluster.
  • Ensemble --- The ID of the ensemble
  • Quorum --- The number of ensemble peers that are either leading or following
  • Nodes --- The number of nodes currently online
  • Leader --- The current leader node for the ensemble
+ +**Note**: The **root ensemble**, designated by `root` in the sample +output above, is a special ensemble that stores a list of nodes and +ensembles in the cluster. + +More in-depth information on ensembles can be found in our [internal +documentation](https://github.com/basho/riak_ensemble/blob/develop/doc/Readme.md). + +### Inspecting Specific Ensembles + +The `ensemble-status` command also enables you to directly inspect the +status of specific ensembles in a cluster. The IDs for all current +ensembles are displayed in the `Ensembles` section of the +`ensemble-status` output described above. + +To inspect a specific ensemble, specify the ID: + +```bash +riak-admin ensemble-status +``` + +The following would inspect ensemble 2: + +```bash +riak-admin ensemble-status 2 +``` + +Below is sample output for a single ensemble: + +``` +================================= Ensemble #2 ================================= +Id: {kv,0,3} +Leader: riak@riak2 (2) +Leader ready: true + +==================================== Peers ==================================== + Peer Status Trusted Epoch Node +------------------------------------------------------------------------------- + 1 following yes 1 riak@riak1 + 2 leading yes 1 riak@riak2 + 3 following yes 1 riak@riak3 +``` + +The table below provides a guide to the output: + +Item | Meaning +:----|:------- +`Id` | The ID for the ensemble used internally by Riak, expressed as a 3-tuple. All ensembles are `kv`; the second element names the ring partition for which the ensemble is responsible; and the third element is the `n_val` for the keys for which the ensemble is responsible. +`Leader` | Identifies the ensemble's leader. In this case, the leader is on node `riak@riak2` and is identified as peer `2` in the ensemble. +`Leader ready` | States whether the ensemble's leader is ready to respond to requests. If not, requests to the ensemble will fail. +`Peers` | A list of peer [vnodes][glossary vnode] associated with the ensemble.
  • Peer --- The ID of the peer
  • Status --- Whether the peer is a leader or a follower
  • Trusted --- Whether the peer's Merkle tree is currently considered trusted or not
  • Epoch --- The current consensus epoch for the peer. The epoch is incremented each time the leader changes.
  • Node --- The node on which the peer resides.
+ +More information on leaders, peers, Merkle trees, and other details can +be found in [Implementation Details](#implementation-details) below. + +## Implementation Details + +Strong consistency in Riak is handled by a subsystem called +[`riak_ensemble`](https://github.com/basho/riak_ensemble/blob/develop/doc/Readme.md) +This system functions differently from other systems in Riak in a number +of ways, and many of these differences are important to bear in mind for +operators configuring their cluster's usage of strong consistency. + +### Basic Operations + +The first major difference is that strongly consistent Riak involves a +different set of operations from [eventually consistent][concept eventual consistency] Riak KV. In strongly consistent buckets, there are four types +of atomic operations on objects: + +* **Get** operations work just as they do against + non-strongly-consistent keys, but with two crucial differences: + 1. Connecting clients are guaranteed to return the most recently + written value (which makes those operations CP, i.e. consistent and + partition tolerant) + 2. Reads on strongly consistent keys *never* return siblings, hence + there is no need to develop any sort of [conflict resolution][usage conflict resolution] + strategy for those keys +* **Conditional put** operations write an object only if no object + currently exists in that key. The operation will fail if the key + already exists; if the key was never written or has been deleted, the + operation succeeds. +* **Conditional modify** operations are compare-and-swap (CAS) + operations that succeed only if the value of a key has not changed + since it was previously read. +* **Delete** operations work mostly like they do against + non-strongly-consistent keys, with the exception that + [tombstones][cluster ops obj deletion] are not harvested, which is + the equivalent of having `delete_mode` set to `keep`. + +**From the standpoint of clients connecting to Riak, there is little +difference between strongly and non-strongly consistent data**. The +operations performed on objects---reads, writes, deletes, etc.---are the +same, which means that the client API for strong consistency is +essentially the same as it is for eventually consistent operations, with +the important exception of error handling. + +### Ensembles + +The main actors in Riak's implementation of strong consistency are +**ensembles**, which are independent groups that watch over a portion of +a Riak cluster's key space and coordinate strongly consistent operations +across nodes. When watching over a given key space, ensembles must act +upon multiple replicas of a given object, the number of which is +specified by `n_val` (more on this in [Replication Properties][apps replication properties]). + +Eventually consistent Riak can service requests even when only a single +object replica is available, using mechanisms like [vector clocks][concept causal context] and [dotted version vectors][concept causal context]---or, in a different way, [Riak Data Types][dev data types])---to ensure eventual consistency between replicas. Strongly consistent Riak is different because it +requires that a **quorum** of object replicas be online and reachable, +where a quorum is defined as `n_val` / 2 + 1. **If a quorum is not +available for a key, all strongly consistent operations against that key +will fail**. + +More information can be found in the section on Fault Tolerance above. + +### Peers, Leaders, Followers, and Workers + +All ensembles in strongly consistent Riak consist of agents called +**peers**. The number of peers in an ensemble is defined by the `n_val` +of that ensemble, i.e. the number of object replicas that the +ensemble watches over. Amongst the peers in the ensemble, there are two +basic actors: **leaders** and **followers**. + +Leaders and followers coordinate with one another on most requests. +While leaders and followers coordinate on all writes, i.e. all puts and +deletes, you can enable leaders to respond to gets without the need to +coordinate with followers. This is known as granting a **leader lease**. +Leader leases are enabled by default, and are disabled (or re-enabled) +at the cluster level. A more in-depth account of ensemble behavior can +be found in our [internal +documentation](https://github.com/basho/riak_ensemble/tree/develop/doc). + +In addition to leaders and followers, ensemble peers use lightweight +Erlang processes called **workers** to perform long-running K/V +operations, allowing peers to remain responsive to requests. The number +of workers assigned to each peer depends on your configuration. + +These terms should be borne in mind in the sections on configuration +below. + +### Integrity Checking + +An essential part of implementing a strong consistency subsystem in a +distributed system is **integrity checking**, which is a process that +guards against data corruption and inconsistency even in the face of +network partitions and other adverse events that Riak was built to +handle gracefully. + +Like Riak's [active anti-entropy][glossary aae] subsystem, strong consistency +integrity checking utilizes [Merkle +trees](http://en.wikipedia.org/wiki/Merkle_tree) that are persisted on +disk. All peers in an ensemble, i.e. all leaders and followers, maintain +their own Merkle trees and update those trees in the event of most +strongly consistent operations. Those updates can occur synchronously or +asynchronously from the standpoint of client operations, depending on +the configuration that you specify. + +While integrity checking takes place automatically in Riak, there are +important aspects of its behavior that you can configure. See the Merkle Tree settings section below for more +information on configurable parameters. + +## Configuring Strong Consistency + +The `riak_ensemble` subsystem provides a wide variety of tunable +parameters that you can adjust to fit the needs of your Riak cluster. +All `riak_ensemble`-specific parameters, with the exception of the +`strong_consistency` parameter used to [enable strong consistency](#enabling-strong-consistency), +must be set in each node's `advanced.config` file, _not_ in `riak.conf` +or `app.config`. + +Information on the syntax and usage of `advanced.config` can be found in +our documentation on [advanced configuration][config reference#advanced]. That same document also contains a full +listing of [strong-consistency-related configuration parameters][config reference#strong-cons]. + +Please note that the sections below require a basic understanding of the +following terms: + +* ensemble +* peer +* leader +* follower +* worker +* integrity checking +* Merkle tree + +For an explanation of these terms, see the [Implementation Details](#implementation-details) section +above. + +#### Leader Behavior + +The `trust_lease` setting determines whether leader leases are used to +optimize reads. When set to `true`, a leader with a valid lease can +handle reads directly without needing to contact any followers. When +`false`, the leader will always contact followers, which can lead to +degraded read performance. The default is `true`. We recommend leaving +leader leases enabled for performance reasons. + +All leaders have periodic duties that they perform, including refreshing +the leader lease. You can determine how frequently this occurs, in +milliseconds, using the `ensemble_tick` setting. The default is 500 +milliseconds. Please note that this setting must be lower than both +the `lease_duration` and `follower_timeout` settings (both explained +below). + +If you set `trust_lease` to `true`, you can also specify how long a +leader lease remains valid without being refreshed using the +`lease_duration` setting, which is specified in milliseconds. This +setting should be higher than `ensemble_tick` to ensure that leaders +have to time to refresh their leases before they time out, and it _must_ +be lower than `follower_timeout`, explained in the section below. The +default is `ensemble_tick` * 3/2, i.e. if `ensemble_tick` is 400, +`lease_duration` will default to 600. + +#### Worker Settings + +You can choose how many workers are assigned to each peer using the +`peer_workers` setting. Workers are lightweight processes spawned by +leaders and followers. While increasing the number of workers will make +the strong consistency subsystem slightly more computationally +expensive, more workers can mean improved performance in some cases, +depending on the workload. The default is 1. + +### Timeouts + +You can establish timeouts for both reads and writes (puts and deletes) +using the `peer_get_timeout` and `peer_put_timeout` settings, +respectively. Both are expressed in milliseconds and default to 60000 +(1 minute). + +Longer timeouts will decrease the likelihood that read or write +operations will fail due to long computation times; shorter timeouts +entail shorter wait times for connecting clients, but at a higher risk +of failed operations under heavy load. + +### Merkle Tree Settings + + +Leaders and followers in Riak's strong consistency system maintain +persistent [Merkle trees](http://en.wikipedia.org/wiki/Merkle_tree) for +all data stored by that peer. More information can be found in the +**Integrity Checking** section above. The two sections directly below +describe Merkle-tree-related parameters. + +#### Tree Validation + +The `tree_validation` parameter determines whether Riak considers Merkle +trees to be trusted after peers are restarted (for whatever reason). +When enabled, i.e. when `tree_validation` is set to `true` (the +default), Riak does not trust peer trees after a restart, instead +requiring the peer to sync with a trusted quorum. While this is the +safest mode because it protects Riak against silent corruption in Merkle +trees, it carries the drawback that it can reduce Riak availability by +requiring more than a simple majority of nodes to be online and +reachable when peers restart. + +If you are using ensembles with N=3, we strongly recommend setting +`tree_validation` to `false`. + +#### Synchronous vs. Asynchronous Tree Updates + +Merkle tree updates can happen synchronously or asynchronously. This is +determined by the `synchronous_tree_updates` parameter. When set to +`false`, which is the default, Riak responds to the client after the +first roundtrip that updates the followers' data but before the second +roundtrip required to update the followers' Merkle trees, allowing the +Merkle tree update to happen asynchronously in the background; when set +to `true`, Riak requires two quorum roundtrips to occur before replying +back to the client, which can increase per-request latency. + +Please note that this setting applies only to Merkle tree updates sent +to followers. Leaders _always_ update their local Merkle trees before +responding to the client. Asynchronous updates can be unsafe in certain +scenarios. For example, if a leader crashes before sending metadata +updates to followers _and_ all followers that had acknowledged the write +somehow revert the object value immediately prior to the write request, +a future read could hypothetically return the immediately preceding +value without realizing that the value was incorrect. Setting +`synchronous_tree_updates` to `false` does bear this possibility, but it +is highly unlikely. + +## Strong Consistency and Active Anti-Entropy + +Riak's [active anti-entropy][glossary aae] \(AAE) feature _can_ repair strongly +consistent data. Although it is not necessary to use active anti-entropy +if you are using strong consistency, we nonetheless recommend doing so. + +Without AAE, all object conflicts are repaired via read repair. +Read repair, however, cannot repair conflicts in so-called "cold data," +i.e. data that may not be read for long periods of time. While using AAE +does entail small performance losses, not using AAE can lead to problems +with silent on-disk corruption. + +## Strong Consistency and Bitcask + +One feature that is offered by Riak's optional [Bitcask][plan backend bitcask] backend is object expiry. If you are using strong consistency and Bitcask together, you should be aware that object metadata is often updated by the strong consistency subsystem during leader changes, which typically take place when nodes go down or during network partitions. When these metadata updates take place, the time to live (TTL) of the object is refreshed, which can lead to general unpredictably in objects' TTL. Although leader changes will be rare in many clusters, we nonetheless recommend that you use object expiry in +strongly consistent buckets only in situations when these occasional +irregularities are acceptable. + +## Important Caveats + +The following Riak features are not currently available in strongly +consistent buckets: + +* [Secondary indexes][cluster ops 2i] --- If you do attach + secondary index metadata to objects in strongly consistent buckets, + strongly consistent operations can still proceed, but that metadata + will be silently ignored. +* [Riak Data Types][dev data types] --- Data Types can currently be + used only in an eventually consistent fashion +* [Using commit hooks][usage commit hooks] --- Neither pre- nor post-commit hooks are supported in strongly consistent buckets. If you do associate a + strongly consistent bucket with one or more commit hooks, strongly + consistent operations can proceed as normal in that bucket, but all + commit hooks will be silently ignored. + +Furthermore, you should also be aware that strong consistency guarantees +are applied only at the level of single keys. There is currently no +support within Riak for strongly consistent operations against multiple +keys, although it is always possible to incorporate client-side write +and read locks in applications that use strong consistency. + +## Known Issues + +There are a few known issues that you should be aware of when using the +latest version of strong consistency. + +* **Consistent reads of never-written keys create tombstones** --- A + [tombstone][cluster ops obj del] will be written if you perform a read + against a key that a majority of peers claims to not exist. This is + necessary for certain corner cases in which offline or unreachable + replicas containing partially written data need to be rolled back in + the future. +* **Consistent keys and key listing** --- In Riak, key listing + operations, such as listing all the keys in a bucket, do not filter + out tombstones. While this is rarely a problem for + non-strongly-consistent keys, it does present an issue for strong + consistency due to the tombstone issues mentioned above. +* **Secondary indexes not supported** --- Strongly consistent + operations do not support [secondary indexes][cluster ops 2i] \(2i) at this time. Furthermore, any other metadata + attached to objects, even if not related to 2i, will be silently + ignored by Riak in strongly consistent buckets. +* **Multi-Datacenter Replication not supported** --- At this time, + consistent keys are *not* replicated across clusters using Multi- + Datacenter Replication \(MDC). This is because MDC Replication currently supports only eventually consistent replication across clusters. Mixing strongly + consistent data within a cluster with eventually consistent data + between clusters is difficult to reason about from the perspective of + applications. In a future version of Riak, we will add support for + strongly consistent replication across multiple datacenters/clusters. +* **Client library exceptions** --- Basho's official [client + libraries][dev client libraries] convert errors returned by Riak into generic exceptions, + with a message derived from the returned server-side error message. diff --git a/content/riak/kv/2.2.6/configuring/v2-multi-datacenter.md b/content/riak/kv/2.2.6/configuring/v2-multi-datacenter.md new file mode 100644 index 0000000000..8896c4fd1c --- /dev/null +++ b/content/riak/kv/2.2.6/configuring/v2-multi-datacenter.md @@ -0,0 +1,156 @@ +--- +title_supertext: "Configuring:" +title: "V2 Multi-Datacenter Replication" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "V2 Multi-Datacenter" + identifier: "configuring_v2" + weight: 210 + parent: "configuring" +toc: true +commercial_offering: true +aliases: + - /riak-docs/riak/2.2.6/ops/mdc/v2/configuration + - /riak-docs/riak/kv/2.2.6/ops/mdc/v2/configuration +--- + +[config v2 ssl]: {{}}riak/kv/2.2.6/configuring/v2-multi-datacenter/ssl + +{{% note title="Deprecation Warning" %}} +v2 Multi-Datacenter Replication is deprecated and will be removed in a future version. Please use [v3]({{}}riak/kv/2.2.6/configuring/v3-multi-datacenter/) instead. +{{% /note %}} + +Riak's Multi-Datacenter Replication capabilities offer a +variety of configurable parameters. + +## File + +The configuration for replication is kept in the `riak_repl` section of +each node's `advanced.config`. That section looks like this: + +```advancedconfig +{riak_repl, [ + {fullsync_on_connect, true}, + {fullsync_interval, 360}, + % Debian/Centos/RHEL: + {data_root, "/var/lib/riak/data/riak_repl"}, + % Solaris: + % {data_root, "/opt/riak/data/riak_repl"}, + % FreeBSD/SmartOS: + % {data_root, "/var/db/riak/riak_repl"}, + {queue_size, 104857600}, + {server_max_pending, 5}, + {client_ack_frequency, 5} + ]} +``` + +## Usage + +These settings are configured using the standard Erlang config file +syntax, i.e. `{Setting, Value}`. For example, if you wished to set +`ssl_enabled` to `true`, you would insert the following line into the +`riak_repl` section (appending a comma if you have more settings to +follow): + +```advancedconfig +{riak_repl, [ + % Other configs + {ssl_enabled, true}, + % Other configs + ]} +``` + +## Settings + +Once your configuration is set, you can verify its correctness by +running the following command: + +```bash +riak chkconfig +``` + +The output from this command will point you to syntactical and other +errors in your configuration files. + +A full list of configurable parameters can be found in the sections +below. + +## Fullsync Settings + +Setting | Options | Default | Description +:-------|:--------|:--------|:----------- +`fullsync_on_connect` | `true`, `false` | `true` | Whether or not to initiate a fullsync on initial connection from the secondary cluster +`fullsync_strategies` | `keylist` | `[keylist]` | A *list* of fullsync strategies to be used by replication.
**Note**: Please contact Basho support for more information. +`fullsync_interval` | `mins` (integer), `disabled` | `360` | How often to initiate a fullsync of data, in minutes. This is measured from the completion of one fullsync operation to the initiation of the next. This setting only applies to the primary cluster (listener). To disable fullsync, set `fullsync_interval` to `disabled` and `fullsync_on_connect` to `false`.** + +## SSL Settings + +Setting | Options | Default | Description +:-------|:--------|:--------|:----------- +`ssl_enabled` | `true`, `false` | `false` | Enable SSL communications +`keyfile` | `path` (string) | `undefined` | Fully qualified path to an SSL `.pem` key file +`cacertdir` | `path` (string) | `undefined` | The `cacertdir` is a fully-qualified directory containing all the CA certificates needed to verify the CA chain back to the root +`certfile` | `path` (string) | `undefined` | Fully qualified path to a `.pem` cert file +`ssl_depth` | `depth` (integer) | `1` | Set the depth to check for SSL CA certs. See [1](#f1). +`peer_common_name_acl` | `cert` (string) | `"*"` | Verify an SSL peer’s certificate common name. You can provide an ACL as a list of common name *patterns*, and you can wildcard the leftmost part of any of the patterns, so `*.basho.com` would match `site3.basho.com` but not `foo.bar.basho.com` or `basho.com`. See [4](#f4). + +## Queue, Object, and Batch Settings + +Setting | Options | Default | Description +:-------|:--------|:--------|:----------- +`data_root` | `path` (string) | `data/riak_repl` | Path (relative or absolute) to the working directory for the replication process +`queue_size` | `bytes` (integer) | `104857600` (100 MiB) | The size of the replication queue in bytes before the replication leader will drop requests. If requests are dropped, a fullsync will be required. Information about dropped requests is available using the `riak-repl status` command +`server_max_pending` | `max` (integer) | `5` | The maximum number of objects the leader will wait to get an acknowledgment from, from the remote location, before queuing the request +`vnode_gets` | `true`, `false` | `true` | If `true`, repl will do a direct get against the vnode, rather than use a `GET` finite state machine +`shuffle_ring` | `true`, `false` | `true `| If `true`, the ring is shuffled randomly. If `false`, the ring is traversed in order. Useful when a sync is restarted to reduce the chance of syncing the same partitions. +`diff_batch_size` | `objects` (integer) | `100` | Defines how many fullsync objects to send before waiting for an acknowledgment from the client site + +## Client Settings + +Setting | Options | Default | Description +:-------|:--------|:--------|:----------- +`client_ack_frequency` | `freq` (integer) | `5` | The number of requests a leader will handle before sending an acknowledgment to the remote cluster +`client_connect_timeout` | `ms` (integer) | `15000` | The number of milliseconds to wait before a client connection timeout occurs +`client_retry_timeout` | `ms` (integer) | `30000` | The number of milliseconds to wait before trying to connect after a retry has occurred + +## Buffer Settings + +Setting | Options | Default | Description +:-------|:--------|:--------|:----------- +`sndbuf` | `bytes` (integer) | OS dependent | The buffer size for the listener (server) socket measured in bytes +`recbuf` | `bytes` (integer) | OS dependent | The buffer size for the site (client) socket measured in bytes + +## Worker Settings + +Setting | Options | Default | Description +:-------|:--------|:--------|:----------- +`max_get_workers` | `max` (integer) | `100` | The maximum number of get workers spawned for fullsync. Every time a replication difference is found, a `GET` will be performed to get the actual object to send. See [2](#f2). +`max_put_workers` | `max` (integer) | `100` | The maximum number of put workers spawned for fullsync. Every time a replication difference is found, a `GET` will be performed to get the actual object to send. See [3](#f3). +`min_get_workers` | `min` (integer) | `5` | The minimum number of get workers spawned for fullsync. Every time a replication difference is found, a `GET` will be performed to get the actual object to send. See [2](#f2). +`min_put_workers` | `min` (integer) | `5` | The minimum number of put workers spawned for fullsync. Every time a replication difference is found, a `GET` will be performed to get the actual object to send. See [3](#f3). + + +1. SSL depth is the maximum number of non-self-issued + intermediate certificates that may follow the peer certificate in a valid + certificate chain. If depth is `0`, the PEER must be signed by the trusted + ROOT-CA directly; if `1` the path can be PEER, CA, ROOT-CA; if depth is `2` + then PEER, CA, CA, ROOT-CA and so on. + +2. Each get worker spawns 2 processes, one for the work and + one for the get FSM (an Erlang finite state machine implementation for `GET` + requests). Be sure that you don't run over the maximum number of allowed + processes in an Erlang VM (check `vm.args` for a `+P` property). + +3. Each put worker spawns 2 processes, one for the work, and + one for the put FSM (an Erlang finite state machine implementation for `PUT` + requests). Be sure that you don't run over the maximum number of allowed + processes in an Erlang VM (check `vm.args` for a `+P` property). + +4. If the ACL is specified and not the special value `*`, + peers presenting certificates not matching any of the patterns will not be + allowed to connect. + If no ACLs are configured, no checks on the common name are done, except + as described for [Identical Local and Peer Common Names][config v2 ssl]. diff --git a/content/riak/kv/2.2.6/configuring/v2-multi-datacenter/nat.md b/content/riak/kv/2.2.6/configuring/v2-multi-datacenter/nat.md new file mode 100644 index 0000000000..971530dd20 --- /dev/null +++ b/content/riak/kv/2.2.6/configuring/v2-multi-datacenter/nat.md @@ -0,0 +1,78 @@ +--- +title_supertext: "V2 Multi-Datacenter Replication:" +title: "With NAT" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "With NAT" + identifier: "configuring_v2_replication_nat" + weight: 101 + parent: "configuring_v2" +toc: true +commercial_offering: true +aliases: + - /riak-docs/riak/2.2.6/ops/mdc/v2/nat + - /riak-docs/riak/kv/2.2.6/ops/mdc/v2/nat +--- + +[config v2 ssl]: {{}}riak/kv/2.2.6/configuring/v2-multi-datacenter/ssl + +{{% note title="Deprecation Warning" %}} +v2 Multi-Datacenter Replication is deprecated and will be removed in a future version. Please use [v3]({{}}riak/kv/2.2.6/configuring/v3-multi-datacenter/nat/) instead. +{{% /note %}} + +Riak supports replication of data on networks that use static +NAT. This capability can be used for replicating data over the internet +where servers have both internal and public IP addresses (see [Riak +REPL SSL][config v2 ssl] if you replicate data over a public network). + +## Requirements + +In order for Multi-Datacenter Replication to work on a server configured +with NAT, the NAT addresses must be configured statically. + +## Example + +Imagine the following scenario: + +* Server A is the source of replicated data +* Servers B and C would like to be clients of the replicated data + +Server A is set up with static NAT, configured for IP addresses: + + * `192.168.1.10` (internal) and `50.16.238.123` (public) + +Server A replication will listen on: + + * the internal IP address `192.168.1.10`, port `9010` + * the public IP address `50.16.238.123`, port `9011` + +Server B is set up with a single public IP address: `50.16.238.200` + + * Server B replication will connect as a client to the public IP + address `50.16.238.123`, port `9011` + +Server C is set up with a single internal IP address: `192.168.1.20` + + * Server C replication will connect as a client to the internal IP + address of `192.168.1.10`, port `9010` + +Configure a listener on Server A: + +```bash +riak-repl add-nat-listener riak@192.168.1.10 192.168.1.10 9010 50.16.238.123 9011 +``` + +Configure a site (client) on Server B: + +```bash +riak-repl add-site 50.16.238.123 9011 server_a_to_b +``` + +Configure a site (client) on Server C: + +```bash +riak-repl add-site 192.168.1.10 9010 server_a_to_c +``` diff --git a/content/riak/kv/2.2.6/configuring/v2-multi-datacenter/quick-start.md b/content/riak/kv/2.2.6/configuring/v2-multi-datacenter/quick-start.md new file mode 100644 index 0000000000..e932c8f61e --- /dev/null +++ b/content/riak/kv/2.2.6/configuring/v2-multi-datacenter/quick-start.md @@ -0,0 +1,367 @@ +--- +title_supertext: "V2 Multi-Datacenter Replication:" +title: "Quickstart" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Quickstart" + identifier: "configuring_v2_quickstart" + weight: 100 + parent: "configuring_v2" +toc: true +commercial_offering: true +aliases: + - /riak-docs/riak/2.2.6/ops/mdc/v2/quick-start + - /riak-docs/riak/kv/2.2.6/ops/mdc/v2/quick-start +--- + +{{% note title="Deprecation Warning" %}} +v2 Multi-Datacenter Replication is deprecated and will be removed in a future version. Please use [v3]({{}}riak/kv/2.2.6/configuring/v3-multi-datacenter/quick-start/) instead. +{{% /note %}} + +The Riak Multi-Datacenter Replication Quick Start will walk you through +the process of configuring Riak's version 2 Replication to perform +replication between two sample Riak clusters in separate networks. This +guide will also cover bidirectional replication, which is accomplished +by setting up unidirectional replication in both directions between the +clusters. + +## Prerequisites + +This Guide assumes that you have completed the following steps: + +* [Installing Riak][install index] +* [Performing system tuning|System Performance Tuning][perf index] +* [Reviewing configuration][config v2 mdc] + +## Scenario + +Configure Riak MDC to perform replication, given the following +3-node Riak clusters: + +#### Cluster 1 + +Name | IP | Node name +:-----|:------------|:---------------- +`node1` | `172.16.1.11` | `riak@172.16.1.11` +`node2` | `172.16.1.12` | `riak@172.16.1.12` +`node3` | `172.16.1.13` | `riak@172.16.1.13` + +#### Cluster 2 + +Name | IP | Node name +:-----|-------------|----------------- +`node4` | `192.168.1.21` | `riak@192.168.1.21` +`node5` | `192.168.1.22` | `riak@192.168.1.22` +`node6` | `192.168.1.23` | `riak@192.168.1.23` + +**Note**: The addresses used in these example clusters are contrived, +non-routable addresses. In real-world applications, however, these +addresses would need to be routable over the public Internet. + +## Set Up Cluster1 → Cluster2 Replication + +### Set Up the Listeners on Cluster1 (Source cluster) + +On a node in Cluster1, `node1` for example, identify the nodes that will +be listening to connections from replication clients with `riak-repl +add-listener ` for each node that will be +listening for replication clients. + +```bash +riak-repl add-listener riak@172.16.1.11 172.16.1.11 9010 +riak-repl add-listener riak@172.16.1.12 172.16.1.12 9010 +riak-repl add-listener riak@172.16.1.13 172.16.1.13 9010 +``` + +### Set Up the Site on Cluster2 (Site cluster) + +On a node in Cluster2, `node4` for example, inform the replication +clients where the Source Listeners are located with `riak-repl add-site + `. Use the IP address(es) and port(s) you +configured in the earlier step. For `sitename` enter `Cluster1`. + +```bash +riak-repl add-site 172.16.1.11 9010 Cluster1 +``` + +**Note**: While a Listener needs to be added to each node, only a single +Site needs to be added on the Site cluster. Once connected to the Source +cluster, it will get the locations of the rest of the Listeners in the +Source cluster. + +### Verify the Replication Configuration + +Verify the replication configuration using `riak-repl status` on both a +Cluster1 node and a Cluster2 node. A full description of the `riak-repl +status` command's output can be found in the documentation for +`riak-repl`'s [status output][cluster ops v2 mdc#status]. + +On the Cluster1 node, verify that there are `listener_`s for +each listening node, and that `leader` and `server_stats` are populated. +They should look similar to the following: + +``` +listener_riak@172.16.1.11: "172.16.1.11:9010" +listener_riak@172.16.1.12: "172.16.1.12:9010" +listener_riak@172.16.1.13: "172.16.1.13:9010" +leader: 'riak@172.16.1.11' +server_stats: [{<8051.3939.0>, + {message_queue_len,0}, + {status,[{site,"Cluster2"}, + {strategy,riak_repl_keylist_server}, + {fullsync_worker,<8051.3940.0>}, + {dropped_count,0}, + {queue_length,0}, + {queue_byte_size,0}, + {state,wait_for_partition}]}}] +``` + +On the Cluster2 node, verify that `Cluster1_ips`, `leader`, and +`client_stats` are populated. They should look similar to the following: + +``` +Cluster1_ips: "172.16.1.11:9010, 172.16.1.12:9010, 172.16.1.13:9010" +leader: 'riak@192.168.1.21' +client_stats: [{<8051.3902.0>, + {message_queue_len,0}, + {status,[{site,"Cluster1"}, + {strategy,riak_repl_keylist_client}, + {fullsync_worker,<8051.3909.0>}, + {put_pool_size,5}, + {connected,"172.16.1.11",9010}, + {state,wait_for_fullsync}]}}] +``` + +### Testing Realtime Replication + +That's all there is to it! When `PUT` requests are coordinated by +Cluster1, these operations will be replicated to Cluster2. + +You can use the following example script to verify that `PUT` operations +sent to Cluster1 are being replicated to Cluster2: + +```bash +#!/bin/bash + +VALUE=`date` +CLUSTER_1_IP=172.16.1.11 +CLUSTER_2_IP=192.168.1.21 + +curl -s -X PUT -d "${VALUE}" http://${CLUSTER_1_IP}:8098/riak/replCheck/c1 + +CHECKPUT_C1=`curl -s http://${CLUSTER_1_IP}:8098/riak/replCheck/c1` + +if [ "${VALUE}" = "${CHECKPUT_C1}" ]; then + echo "C1 PUT Successful" +else + echo "C1 PUT Failed" + exit 1 +fi + +CHECKREPL_C1_TO_C2=`curl -s http://${CLUSTER_2_IP}:8098/riak/replCheck/c1` + +if [ "${VALUE}" = "${CHECKREPL_C1_TO_C2}" ]; then + echo "C1 to C2 consistent" +else + echo "C1 to C2 inconsistent + C1:${CHECKPUT_C1} + C2:${CHECKREPL_C1_TO_C2}" + exit 1 +fi + +exit 0 +``` + +You will have to change some of the above variables for your own +environment, such as IP addresses or ports. + +If you run this script and things are working as expected, you will get +the following output: + +``` +C1 PUT Successful +C1 to C2 consistent +``` + +## Set Up Cluster2 → Cluster1 Replication + +### About Bidirectional Replication + +Multi-Datacenter support can also be configured to replicate in both +directions, ensuring eventual consistency between your two datacenters. +Setting up bidirectional replication is as simple as repeating the steps +above in the other direction, i.e. from Cluster2 to Cluster1. + +### Set Up the Listeners on Cluster2 (Source cluster) + +On a node in Cluster2, `node4` for example, identify the nodes that will +be listening to connections from replication clients with `riak-repl +add-listener ` for each node that will be +listening for replication clients. + +```bash +riak-repl add-listener riak@192.168.1.21 192.168.1.21 9010 +riak-repl add-listener riak@192.168.1.22 192.168.1.22 9010 +riak-repl add-listener riak@192.168.1.23 192.168.1.23 9010 +``` + +### Set Up the Site on Cluster1 (Site cluster) + +On a node in Cluster1, `node1` for example, inform the replication +clients where the Source Listeners are with `riak-repl add-site + `. Use the IP address(es) and port(s) you configured in +the earlier step. For `sitename` enter **Cluster2**. + +```bash +riak-repl add-site 192.168.1.21 9010 Cluster2 +``` + +### Verify the Replication Configuration + +Verify the replication configuration using `riak-repl status` on a +Cluster1 node and a Cluster2 node. A full description of the `riak-repl +status` command's output can be found in the documentation for +`riak-repl`'s [status output][cluster ops v2 mdc#status]. + +On the Cluster1 node, verify that `Cluster2_ips`, `leader`, and +`client_stats` are populated. They should look similar to the following: + +``` +Cluster2_ips: "192.168.1.21:9010, 192.168.1.22:9010, 192.168.1.23:9010" +leader: 'riak@172.16.1.11' +client_stats: [{<8051.3902.0>, + {message_queue_len,0}, + {status,[{site,"Cluster2"}, + {strategy,riak_repl_keylist_client}, + {fullsync_worker,<8051.3909.0>}, + {put_pool_size,5}, + {connected,"192.168.1.21",9010}, + {state,wait_for_fullsync}]}}] +``` + +On the Cluster2 node, verify that there are listener entries for each +listening node, and that `leader` and `server_stats` are populated. They +should look similar to the following: + +``` +listener_riak@192.168.1.21: "192.168.1.21:9010" +listener_riak@192.168.1.22: "192.168.1.22:9010" +listener_riak@192.168.1.23: "192.168.1.23:9010" +leader: 'riak@192.168.1.21' +server_stats: [{<8051.3939.0>, + {message_queue_len,0}, + {status,[{site,"Cluster1"}, + {strategy,riak_repl_keylist_server}, + {fullsync_worker,<8051.3940.0>}, + {dropped_count,0}, + {queue_length,0}, + {queue_byte_size,0}, + {state,wait_for_partition}]}}] +``` + +### Testing Realtime Replication + +You can use the following script to perform `PUT`s and `GET`s on both +sides of the replication and verify that those changes are replicated to +the other side. + +```bash +#!/bin/bash + +VALUE=`date` +CLUSTER_1_IP=172.16.1.11 +CLUSTER_2_IP=192.168.1.21 + +curl -s -X PUT -d "${VALUE}" http://${CLUSTER_1_IP}:8098/riak/replCheck/c1 + +CHECKPUT_C1=`curl -s http://${CLUSTER_1_IP}:8098/riak/replCheck/c1` + +if [ "${VALUE}" = "${CHECKPUT_C1}" ]; then + echo "C1 PUT Successful" +else + echo "C1 PUT Failed" + exit 1 +fi + +curl -s -X PUT -d "${VALUE}" http://${CLUSTER_2_IP}:8098/riak/replCheck/c2 +CHECKPUT_C2=`curl -s http://${CLUSTER_2_IP}:8098/riak/replCheck/c2` + +if [ "${VALUE}" = "${CHECKPUT_C2}" ]; then + echo "C2 PUT Successful" +else + echo "C2 PUT Failed" + exit 1 +fi + +CHECKREPL_C1_TO_C2=`curl -s http://${CLUSTER_2_IP}:8098/riak/replCheck/c1` +CHECKREPL_C2_TO_C1=`curl -s http://${CLUSTER_1_IP}:8098/riak/replCheck/c2` + +if [ "${VALUE}" = "${CHECKREPL_C1_TO_C2}" ]; then + echo "C1 to C2 consistent" +else + echo "C1 to C2 inconsistent + C1:${CHECKPUT_C1} + C2:${CHECKREPL_C1_TO_C2}" + exit 1 +fi + +if [ "${VALUE}" = "${CHECKREPL_C2_TO_C1}" ]; then + echo "C2 to C1 consistent" +else + echo "C2 to C1 inconsistent + C2:${CHECKPUT_C2} + C1:${CHECKREPL_C2_TO_C1}" + exit 1 +fi + +exit 0 +``` + +You will have to change some of the above variables for your own +environment, such as IP addresses or ports. + +If you run this script and things are working as expected, you will get +the following output: + +``` +C1 PUT Successful +C2 PUT Successful +C1 to C2 consistent +C2 to C1 consistent +``` + +## Fullsync + +During realtime replication, operations coordinated by the Source +cluster will be replicated to the Site cluster. Riak Objects are placed +in a queue on the Source cluster and streamed to the Site cluster. When +the queue is full due to high traffic or a bulk loading operation, some +objects will be dropped from replication. These dropped objects can be +sent to the Site cluster by running a fullsync operation. The settings +for the realtime replication queue and their explanations are available +in the [configuration][config v2 mdc] documentation. + +### Initiating a fullsync + +To start a fullsync operation, issue the following command on your +leader node: + +```bash +riak-repl start-fullsync +``` + +A fullsync operation may also be cancelled. If a partition is in +progress, synchronization will stop after that partition completes. +During cancellation, `riak-repl status` will show 'cancelled' in the +status. + +```bash +riak-repl cancel-fullsync +``` + +Fullsync operations may also be paused, resumed, or scheduled for +certain times using cron jobs. A complete list of fullsync commands is +available in the [MDC Operations][cluster ops v2 mdc] documentation. diff --git a/content/riak/kv/2.2.6/configuring/v2-multi-datacenter/ssl.md b/content/riak/kv/2.2.6/configuring/v2-multi-datacenter/ssl.md new file mode 100644 index 0000000000..4ad5b8aa5f --- /dev/null +++ b/content/riak/kv/2.2.6/configuring/v2-multi-datacenter/ssl.md @@ -0,0 +1,160 @@ +--- +title_supertext: "V2 Multi-Datacenter Replication:" +title: "SSL" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "SSL" + identifier: "configuring_v2_replication_ssl" + weight: 103 + parent: "configuring_v2" +toc: true +commercial_offering: true +aliases: + - /riak-docs/riak/2.2.6/ops/mdc/v2/ssl + - /riak-docs/riak/kv/2.2.6/ops/mdc/v2/ssl +--- + +{{% note title="Deprecation Warning" %}} +v2 Multi-Datacenter Replication is deprecated and will be removed in a future version. Please use [v3]({{}}riak/kv/2.2.6/configuring/v3-multi-datacenter/ssl/) instead. +{{% /note %}} + +## Features + +Riak REPL SSL support consists of the following items: + + * Encryption of replication data + * SSL certificate chain validation + * SSL common name whitelisting support + +## SSL Configuration + +To configure SSL, you will need to include the following four settings +in the `riak-repl` section of your `advanced.config`: + +```advancedconfig +{riak-repl, [ + % ... + {ssl_enabled, true}, + {certfile, "/full/path/to/site1-cert.pem"}, + {keyfile, "/full/path/to/site1-key.pem"}, + {cacertdir, "/full/path/to/cacertsdir"} + % ... + ]} + +``` + +The `cacertdir` is a directory containing all of the CA certificates +needed to verify the CA chain back to the root. + +## Verifying Peer Certificates + +Verification of a peer's certificate common name *(CN)* is enabled by using +the `peer_common_name_acl` property in the `riak_repl` section of your +`advanced.config` to specify an Access Control List *(ACL)*. + +The ACL is a list of one or more *patterns*, separated by commas. Each +pattern may be either the exact CN of a certificate to allow, or a +wildcard in the form `*.some.domain.name`. Pattern comparison is +case-insensitive, and a CN matching any of the patterns is allowed to connect. + +For example, `["*.corp.com"]` would match `site3.corp.com` but not +`foo.bar.corp.com` or `corp.com`. If the ACL were +`["*.corp.com", "foo.bar.corp.com"]`, `site3.corp.com` and `foo.bar.corp.com` +would be allowed to connect, but `corp.com` still would not. + +If no ACL (or only the special value `"*"`) is specified, no CN filtering +is performed, except as described below. + +{{% note title="Identical Local and Peer Common Names" %}} +As a special case supporting the view that a host's CN is a fully-qualified +domain name that uniquely identifies a single network device, if the CNs of +the local and peer certificates are the same, the nodes will *NOT* be allowed +to connect. + +This evaluation supercedes ACL checks, so it cannot be overridden with any +setting of the `peer_common_name_acl` property. +{{% /note %}} + +### Examples + +The following example will only allow connections from peer certificate +names like `db.bashosamplecorp.com` and `security.bashosamplecorp.com`: + +```advancedconfig +{riak_repl, [ + % ... + {peer_common_name_acl, ["db.bashosamplecorp.com", "security.bashosamplecorp.com"]} + % ... + ]} +``` + +The following example will allow connections from peer certificate names +like `foo.bashosamplecorp.com` or `db.bashosamplecorp.com`, but not a +peer certificate name like `db.backup.bashosamplecorp.com`: + +```advancedconfig +{riak_repl, [ + % ... + {peer_common_name_acl, ["*.bashosamplecorp.com"]} + % ... + ]} + +``` + +This example will match any peer certificate name (and is the default): + +```advancedconfig +{riak_repl, [ + % ... + {peer_common_name_acl, "*"} + % ... + ]} +``` + +## SSL CA Validation + +You can adjust the way CA certificates are validated by adding the +following to the `riak_repl` section of your `advanced.config`: + +```advancedconfig +{riak_repl, [ + % ... + {ssl_depth, ...} + % ... + ]} +``` + +**Note**: `ssl_depth` takes an integer parameter. + +The depth specifies the maximum number of intermediate certificates that +may follow the peer certificate in a valid certification path. By default, +no more than one (1) intermediate certificate is allowed between the peer +certificate and root CA. By definition, intermediate certificates cannot +be self signed. + +For example: + + * A depth of 0 indicates that the certificate must be signed directly + by a root certificate authority (CA) + * A depth of 1 indicates that the certificate may be signed by at most + 1 intermediate CA's, followed by a root CA + * A depth of 2 indicates that the certificate may be signed by at most + 2 intermediate CA's, followed by a root CA + +## Compatibility + +Replication SSL is ONLY available in Riak 1.2+. + +If SSL is enabled and a connection is made to a Riak Enterprise 1.0 or +1.1 node, the connection will be denied and an error will be logged. + +### Self-Signed Certificates + +You can generate your own CA and keys by using [this +guide](http://www.debian-administration.org/articles/618). + +Make sure that you remove the password protection from the keys you +generate. diff --git a/content/riak/kv/2.2.6/configuring/v3-multi-datacenter.md b/content/riak/kv/2.2.6/configuring/v3-multi-datacenter.md new file mode 100644 index 0000000000..f3fd19cd04 --- /dev/null +++ b/content/riak/kv/2.2.6/configuring/v3-multi-datacenter.md @@ -0,0 +1,157 @@ +--- +tile_supertext: "Configuring:" +title: "V3 Multi-Datacenter Replication" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "V3 Multi-Datacenter" + identifier: "configuring_v3" + weight: 200 + parent: "configuring" +toc: true +commercial_offering: true +aliases: + - /riak-docs/riak/2.2.6/ops/mdc/v3/configuration + - /riak-docs/riak/kv/2.2.6/ops/mdc/v3/configuration +--- + +[config reference#advanced]: {{}}riak/kv/2.2.6/configuring/reference/#advanced-configuration +[config v3 ssl#verify-peer]: {{}}riak/kv/2.2.6/configuring/v3-multi-datacenter/ssl/#verifying-peer-certificates + +> **Note on the `cluster_mgr` setting** +> +> The `cluster_mgr` setting _must_ be set in order for version 3 replication to run. + + +The configuration for Multi-Datacenter (MDC) Replication is kept in +both the `riak_core` and `riak_repl` sections of the `app.config` +configuration file. + +If you are using Riak KV version 2.0, configuration is managed +using the `advanced.config` files on +each node. The semantics of the `advanced.config` file are similar to +the formerly used `app.config` file. For more information and for a list +of configurable parameters, see our documentation on [Advanced Configuration][config reference#advanced]. + +Here is a sample of the syntax: + +```advancedconfig +{riak_core, [ + %% Every *node* runs one cluster_mgr + {cluster_mgr, {"0.0.0.0", 9080 }}, + % ... +]}, +{riak_repl, [ + %% Pick the correct data_root for your platform + %% Debian/Centos/RHEL: + {data_root, "/var/lib/riak/data/riak_repl"}, + %% Solaris: + %% {data_root, "/opt/riak/data/riak_repl"}, + %% FreeBSD/SmartOS: + %% {data_root, "/var/db/riak/riak_repl"}, + {max_fssource_cluster, 5}, + {max_fssource_node, 2}, + {max_fssink_node, 2}, + {fullsync_on_connect, false}, + % ... +]} +``` + +## Settings + +Riak MDC configuration is set using the standard Erlang config file +syntax `{Setting, Value}`. For example, if you wished to set +`fullsync_on_connect` to `false`, you would insert this line into the +`riak_repl` section (appending a comma if you have more settings to +follow): + +```advancedconfig +{fullsync_on_connect, false} +``` + +Once your configuration is set, you can verify its correctness by +running the `riak` command-line tool: + +```bash +riak chkconfig +``` + +## riak_repl Settings + +Setting | Options | Default | Description +:-------|:--------|:--------|:----------- +`cluster_mgr` | `{ip_address, port}` | **REQUIRED** | The cluster manager will listen for connections from remote clusters on this `ip_address` and `port`. Every node runs one cluster manager, but only the cluster manager running on the `cluster_leader` will service requests. This can change as nodes enter and leave the cluster. The value is a combination of an IP address (**not hostname**) followed by a port number. +`max_fssource_cluster` | `nodes` (integer) | `5` | The hard limit on the number of workers which will participate in the source cluster during a fullsync replication. This means that if one has configured fullsync for two different clusters, both with a `max_fssource_cluster` of 5, 10 fullsync workers can be in progress. Only affects nodes on the source cluster on which this parameter is defined via the configuration file or command line. +`max_fssource_node` | `nodes` (integer) | `1` | Limits the number of fullsync workers that will be running on each individual node in a source cluster. This is a hard limit for all fullsyncs enabled; additional fullsync configurations will not increase the number of fullsync workers allowed to run on any node. Only affects nodes on the source cluster on which this parameter is defined via the configuration file or command line. +`max_fssink_node` | `nodes` (integer) | `1` | Limits the number of fullsync workers allowed to run on each individual node in a sink cluster. This is a hard limit for all fullsync sources interacting with the sink cluster. Thus, multiple simultaneous source connections to the sink cluster will have to share the sink nodes number of maximum connections. Only affects nodes on the sink cluster on which this parameter is defined via the configuration file or command line. +`fullsync_on_connect` | `true`, `false` | `true` | Whether to initiate a fullsync on initial connection from the secondary cluster +`data_root` | `path` (string) | `data/riak_repl` | Path (relative or absolute) to the working directory for the replication process +`fullsync_interval` | `minutes` (integer) OR `[{sink_cluster, minutes(integer)}, ...]` | `360` | A single integer value representing the duration to wait in minutes between fullsyncs, or a list of `{"clustername", time_in_minutes}` pairs for each sink participating in fullsync replication. +`rtq_overload_threshold` | `length` (integer) | `2000` | The maximum length to which the realtime replication queue can grow before new objects are dropped. Dropped objects will need to be replicated with a fullsync. +`rtq_overload_recover` | `length` (integer) | `1000` | The length to which the realtime replication queue, in an overload mode, must shrink before new objects are replicated again. +`rtq_max_bytes` | `bytes` (integer) | `104857600` | The maximum size to which the realtime replication queue can grow before new objects are dropped. Defaults to 100MB. Dropped objects will need to be replicated with a fullsync. +`proxy_get` | `enabled`, `disabled` | `disabled` | Enable Riak CS `proxy_get` and block filter. +`rt_heartbeat_interval` | `seconds` (integer) | `15` | A full explanation can be found [below](#heartbeat-settings). +`rt_heartbeat_timeout` | `seconds` (integer) | `15` | A full explanation can be found [below](#heartbeat-settings). + + +## riak_core Settings + +Setting | Options | Default | Description +:-------|:--------|:--------|:----------- +`keyfile` | `path` (string) | `undefined` | Fully qualified path to an ssl `.pem` key file +`cacertdir` | `path` (string) | `undefined` | The `cacertdir` is a fully-qualified directory containing all the CA certificates needed to verify the CA chain back to the root +`certfile` | `path` (string) | `undefined` | Fully qualified path to a `.pem` cert file +`ssl_depth` | `depth` (integer) | `1` | Set the depth to check for SSL CA certs. See [1](#f1). +`ssl_enabled` | `true`, `false` | `false` | Enable SSL communications +`peer_common_name_acl` | `cert` (string) | `"*"` | Verify an SSL peer’s certificate common name. You can provide an ACL as a list of common name *patterns*, and you can wildcard the leftmost part of any of the patterns, so `*.basho.com` would match `site3.basho.com` but not `foo.bar.basho.com` or `basho.com`. See [2](#f2). + + +## Heartbeat Settings + +There are two realtime-replication-related settings in the `riak_repl` +section of `advanced.config` related to the periodic "heartbeat" that is sent +from the source to the sink cluster to verify the sink cluster's +liveness. The `rt_heartbeat_interval` setting determines how often the +heartbeat is sent (in seconds). If a heartbeat is sent and a response is +not received, Riak will wait `rt_heartbeat_timeout` seconds before +attempting to re-connect to the sink; if any data is received from the +sink, even if it is not heartbeat data, the timer will be reset. Setting +`rt_heartbeat_interval` to `undefined` will disable the heartbeat. + +One of the consequences of lowering the timeout threshold arises when +connections are working properly but are slow to respond (perhaps due to +heavy load). In this case, shortening the timeout means that Riak may +attempt to re-connect more often that it needs to. On the other hand, +lengthening the timeout will make Riak less sensitive to cases in which +the connection really has been compromised. + +1. SSL depth is the maximum number of non-self-issued + intermediate certificates that may follow the peer certificate in a valid + certificate chain. If depth is `0`, the PEER must be signed by the trusted + ROOT-CA directly; if `1` the path can be PEER, CA, ROOT-CA; if depth is `2` + then PEER, CA, CA, ROOT-CA and so on. + +2. If the ACL is specified and not the special value `*`, + peers presenting certificates not matching any of the patterns will not be + allowed to connect. + If no ACLs are configured, no checks on the common name are done, except + as described for [Identical Local and Peer Common Names][config v3 ssl#verify-peer]. + +## Default Bucket Properties + +Riak KV version 2.2.0 changed the values of the default bucket properties hash. This will cause an issue replicating between Riak KV clusters with versions 2.2.0 or greater and Riak KV clusters with versions less than 2.2.0. + +To replicate between Riak KV versions 2.2.0 or greater and Riak KV clusters less than version 2.2.0, add the necessary override in the advanced.config file: + +```advanced.config +{riak_repl, [ + {override_capability, [ + {default_bucket_props_hash, [{use, [consistent, datatype, n_val, allow_mult, last_write_wins]}] } + ]} +]} +``` + +If all of the Replication clusters are running Riak KV 2.2.0 or greater, this override is no longer necessary and should be removed. diff --git a/content/riak/kv/2.2.6/configuring/v3-multi-datacenter/nat.md b/content/riak/kv/2.2.6/configuring/v3-multi-datacenter/nat.md new file mode 100644 index 0000000000..88073ba7e6 --- /dev/null +++ b/content/riak/kv/2.2.6/configuring/v3-multi-datacenter/nat.md @@ -0,0 +1,167 @@ +--- +title_supertext: "V3 Multi-Datacenter Replication:" +title: "With NAT" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "With NAT" + identifier: "configuring_v3_replication_nat" + weight: 101 + parent: "configuring_v3" +toc: true +commercial_offering: true +aliases: + - /riak-docs/riak/2.2.6/ops/mdc/v3/nat + - /riak-docs/riak/kv/2.2.6/ops/mdc/v3/nat +--- + +[config v3 ssl]: {{}}riak/kv/2.2.6/configuring/v3-multi-datacenter/ssl + +Riak's Version 3 Replication supports replication of data on +networks that use static NAT. + +This can be used for replicating data over the internet where servers +have both internal and public IP addresses (see the [Replication SSL docs][config v3 ssl] if you replicate data over a public network). + +### Requirements + +In order for Replication to work on a server configured with NAT, the +NAT addresses must be configured *statically*. + +## Configuration + +NAT rules can be configured at runtime, from the command line. + +* `riak-repl nat-map show` + + Shows the current NAT mapping table + +* `riak-repl nat-map add [:port] ` + + Adds a NAT map from the external IP, with an optional port, to an + internal IP. The port number refers to a port that is automatically + mapped to the internal `cluster_mgr` port number. + +* `riak-repl nat-map del [:port] ` + + Deletes a specific NAT map entry. + +### Applying Changes at Runtime + +* Realtime NAT replication changes will be applied once realtime is + stopped and started using the following command: + + * `riak-repl realtime stop ` + * `riak-repl realtime start ` + +* Fullsync NAT replication changes will be applied on the next run of a + fullsync, or you can stop and start the current fullsync. + + * `riak-repl fullsync stop ` + * `riak-repl fullsync start ` + + +## Example + +* Cluster_A is the **source** of replicated data. +* Cluster_B and Cluster_C are the **sinks** of the replicated data. + +### Cluster_A Setup + +Cluster_A is set up with nodes using the following **internal** IP +addresses: + +Internal IP | Public IP +---------------|------------------- +`192.168.1.20` | - +`192.168.1.21` | - +`192.168.1.22` | - +`192.168.1.23` | - +`192.168.1.24` | - + +### Cluster_B Setup + +A node from Cluster_B will be configured as follows: + +Internal IP | Public IP +---------------|------------------- +`192.168.2.40` | `50.16.238.120:5555` +`192.168.2.41` | `50.16.238.121:5555` +`192.168.2.42` | `50.16.238.122:5555` +`192.168.2.43` | `50.16.238.123:5555` +`192.168.2.44` | `50.16.238.124:5555` + +In this example, the `cluster_mgr` port number is the default of `9080`, +while the configured NAT port listens on `5555`. + +### Cluster_C Setup + +A node from Cluster_C is set up with **static NAT**, configured with the +following IP addresses: + +Internal IP | Public IP +---------------|------------------- +`192.168.3.60` | `50.16.238.200:5550` +`192.168.3.61` | `50.16.238.200:5551` +`192.168.3.62` | `50.16.238.200:5552` +`192.168.3.63` | `50.16.238.200:5553` +`192.168.3.64` | `50.16.238.200:5554` + +In this example, the `cluster_mgr` port number is the default of `9080`, +while the configured NAT port listens on `5566`. + +```bash +# on any node of Cluster_A +riak-repl clustername Server_A + +# on any node of Cluster_B +riak-repl clustername Server_B + +# on any node of Cluster_C +riak-repl clustername Server_C + +# on 50.16.238.120 of Cluster_B +riak-repl nat-map add 50.16.238.120:5555 192.168.2.40 +# on 50.16.238.121 of Cluster_B +riak-repl nat-map add 50.16.238.121:5555 192.168.2.41 +# on 50.16.238.122 of Cluster_B +riak-repl nat-map add 50.16.238.122:5555 192.168.2.42 +# on 50.16.238.123 of Cluster_B +riak-repl nat-map add 50.16.238.123:5555 192.168.2.43 +# on 50.16.238.124 of Cluster_B +riak-repl nat-map add 50.16.238.124:5555 192.168.2.44 + +# on 192.168.3.60 of Cluster_C +riak-repl nat-map add 50.16.238.200:5550 192.168.3.60 +# on 192.168.3.61 of Cluster_C +riak-repl nat-map add 50.16.238.200:5551 192.168.3.61 +# on 192.168.3.62 of Cluster_C +riak-repl nat-map add 50.16.238.200:5552 192.168.3.62 +# on 192.168.3.63 of Cluster_C +riak-repl nat-map add 50.16.238.200:5553 192.168.3.63 +# on 192.168.3.64 of Cluster_C +riak-repl nat-map add 50.16.238.200:5554 192.168.3.64 + + +# Connect replication from Cluster_A to Cluster_B: +# on any node of Cluster_A +riak-repl connect 50.16.238.120:5555 +# You can connect to any node in Cluster_B with NAT mapped IP's/ports +# This command only needs to be run *once* for a cluster. + +# Connect replication from Cluster_A to Cluster_C: +# on any node of Cluster_A +riak-repl connect 50.16.238.200:5550 +# You can connect to any node in Cluster_C with NAT mapped IP's/ports +# This command only needs to be run *once* for a cluster. + + +# on any node from Cluster_A +riak-repl realtime enable Cluster_B +riak-repl realtime enable Cluster_C + +riak-repl realtime start Cluster_B +riak-repl realtime start Cluster_C +``` diff --git a/content/riak/kv/2.2.6/configuring/v3-multi-datacenter/quick-start.md b/content/riak/kv/2.2.6/configuring/v3-multi-datacenter/quick-start.md new file mode 100644 index 0000000000..f22a1aeab0 --- /dev/null +++ b/content/riak/kv/2.2.6/configuring/v3-multi-datacenter/quick-start.md @@ -0,0 +1,168 @@ +--- +title_supertext: "V3 Multi-Datacenter Replication:" +title: "Quickstart" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Quickstart" + identifier: "configuring_v3_quickstart" + weight: 100 + parent: "configuring_v3" +toc: true +commercial_offering: true +aliases: + - /riak-docs/riak/2.2.6/ops/mdc/v3/quick-start + - /riak-docs/riak/kv/2.2.6/ops/mdc/v3/quick-start +--- + +[perf index]: {{}}riak/kv/2.2.6/using/performance +[config v3 mdc]: {{}}riak/kv/2.2.6/configuring/v3-multi-datacenter +[cluster ops v3 mdc]: {{}}riak/kv/2.2.6/using/cluster-operations/v3-multi-datacenter + +This guide will walk you through the process of configuring Riak's v3 +Replication to perform replication between two sample Riak clusters on +separate networks. This guide will also cover bidirectional replication, +which is accomplished by setting up unidirectional replication in both +directions between the clusters. It is important to note that both +clusters must have the same ring size, but can have a different number +of nodes. + +## Prerequisites + +This guide assumes that you have completed the following steps: + +* Install [Riak][install index] +* Perform [System Tuning][perf index] +* Review [Configuration][config v3 mdc] + +## About v3 Replication in 1.3 and higher + +In Riak's v3 Replication from Riak KV version 1.3 onwards, the nomenclature for Source and Site +clusters has changed. To more accurately reflect the behavior of each of +the clusters, "listeners" and "sites" are now known as "sources" and +"sinks." Data transfer now originates at the "source" and replicates to +the "sink;" initiation is always from the primary (source) to the backup +(sink) data center. + +Additionally, knowledge of the state of each cluster is now managed by a +**cluster manager** process, which greatly simplifies the setup and +maintenance of Multi-Datacenter replication. + +## Scenario + +Configure Riak MDC to perform replication, given the following two +Riak Clusters, each of which consists of three nodes: + +### Cluster 1 + +Name | IP | Node name +:-----|:-------------|----------------- +`node1` | `10.60.67.149` | `riak@10.60.67.149` +`node2` | `10.60.83.39` | `riak@10.60.83.39` +`node3` | `10.60.90.252` | `riak@10.60.90.252` + +### Cluster 2 + +Name | IP | Node name +:-----|:------------|:---------------- +`node4` | `10.60.77.10` | `riak@10.60.77.10` +`node5` | `10.60.84.41` | `riak@10.60.84.41` +`node6` | `10.60.92.44` | `riak@10.60.92.44` + + +### Set up Cluster1 → Cluster2 Connection + +#### Set up the Source on Cluster1 + +On a node in Cluster1, `node1` for example, initiate and name this +cluster with `riak-repl clustername `: + +```bash +riak-repl clustername Cluster1 +``` + +#### Setup the Sink on Cluster2 + +On a node in Cluster2, `node4` for example, initiation and name this +cluster with `riak-repl clustername `: + +```bash +riak-repl clustername Cluster2 +``` + +#### Connect the Source to the Sink + +From Cluster1, connect to the IP and port of Cluster2 with `riak-repl +connect :`: + +```bash +riak-repl connect 10.60.77.10:9080 +``` + +> The port can be found in the `riak_core` section of the `advanced.config` +> under `cluster_mgr`. + +#### View your active connections + +From Cluster1, view your active connections with `riak-repl +connections`: + +``` +Sink Cluster Name [Members] +---- ------------ ---------- --------- +Cluster2 Cluster2 <0.7985.0> ["10.60.77.10:9080"] (via 10.60.77.10:9080) +``` + +### Set up Cluster2 → Cluster1 Connection (if bidirectional replication is desired) + +#### Connect the Source to the Sink + +From Cluster2, connect to the IP and port of Cluster1 with `riak-repl +connect :`: + +```bash +riak-repl connect 10.60.67.149:9080 +``` + +#### View Your Active Connections + +From Cluster2, view your active connections with `riak-repl +connections`: + +``` +Sink Cluster Name [Members] +---- ------------ ---------- --------- +Cluster1 Cluster1 <0.4456.0> ["10.60.67.149:9080"] (via 10.60.67.149:9080) +``` + +{{% note title="Note on connections" %}} +At this point, if you do not have connections, replication will not work. +Check your IP bindings by running `netstat -a` on all nodes. You should see +`*:9080 LISTENING`. If not, you have configuration problems. +{{% /note %}} + +### Enable Realtime Replication + +From Cluster1, run `riak-repl realtime enable ` to start +queuing updates on Cluster1 for replication: + +```bash +riak-repl realtime enable Cluster2 +``` + +Also on Cluster1, run `riak-repl realtime start ` to +establish connectivity from Cluster1 to Cluster2 to push queued updates: + +```bash +riak-repl realtime start Cluster2 +``` + +To enable bidirectional replication, do the reverse from Cluster2. +Once this is done, bidirectional replication should be operating. + +## More information + +For a full list of commands, you may enter `riak-repl` to see full +instructions on usage, or check the [Operations][cluster ops v3 mdc] documentation. diff --git a/content/riak/kv/2.2.6/configuring/v3-multi-datacenter/ssl.md b/content/riak/kv/2.2.6/configuring/v3-multi-datacenter/ssl.md new file mode 100644 index 0000000000..f5f120743c --- /dev/null +++ b/content/riak/kv/2.2.6/configuring/v3-multi-datacenter/ssl.md @@ -0,0 +1,170 @@ +--- +title_supertext: "V3 Multi-Datacenter Replication:" +title: "SSL" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "SSL" + identifier: "configuring_v3_replication_ssl" + weight: 103 + parent: "configuring_v3" +toc: true +commercial_offering: true +aliases: + - /riak-docs/riak/2.2.6/ops/mdc/v3/ssl + - /riak-docs/riak/kv/2.2.6/ops/mdc/v3/ssl +--- + +[config reference#advanced.config]: {{}}riak/kv/2.2.6/configuring/reference/#the-advanced-config-file + +## Features + +Riak Multi-Datacenter (MDC) Replication SSL consists of the following +items: + + * Encryption of replication data + * SSL certificate chain validation + * SSL common name whitelisting support + +> **Note on cross-internet traffic** +> +> As an alternative to Riak's built-in SSL capabilities, we +recommend using [stunnel](https://www.stunnel.org/index.html) or a +virtual private network (VPM) for inter-datacenter connections. + +## SSL Configuration + +To configure SSL, you will need to include the following 4 settings in +the `riak-core` section of [`advanced.confg`][config reference#advanced.config]: + +```advancedconfig +{riak_core, [ + % ... + {ssl_enabled, true}, + {certfile, "/full/path/to/site1-cert.pem"}, + {keyfile, "/full/path/to/site1-key.pem"}, + {cacertdir, "/full/path/to/cacertsdir"} + % ... + ]} + +``` + +The `cacertsdir` is a directory containing all the CA certificates +needed to verify the CA chain back to the root. + +{{% note title="Note on configuration" %}} +In Version 3 replication, the SSL settings need to be placed in the +`riak-core` section of `advanced.config` as opposed to the `riak-repl` section +used by Version 2 replication. +{{% /note %}} + +## Verifying Peer Certificates + +Verification of a peer's certificate common name *(CN)* is enabled by using +the `peer_common_name_acl` property in the `riak_core` section of your +`advanced.config` to specify an Access Control List *(ACL)*. + +The ACL is a list of one or more *patterns*, separated by commas. Each +pattern may be either the exact CN of a certificate to allow, or a +wildcard in the form `*.some.domain.name`. Pattern comparison is +case-insensitive, and a CN matching any of the patterns is allowed to connect. + +For example, `["*.corp.com"]` would match `site3.corp.com` but not +`foo.bar.corp.com` or `corp.com`. If the ACL were +`["*.corp.com", "foo.bar.corp.com"]`, `site3.corp.com` and `foo.bar.corp.com` +would be allowed to connect, but `corp.com` still would not. + +If no ACL (or only the special value `"*"`) is specified, no CN filtering +is performed, except as described below. + +{{% note title="Identical Local and Peer Common Names" %}} +As a special case supporting the view that a host's CN is a fully-qualified +domain name that uniquely identifies a single network device, if the CNs of +the local and peer certificates are the same, the nodes will *NOT* be allowed +to connect. + +This evaluation supercedes ACL checks, so it cannot be overridden with any +setting of the `peer_common_name_acl` property. +{{% /note %}} + +### Examples + +The following example will only allow connections from peer certificate +names like `db.bashosamplecorp.com` and `security.bashosamplecorp.com`: + +```advancedconfig +{riak_core, [ + % ... + {peer_common_name_acl, ["db.bashosamplecorp.com", "security.bashosamplecorp.com"]} + % ... + ]} + +``` + +The following example will allow connections from peer certificate names +like `foo.bashosamplecorp.com` or `db.bashosamplecorp.com`, but not a +peer certificate name like `db.backup.bashosamplecorp.com`. + +```advancedconfig +{riak_core, [ + % ... + {peer_common_name_acl, ["*.bashosamplecorp.com"]} + % ... + ]} + +``` + +This example will match any peer certificate name (and is the default): + +```advancedconfig +{riak_core, [ + % ... + {peer_common_name_acl, "*"} + % ... + ]} + +``` + +## SSL CA Validation + +You can adjust the way CA certificates are validated by adding the +following to the `riak_repl` section of `advanced.config`: + +```advancedconfig +{riak_core, [ + % ... + {ssl_depth, 3} % Sets the depth to 3 + % ... + ]} + +``` + +**Note**: `ssl_depth` takes an integer parameter. + +The depth specifies the maximum number of intermediate certificates that +may follow the peer certificate in a valid certification path. The +intermediate certificates must not be self signed. + +The following example depths illustrate this: + + * a depth of `0` indicates that the certificate must be signed + directly by a root certificate authority (CA) + * a depth of `1` indicates that the certificate may be signed by at + most 1 intermediate CA's, followed by a root CA + * a depth of `2` indicates that the certificate may be signed by at + most 2 intermediate CA's, followed by a root CA + +## Compatibility + +Replication SSL for *Version 3* is available in *Riak 1.4+*. + +If SSL is enabled and a connection is made to a Riak Enterprise 1.0 or +1.1 node, the connection will be denied and an error will be logged. + +### Self-Signed Certificates + +Read how to [generate your own CA and +keys](http://www.debian-administration.org/articles/618). Ensure that +you remove the password protection from the keys you generate. diff --git a/content/riak/kv/2.2.6/developing.md b/content/riak/kv/2.2.6/developing.md new file mode 100644 index 0000000000..f537129267 --- /dev/null +++ b/content/riak/kv/2.2.6/developing.md @@ -0,0 +1,73 @@ +--- +title: "Developing with Riak KV" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Developing" + identifier: "developing" + weight: 300 + pre: lambda +toc: true +--- + +[getting started]: ../developing/getting-started +[usage index]: ../developing/usage +[client libraries]: ../developing/client-libraries +[dev data types]: ../developing/data-types +[dev data modeling]: ../developing/data-modeling +[apps index]: ../developing/app-guide +[dev api index]: ../developing/api +[dev faq]: ../developing/faq + +## In This Section + +#### [Getting Started][getting started] + +Step-by-step guide for getting started developing with Riak KV. + +[Learn More >>][getting started] + +#### [Usage][usage index] + +A set of tutorials covering common development tasks such as performing CRUD operations, working with search, and using bucket types. + +[Learn More >>][usage index] + +#### [Client Libraries][client libraries] + +Overview of client libraries for a variety of programming languages and environments. + +[Learn More >>][client libraries] + +#### [Data Types][dev data types] + +Overview and guide to working with data types in Riak KV. + +[Learn More >>][dev data types] + +#### [Data Modeling][dev data modeling] + +Information on use cases and data models that are a good fit for Riak KV. + +[Learn More >>][dev data modeling] + +#### [Application Guide][apps index] + +A guide that will walk you through questions to ask about your use case before getting started developing applications with Riak KV. + +[Learn More >>][apps index] + +#### [APIs Reference][dev api index] + +Information and reference material on Riak KV APIs. + +[Learn More >>][dev api index] + +#### [FAQ][dev faq] + +Frequently asked questions when developing applications with Riak KV. + +[Learn More >>][dev faq] + diff --git a/content/riak/kv/2.2.6/developing/api.md b/content/riak/kv/2.2.6/developing/api.md new file mode 100644 index 0000000000..1746d53eaa --- /dev/null +++ b/content/riak/kv/2.2.6/developing/api.md @@ -0,0 +1,37 @@ +--- +title: "APIs" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "APIs" + identifier: "developing_apis" + weight: 107 + parent: "developing" +toc: true +--- + +[dev api http]: ./http +[dev api backend]: ./backend +[dev api pbc]: ./protocol-buffers/ + +## In This Section + +#### [HTTP APIs][dev api http] + +Documentation on Riak KV's HTTP API. + +[Learn More >>][dev api http] + +#### [Protocol Buffers][dev api pbc] + +Information on Riak KV's Protocol Buffer Client API + +[Learn More >>][dev api pbc] + +#### [Backend API][dev api backend] + +Overview of Riak KV's storage backend API. + +[Learn More >>][dev api backend] diff --git a/content/riak/kv/2.2.6/developing/api/backend.md b/content/riak/kv/2.2.6/developing/api/backend.md new file mode 100644 index 0000000000..97ef2178bf --- /dev/null +++ b/content/riak/kv/2.2.6/developing/api/backend.md @@ -0,0 +1,114 @@ +--- +title: "Backend API" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Backend API" + identifier: "apis_backend" + weight: 101 + parent: "developing_apis" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/references/backend-api + - /riak-docs/riak/kv/2.2.6/dev/references/backend-api +--- + +[plan backend]: {{}}riak/kv/2.2.6/setup/planning/backend + +Riak's storage API uniformly applies to all of the +[supported backends][plan backend]. This page presents the details of +the storage backend API in the form of +[Erlang type specifications](http://www.erlang.org/doc/reference_manual/typespec.html) +(specs). + +Specs are used by [dialyzer](http://www.erlang.org/doc/man/dialyzer.html), +an Erlang static analysis tool. We recommend copying these specs into any +custom backend modules and use them as a guide for development to +avoid errors and ensure full compatibility with Riak. + +Also included below is the function export list that can be pasted directly +into a custom storage backend module. + +```erlang +%% Riak Storage Backend API +-export([api_version/0, + start/2, + stop/1, + get/3, + put/5, + delete/4, + drop/1, + fold_buckets/4, + fold_keys/4, + fold_objects/4, + is_empty/1, + status/1, + callback/3]). + +%% =================================================================== +%% Public API +%% =================================================================== + +%% @doc Return the major version of the +%% current API and a capabilities list. +%% The current valid capabilities are async_fold +%% and indexes. +-spec api_version() -> {integer(), [atom()]}. + +%% @doc Start the backend +-spec start(integer(), config()) -> {ok, state()} | {error, term()}. + +%% @doc Stop the backend +-spec stop(state()) -> ok. + +%% @doc Retrieve an object from the backend +-spec get(riak_object:bucket(), riak_object:key(), state()) -> + {ok, any(), state()} | + {ok, not_found, state()} | + {error, term(), state()}. + +%% @doc Insert an object into the backend. +-type index_spec() :: {add, Index, SecondaryKey} | {remove, Index, SecondaryKey}. +-spec put(riak_object:bucket(), riak_object:key(), [index_spec()], binary(), state()) -> + {ok, state()} | + {error, term(), state()}. + +%% @doc Delete an object from the backend +-spec delete(riak_object:bucket(), riak_object:key(), [index_spec()], state()) -> + {ok, state()} | + {error, term(), state()}. + +%% @doc Fold over all the buckets +-spec fold_buckets(riak_kv_backend:fold_buckets_fun(), + any(), + [], + state()) -> {ok, any()} | {async, fun()}. + +%% @doc Fold over all the keys for one or all buckets. +-spec fold_keys(riak_kv_backend:fold_keys_fun(), + any(), + [{atom(), term()}], + state()) -> {ok, term()} | {async, fun()}. + +%% @doc Fold over all the objects for one or all buckets. +-spec fold_objects(riak_kv_backend:fold_objects_fun(), + any(), + [{atom(), term()}], + state()) -> {ok, any()} | {async, fun()}. + +%% @doc Delete all objects from this backend +%% and return a fresh reference. +-spec drop(state()) -> {ok, state()} | {error, term(), state()}. + +%% @doc Returns true if this backend contains any +%% non-tombstone values; otherwise returns false. +-spec is_empty(state()) -> boolean() | {error, term()}. + +%% @doc Get the status information for this backend +-spec status(state()) -> [{atom(), term()}]. + +%% @doc Register an asynchronous callback +-spec callback(reference(), any(), state()) -> {ok, state()}. +``` diff --git a/content/riak/kv/2.2.6/developing/api/http.md b/content/riak/kv/2.2.6/developing/api/http.md new file mode 100644 index 0000000000..e8e7a6ac1d --- /dev/null +++ b/content/riak/kv/2.2.6/developing/api/http.md @@ -0,0 +1,89 @@ +--- +title: "HTTP API" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "HTTP API" + identifier: "apis_http" + weight: 102 + parent: "developing_apis" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/references/http + - /riak-docs/riak/kv/2.2.6/dev/references/http +--- + +Riak has a rich, full-featured HTTP 1.1 API. This is an overview of the +operations you can perform via HTTP and can be used as a guide for +developing a compliant client. All URLs assume the default configuration +values where applicable. All examples use `curl` to interact with Riak. + +> **URL Escaping** +> +> Buckets, keys, and link specifications may not contain unescaped +slashes. Use a URL-escaping library or replace slashes with `%2F`. + +## Bucket-related Operations + +Method | URL | Doc +:------|:----|:--- +`GET` | `/types//buckets//props` | [HTTP Get Bucket Properties]({{}}riak/kv/2.2.6/developing/api/http/get-bucket-props) +`PUT` | `/types//buckets//props` | [HTTP Set Bucket Properties]({{}}riak/kv/2.2.6/developing/api/http/set-bucket-props) +`DELETE` | `/types//buckets//props` | [HTTP Reset Bucket Properties]({{}}riak/kv/2.2.6/developing/api/http/reset-bucket-props) +`GET` | `/types//buckets?buckets=true` | [HTTP List Buckets]({{}}riak/kv/2.2.6/developing/api/http/list-buckets) +`GET` | `/types//buckets//keys?keys=true` | [HTTP List Keys]({{}}riak/kv/2.2.6/developing/api/http/list-keys) + +## Object-related Operations + +Method | URL | Doc +:------|:----|:--- +`GET` | `/types//buckets//keys/` | [HTTP Fetch Object]({{}}riak/kv/2.2.6/developing/api/http/fetch-object) +`POST` | `/types//buckets//keys` | [HTTP Store Object]({{}}riak/kv/2.2.6/developing/api/http/store-object) +`PUT` | `/types//buckets//keys/` | [HTTP Store Object]({{}}riak/kv/2.2.6/developing/api/http/store-object) +`POST` | `/types//buckets//keys/` | [HTTP Store Object]({{}}riak/kv/2.2.6/developing/api/http/store-object) +`DELETE` | `/types//buckets//keys/` | [HTTP Delete Object]({{}}riak/kv/2.2.6/developing/api/http/delete-object) + +## Riak-Data-Type-related Operations + +Method | URL +:------|:---- +`GET` | `/types//buckets//datatypes/` +`POST` | `/types//buckets//datatypes` +`POST` | `/types//buckets//datatypes/` + +For documentation on the HTTP API for [Riak Data Types]({{}}riak/kv/2.2.6/learn/concepts/crdts), +see the `curl` examples in [Using Data Types]({{}}riak/kv/2.2.6/developing/data-types/#usage-examples) +and subpages e.g. [sets]({{}}riak/kv/2.2.6/developing/data-types/sets). + +Advanced users may consult the technical documentation inside the Riak +KV internal module `riak_kv_wm_crdt`. + +## Query-related Operations + +Method | URL | Doc +:------|:----|:--- +`POST` | `/mapred` | [HTTP MapReduce]({{}}riak/kv/2.2.6/developing/api/http/mapreduce) +`GET` | `/types//buckets//index//` | [HTTP Secondary Indexes]({{}}riak/kv/2.2.6/developing/api/http/secondary-indexes) +`GET` | `/types//buckets//index///` | [HTTP Secondary Indexes]({{}}riak/kv/2.2.6/developing/api/http/secondary-indexes) + +## Server-related Operations + +Method | URL | Doc +:------|:----|:--- +`GET` | `/ping` | [HTTP Ping]({{}}riak/kv/2.2.6/developing/api/http/ping) +`GET` | `/stats` | [HTTP Status]({{}}riak/kv/2.2.6/developing/api/http/status) +`GET` | `/` | [HTTP List Resources]({{}}riak/kv/2.2.6/developing/api/http/list-resources) + +## Search-related Operations + +Method | URL | Doc +:------|:----|:--- +`GET` | `/search/query/` | [HTTP Search Query]({{}}riak/kv/2.2.6/developing/api/http/search-query) +`GET` | `/search/index` | [HTTP Search Index Info]({{}}riak/kv/2.2.6/developing/api/http/search-index-info) +`GET` | `/search/index/` | [HTTP Fetch Search Index]({{}}riak/kv/2.2.6/developing/api/http/fetch-search-index) +`PUT` | `/search/index/` | [HTTP Store Search Index]({{}}riak/kv/2.2.6/developing/api/http/store-search-index) +`DELETE` | `/search/index/` | [HTTP Delete Search Index]({{}}riak/kv/2.2.6/developing/api/http/delete-search-index) +`GET` | `/search/schema/` | [HTTP Fetch Search Schema]({{}}riak/kv/2.2.6/developing/api/http/fetch-search-schema) +`PUT` | `/search/schema/` | [HTTP Store Search Schema]({{}}riak/kv/2.2.6/developing/api/http/store-search-schema) diff --git a/content/riak/kv/2.2.6/developing/api/http/counters.md b/content/riak/kv/2.2.6/developing/api/http/counters.md new file mode 100644 index 0000000000..6b0962abe5 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/api/http/counters.md @@ -0,0 +1,78 @@ +--- +title: "HTTP Counters" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Counters" + identifier: "http_counters" + weight: 118 + parent: "apis_http" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/references/http/counters + - /riak-docs/riak/kv/2.2.6/dev/references/http/counters +--- + +Riak counters are a CRDT (convergent replicated data type) that (eventually) +converge to the correct total. You merely increment the counter with some +integer, and any potential conflicts will be automatically resolved by Riak. + +## Setup + +Riak counters can only be used if the bucket has the `allow_mult` property +set to `true`. + +``` +curl -XPUT localhost:8098/buckets/BUCKET/props \ + -H "Content-Type: application/json" \ + -d "{\"props\" : {\"allow_mult\": true}}" +``` + +If you attempt to use counters without setting the above, you'll get this +message: + +``` +Counters require bucket property 'allow_mult=true' +``` + +## Request + +To insert just POST an integer value using the `/counters` resource. This will +increment that keyed value by the given amount. + +``` +POST /buckets/BUCKET/counters/KEY +``` + +To receive the current value is a GET using `/counters` + +``` +GET /buckets/BUCKET/counters/KEY +``` + +## Response + +The regular POST/PUT ([HTTP Store Object]({{}}riak/kv/2.2.6/developing/api/http/store-object)) and GET ([HTTP Fetch Object]({{}}riak/kv/2.2.6/developing/api/http/fetch-object)) responses apply here. + +Caveats: Counters have no support for Secondary Indexes (2i), Links or Custom HTTP Metadata. + +## Example + +The body must be an integer (positive or negative). + +``` +curl -XPOST http://localhost:8098/buckets/my_bucket/counters/my_key -d "1" + +curl http://localhost:8098/buckets/my_bucket/counters/my_key +1 + +curl -XPOST http://localhost:8098/buckets/my_bucket/counters/my_key -d "100" + +curl http://localhost:8098/buckets/my_bucket/counters/my_key +101 + +curl -XPOST http://localhost:8098/buckets/my_bucket/counters/my_key -d "-1" +100 +``` diff --git a/content/riak/kv/2.2.6/developing/api/http/delete-object.md b/content/riak/kv/2.2.6/developing/api/http/delete-object.md new file mode 100644 index 0000000000..b9609c827b --- /dev/null +++ b/content/riak/kv/2.2.6/developing/api/http/delete-object.md @@ -0,0 +1,75 @@ +--- +title: "HTTP Delete Object" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Delete Object" + identifier: "http_delete_object" + weight: 107 + parent: "apis_http" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/references/http/delete-object + - /riak-docs/riak/kv/2.2.6/dev/references/http/delete-object +--- + +Deletes an object from the specified bucket / key. + +## Request + +``` +DELETE /types/type/buckets/bucket/keys/key +DELETE /buckets/bucket/keys/key +``` + +Optional query parameters: + +* `rw` - quorum for both operations (get and put) involved in deleting an +object (default is set at the bucket level) +* `r` - (read quorum) how many replicas need to agree when retrieving the object +* `pr` - (primary read quorum) works like `r` but requires that the nodes +read from are not fallback nodes +* `w` - (write quorum) how many replicas must confirm receiving writes before returning a successful response +* `dw` - (durable write quorum) how many replicas to commit to durable storage +before returning a successful response +* `pw` - (primary write quorum) how many replicas to commit to primary nodes +before returning a successful response + +## Response + +Normal response codes: + +* `204 No Content` +* `404 Not Found` + +Typical error codes: + +* `400 Bad Request` - e.g. when rw parameter is invalid (> N) + +`404` responses are "normal" in the sense that DELETE operations are idempotent +and not finding the resource has the same effect as deleting it. + +## Example + +```curl +$ curl -v -X DELETE http://127.0.0.1:8098/buckets/test/keys/test2 +* About to connect() to 127.0.0.1 port 8098 (#0) +* Trying 127.0.0.1... connected +* Connected to 127.0.0.1 (127.0.0.1) port 8098 (#0) +> DELETE /buckets/test/keys/test2 HTTP/1.1 +> User-Agent: curl/7.19.4 (universal-apple-darwin10.0) libcurl/7.19.4 OpenSSL/0.9.8l zlib/1.2.3 +> Host: 127.0.0.1:8098 +> Accept: */* +> +< HTTP/1.1 204 No Content +< Vary: Accept-Encoding +< Server: MochiWeb/1.1 WebMachine/1.9.0 (participate in the frantic) +< Date: Fri, 30 Sep 2011 15:24:35 GMT +< Content-Type: application/json +< Content-Length: 0 +< +* Connection #0 to host 127.0.0.1 left intact +* Closing connection #0 +``` diff --git a/content/riak/kv/2.2.6/developing/api/http/delete-search-index.md b/content/riak/kv/2.2.6/developing/api/http/delete-search-index.md new file mode 100644 index 0000000000..cf12a006d6 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/api/http/delete-search-index.md @@ -0,0 +1,33 @@ +--- +title: "HTTP Delete Search Index" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Delete Search Index" + identifier: "http_delete_search_index" + weight: 116 + parent: "apis_http" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/references/http/delete-search-index + - /riak-docs/riak/kv/2.2.6/dev/references/http/delete-search-index +--- + +Deletes a Riak Search index. + +## Request + +``` +DELETE /search/index/ +``` + +## Normal Response Codes + +* `204 No Content` --- The index was successfully deleted (also returned + if the index did not exist to begin with) + +## Typical Error Codes + +* `503 Service Unavailable` --- The request timed out internally diff --git a/content/riak/kv/2.2.6/developing/api/http/fetch-object.md b/content/riak/kv/2.2.6/developing/api/http/fetch-object.md new file mode 100644 index 0000000000..83fa532939 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/api/http/fetch-object.md @@ -0,0 +1,242 @@ +--- +title: "HTTP Fetch Object" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Fetch Object" + identifier: "http_fetch_object" + weight: 105 + parent: "apis_http" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/references/http/fetch-object + - /riak-docs/riak/kv/2.2.6/dev/references/http/fetch-object +--- + +Reads an object from the specified bucket/key. + +## Request + +```bash +GET /types/type/buckets/bucket/keys/key +GET /buckets/bucket/keys/key +``` + +Important headers: + +* `Accept` - When `multipart/mixed` is the preferred content-type, objects with +siblings will return all siblings in single request. See [Siblings examples](#siblings-examples). See +also RFC 2616 - [Accept header definition](http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.1). + +Optional headers: + +* `If-None-Match` and `If-Modified-Since` invoke conditional request semantics, +matching on the `ETag` and `Last-Modified` of the object, respectively. If the +object fails one of the tests (that is, if the ETag is equal or the object is +unmodified since the supplied timestamp), Riak will return a `304 Not Modified` +response. See also RFC 2616 - [304 Not Modified](http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3.5). + +Optional query parameters: + +* `r` - (read quorum) how many replicas need to agree when retrieving the +object ([default is defined by the bucket]({{}}riak/kv/2.2.6/developing/api/http/set-bucket-props)) +* `pr` - how many primary replicas need to be online when doing the read +([default is defined by the bucket]({{}}riak/kv/2.2.6/developing/api/http/set-bucket-props)) +* `basic_quorum` - whether to return early in some failure cases (eg. when r=1 +and you get 2 errors and a success `basic_quorum=true` would return an error) +([default is defined by the bucket]({{}}riak/kv/2.2.6/developing/api/http/set-bucket-props)) +* `notfound_ok` - whether to treat notfounds as successful reads for the +purposes of R ([default is defined by the bucket]({{}}riak/kv/2.2.6/developing/api/http/set-bucket-props)) +* `vtag` - when accessing an object with siblings, which sibling to retrieve. +Scroll down to the [Manually requesting siblings](#manually-requesting-siblings) example for more information. + +## Response + +Normal response codes: + +* `200 OK` +* `300 Multiple Choices` +* `304 Not Modified` (when using conditional request semantics) + +Typical error codes: + +* `400 Bad Request` - e.g. when r parameter is invalid (> N) +* `404 Not Found` - the object could not be found on enough partitions +* `503 Service Unavailable` - the request timed out internally + +Important headers: + +* `Content-Type` - the media type/format +* `X-Riak-Vclock` - the opaque vector clock for the object +* `X-Riak-Meta-*` - any user-defined metadata defined when storing the object +* `ETag` - the entity tag for the object, useful for conditional GET operations +and validation-based caching +* `Last-Modified` - a timestamp for when the object was last written, in HTTP +datetime format +* `Link` - user- and system-defined links to other resources. [Read more about Links.]({{}}riak/kv/2.2.6/learn/glossary/#links) + +The body of the response will be the contents of the object except when siblings +are present. + +{{% note title="Siblings" %}} +When `allow_mult` is set to true in the bucket properties, concurrent updates +are allowed to create "sibling" objects, meaning that the object has any +number of different values that are related to one another by the vector +clock. This allows your application to use its own conflict resolution +technique. + +An object with multiple sibling values will result in a `300 Multiple Choices` +response. If the `Accept` header prefers `multipart/mixed`, all siblings will +be returned in a single request as sections of the `multipart/mixed` response +body. Otherwise, a list of "vtags" will be given in a simple text format. You +can request individual siblings by adding the `vtag` query parameter. Scroll +down to the 'manually requesting siblings' example below for more information. + +To resolve the conflict, store the resolved version with the `X-Riak-Vclock` +given in the response. +{{% /note %}} + +## Simple Example + +```curl +$ curl -v http://127.0.0.1:8098/buckets/test/keys/doc2 +* About to connect() to 127.0.0.1 port 8098 (#0) +* Trying 127.0.0.1... connected +* Connected to 127.0.0.1 (127.0.0.1) port 8098 (#0) +> GET /buckets/test/keys/doc2 HTTP/1.1 +> User-Agent: curl/7.19.4 (universal-apple-darwin10.0) libcurl/7.19.4 OpenSSL/0.9.8l zlib/1.2.3 +> Host: 127.0.0.1:8098 +> Accept: */* +> +< HTTP/1.1 200 OK +< X-Riak-Vclock: a85hYGBgzGDKBVIsbLvm1WYwJTLmsTLcjeE5ypcFAA== +< Vary: Accept-Encoding +< Server: MochiWeb/1.1 WebMachine/1.9.0 (participate in the frantic) +< Link: ; rel="up" +< Last-Modified: Wed, 10 Mar 2010 18:11:41 GMT +< ETag: 6dQBm9oYA1mxRSH0e96l5W +< Date: Fri, 30 Sep 2011 15:24:35 GMT +< Content-Type: application/json +< Content-Length: 13 +< +* Connection #0 to host 127.0.0.1 left intact +* Closing connection #0 +{"foo":"bar"} +``` + +## Siblings examples + +### Manually requesting siblings + +Simple call to fetch an object that has siblings: + +```curl +$ curl -v http://127.0.0.1:8098/buckets/test/keys/doc +* About to connect() to 127.0.0.1 port 8098 (#0) +* Trying 127.0.0.1... connected +* Connected to 127.0.0.1 (127.0.0.1) port 8098 (#0) +> GET /buckets/test/keys/doc HTTP/1.1 +> User-Agent: curl/7.19.4 (universal-apple-darwin10.0) libcurl/7.19.4 OpenSSL/0.9.8l zlib/1.2.3 +> Host: 127.0.0.1:8098 +> Accept: */* +> +< HTTP/1.1 300 Multiple Choices +< X-Riak-Vclock: a85hYGDgyGDKBVIszMk55zKYEhnzWBlKIniO8kGF2TyvHYIKf0cIszUnMTBzHYVKbIhEUl+VK4spDFTPxhHzFyqhEoVQz7wkSAGLMGuz6FSocFIUijE3pt5HlsgCAA== +< Vary: Accept, Accept-Encoding +< Server: MochiWeb/1.1 WebMachine/1.9.0 (participate in the frantic) +< Date: Fri, 30 Sep 2011 15:24:35 GMT +< Content-Type: text/plain +< Content-Length: 102 +< +Siblings: +16vic4eU9ny46o4KPiDz1f +4v5xOg4bVwUYZdMkqf0d6I +6nr5tDTmhxnwuAFJDd2s6G +6zRSZFUJlHXZ15o9CG0BYl +* Connection #0 to host 127.0.0.1 left intact +* Closing connection #0 +``` + +Now request one of the siblings directly: + +```curl +$ curl -v http://127.0.0.1:8098/buckets/test/keys/doc?vtag=16vic4eU9ny46o4KPiDz1f +* About to connect() to 127.0.0.1 port 8098 (#0) +* Trying 127.0.0.1... connected +* Connected to 127.0.0.1 (127.0.0.1) port 8098 (#0) +> GET /buckets/test/keys/doc?vtag=16vic4eU9ny46o4KPiDz1f HTTP/1.1 +> User-Agent: curl/7.19.4 (universal-apple-darwin10.0) libcurl/7.19.4 OpenSSL/0.9.8l zlib/1.2.3 +> Host: 127.0.0.1:8098 +> Accept: */* +> +< HTTP/1.1 200 OK +< X-Riak-Vclock: a85hYGDgyGDKBVIszMk55zKYEhnzWBlKIniO8kGF2TyvHYIKf0cIszUnMTBzHYVKbIhEUl+VK4spDFTPxhHzFyqhEoVQz7wkSAGLMGuz6FSocFIUijE3pt5HlsgCAA== +< Vary: Accept-Encoding +< Server: MochiWeb/1.1 WebMachine/1.9.0 (participate in the frantic) +< Link: ; rel="up" +< Last-Modified: Wed, 10 Mar 2010 18:01:06 GMT +< ETag: 16vic4eU9ny46o4KPiDz1f +< Date: Fri, 30 Sep 2011 15:24:35 GMT +< Content-Type: application/x-www-form-urlencoded +< Content-Length: 13 +< +* Connection #0 to host 127.0.0.1 left intact +* Closing connection #0 +{"bar":"baz"} +``` + +### Get all siblings in one request + +```curl +$ curl -v http://127.0.0.1:8098/buckets/test/keys/doc -H "Accept: multipart/mixed" +* About to connect() to 127.0.0.1 port 8098 (#0) +* Trying 127.0.0.1... connected +* Connected to 127.0.0.1 (127.0.0.1) port 8098 (#0) +> GET /buckets/test/keys/doc HTTP/1.1 +> User-Agent: curl/7.19.4 (universal-apple-darwin10.0) libcurl/7.19.4 OpenSSL/0.9.8l zlib/1.2.3 +> Host: 127.0.0.1:8098 +> Accept: multipart/mixed +> +< HTTP/1.1 300 Multiple Choices +< X-Riak-Vclock: a85hYGDgyGDKBVIszMk55zKYEhnzWBlKIniO8kGF2TyvHYIKf0cIszUnMTBzHYVKbIhEUl+VK4spDFTPxhHzFyqhEoVQz7wkSAGLMGuz6FSocFIUijE3pt5HlsgCAA== +< Vary: Accept, Accept-Encoding +< Server: MochiWeb/1.1 WebMachine/1.9.0 (participate in the frantic) +< Date: Fri, 30 Sep 2011 15:24:35 GMT +< Content-Type: multipart/mixed; boundary=YinLMzyUR9feB17okMytgKsylvh +< Content-Length: 766 +< + +--YinLMzyUR9feB17okMytgKsylvh +Content-Type: application/x-www-form-urlencoded +Link: ; rel="up" +Etag: 16vic4eU9ny46o4KPiDz1f +Last-Modified: Wed, 10 Mar 2010 18:01:06 GMT + +{"bar":"baz"} +--YinLMzyUR9feB17okMytgKsylvh +Content-Type: application/json +Link: ; rel="up" +Etag: 4v5xOg4bVwUYZdMkqf0d6I +Last-Modified: Wed, 10 Mar 2010 18:00:04 GMT + +{"bar":"baz"} +--YinLMzyUR9feB17okMytgKsylvh +Content-Type: application/json +Link: ; rel="up" +Etag: 6nr5tDTmhxnwuAFJDd2s6G +Last-Modified: Wed, 10 Mar 2010 17:58:08 GMT + +{"bar":"baz"} +--YinLMzyUR9feB17okMytgKsylvh +Content-Type: application/json +Link: ; rel="up" +Etag: 6zRSZFUJlHXZ15o9CG0BYl +Last-Modified: Wed, 10 Mar 2010 17:55:03 GMT + +{"foo":"bar"} +--YinLMzyUR9feB17okMytgKsylvh-- +* Connection #0 to host 127.0.0.1 left intact +* Closing connection #0 +``` diff --git a/content/riak/kv/2.2.6/developing/api/http/fetch-search-index.md b/content/riak/kv/2.2.6/developing/api/http/fetch-search-index.md new file mode 100644 index 0000000000..89a4017d53 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/api/http/fetch-search-index.md @@ -0,0 +1,47 @@ +--- +title: "HTTP Fetch Search Index" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Fetch Search Index" + identifier: "http_fetch_search_index" + weight: 115 + parent: "apis_http" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/references/http/fetch-search-index + - /riak-docs/riak/kv/2.2.6/dev/references/http/fetch-search-index +--- + +Retrieves information about a Riak Search [index]({{}}riak/kv/2.2.6/developing/usage/search/#simple-setup). + +## Request + +``` +GET /search/index/ +``` + +## Normal Response Codes + +* `200 OK` + +## Typical Error Codes + +* `404 Object Not Found` --- No Search index with that name is currently + available +* `503 Service Unavailable` --- The request timed out internally + +## Response + +If the index is found, Riak will output a JSON object describing the +index, including its name, the [`n_val`]({{}}riak/kv/2.2.6/developing/app-guide/replication-properties/#a-primer-on-n-r-and-w) associated with it, and the [search schema]({{}}riak/kv/2.2.6/developing/usage/search-schemas) used by the index. Here is an example: + +```json +{ + "name": "my_index", + "n_val": 3, + "schema": "_yz_default" +} +``` diff --git a/content/riak/kv/2.2.6/developing/api/http/fetch-search-schema.md b/content/riak/kv/2.2.6/developing/api/http/fetch-search-schema.md new file mode 100644 index 0000000000..512e84b57c --- /dev/null +++ b/content/riak/kv/2.2.6/developing/api/http/fetch-search-schema.md @@ -0,0 +1,38 @@ +--- +title: "HTTP Fetch Search Schema" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Fetch Search Schema" + identifier: "http_fetch_search_schema" + weight: 116 + parent: "apis_http" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/references/http/fetch-search-schema + - /riak-docs/riak/kv/2.2.6/dev/references/http/fetch-search-schema +--- + +Retrieves a Riak KV [search schema]({{}}riak/kv/2.2.6/developing/usage/search-schemas). + +## Request + +``` +GET /search/schema/ +``` + +## Normal Response Codes + +* `200 OK` + +## Typical Error Codes + +* `404 Object Not Found` +* `503 Service Unavailable` --- The request timed out internally + +## Response + +If the schema is found, Riak will return the contents of the schema as +XML (all Riak Search schemas are XML). diff --git a/content/riak/kv/2.2.6/developing/api/http/get-bucket-props.md b/content/riak/kv/2.2.6/developing/api/http/get-bucket-props.md new file mode 100644 index 0000000000..63c71cc3fe --- /dev/null +++ b/content/riak/kv/2.2.6/developing/api/http/get-bucket-props.md @@ -0,0 +1,82 @@ +--- +title: "HTTP Get Bucket Properties" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Get Bucket Properties" + identifier: "http_get_bucket_props" + weight: 100 + parent: "apis_http" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/references/http/get-bucket-props + - /riak-docs/riak/kv/2.2.6/dev/references/http/get-bucket-props +--- + +Reads the bucket or bucket type properties. + +## Request + +```bash +GET /buckets/bucket/props +``` + +Or, to read bucket properties from a bucket in a bucket type: + +```bash +GET /types/type/buckets/bucket/props +``` + +Optional query parameters (only valid for the old format): + +* `props` - whether to return the bucket properties (`true` is the default) +* `keys` - whether to return the keys stored in the bucket. (`false` is the +default). See also [HTTP List Keys]({{}}riak/kv/2.2.6/developing/api/http/list-keys). + +## Response + +Normal status codes: + +* `200 OK` + +Important headers: + +* `Content-Type` - `application/json` + +The JSON object in the response will contain up to two entries, `"props"` and +`"keys"`, which are present or missing, according to the optional query +parameters. The default is for only `"props"` to be present. + +See [HTTP Set Bucket Properties]({{}}riak/kv/2.2.6/developing/api/http/set-bucket-props) for more information about the available +bucket properties. See [Managing Bucket Types Through the Command Line](http://docs.basho.com/riak/kv/2.2.0/using/reference/bucket-types/#managing-bucket-types-through-the-command-line) for more details about reading bucket types using the `riak-admin bucket-type` interface. + +## Example + +```curl +$ curl -v http://127.0.0.1:8098/buckets/test/props +* About to connect() to 127.0.0.1 port 8098 (#0) +* Trying 127.0.0.1... connected +* Connected to 127.0.0.1 (127.0.0.1) port 8098 (#0) +> GET /buckets/test/props HTTP/1.1 +> User-Agent: curl/7.19.7 (universal-apple-darwin10.0) libcurl/7.19.7 +OpenSSL/0.9.8l zlib/1.2.3 +> Host: 127.0.0.1:8098 +> Accept: */* +> +< HTTP/1.1 200 OK +< Vary: Accept-Encoding +< Server: MochiWeb/1.1 WebMachine/1.9.0 (participate in the frantic) +< Date: Fri, 30 Sep 2011 15:24:35 GMT +< Content-Type: application/json +< Content-Length: 368 +< +* Connection #0 to host 127.0.0.1 left intact +* Closing connection #0 +{"props":{"name":"test","n_val":3,"allow_mult":false,"last_write_wins":false," +precommit":[],"postcommit":[],"chash_keyfun":{"mod":"riak_core_util","fun":" +chash_std_keyfun"},"linkfun":{"mod":"riak_kv_wm_link_walker","fun":" +mapreduce_linkfun"},"old_vclock":86400,"young_vclock":20,"big_vclock":50," +small_vclock":10,"r":"quorum","w":"quorum","dw":"quorum","rw":"quorum"}} +``` diff --git a/content/riak/kv/2.2.6/developing/api/http/link-walking.md b/content/riak/kv/2.2.6/developing/api/http/link-walking.md new file mode 100644 index 0000000000..c812e959a8 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/api/http/link-walking.md @@ -0,0 +1,125 @@ +--- +title: "HTTP Link Walking" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Link Walking" + identifier: "http_link_walking" + weight: 118 + parent: "apis_http" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/references/http/link-walking + - /riak-docs/riak/kv/2.2.6/dev/references/http/link-walking +--- + +{{% note title="Deprecation Warning" %}} +This feature is deprecated and will be removed in a future version. +{{% /note %}} + +Link walking (traversal) finds and returns objects by following links attached +to them, starting from the object specified by the bucket and key portion. It +is a special case of [MapReduce]({{}}riak/kv/2.2.6/developing/usage/mapreduce), and can be expressed more verbosely as such. +[Read more about Links]({{}}riak/kv/2.2.6/learn/glossary/#links). + +## Request + +```bash +GET /buckets/bucket/keys/key/[bucket],[tag],[keep] +``` + +{{% note title="Link filters" %}} +A link filter within the request URL is made of three parts, separated by +commas: + +* Bucket - a bucket name to limit the links to +* Tag - a "riaktag" to limit the links to +* Keep - 0 or 1, whether to return results from this phase + +Any of the three parts may be replaced with `_` (underscore), signifying that +any value is valid. Multiple phases of links can be followed by adding +additional path segments to the URL, separating the link filters by slashes. +The final phase in the link-walking query implicitly returns its results. +{{% /note %}} + +## Response + +Normal status codes: + +* `200 OK` + +Typical error codes: + +* `400 Bad Request` - if the format of the query in the URL is invalid +* `404 Not Found` - if the origin object of the walk was missing + +Important headers: + +* `Content-Type` - always `multipart/mixed`, with a boundary specified + +> **Understanding the response body** +> +> The response body will always be `multipart/mixed`, with each +chunk representing a single phase of the link-walking query. Each phase will +also be encoded in `multipart/mixed`, with each chunk representing a +single object that was found. If no objects were found or "keep" was not set on +the phase, no chunks will be present in that phase. Objects inside phase +results will include `Location` headers that can be used to determine +bucket and key. In fact, you can treat each object-chunk similarly to a complete +response from [fetching the object]({{}}riak/kv/2.2.6/developing/api/http/fetch-object), without the status +code. + +## Example + +```curl +$ curl -v http://127.0.0.1:8098/riak/test/doc3/test,_,1/_,next,1 +* About to connect() to 127.0.0.1 port 8098 (#0) +* Trying 127.0.0.1... connected +* Connected to 127.0.0.1 (127.0.0.1) port 8098 (#0) +> GET /riak/test/doc3/test,_,1/_,next,1 HTTP/1.1 +> User-Agent: curl/7.19.4 (universal-apple-darwin10.0) libcurl/7.19.4 OpenSSL/0.9.8l zlib/1.2.3 +> Host: 127.0.0.1:8098 +> Accept: */* +> +< HTTP/1.1 200 OK +< Server: MochiWeb/1.1 WebMachine/1.9.0 (participate in the frantic) +< Expires: Wed, 10 Mar 2010 20:24:49 GMT +< Date: Fri, 30 Sep 2011 15:24:35 GMT +< Content-Type: multipart/mixed; boundary=JZi8W8pB0Z3nO3odw11GUB4LQCN +< Content-Length: 970 +< + +--JZi8W8pB0Z3nO3odw11GUB4LQCN +Content-Type: multipart/mixed; boundary=OjZ8Km9J5vbsmxtcn1p48J91cJP + +--OjZ8Km9J5vbsmxtcn1p48J91cJP +X-Riak-Vclock: a85hYGDgymDKBVIszMk55zKYEhnzWBlKIniO8kGF2TyvHYIKf0cIszUnMTBzHYVKbIhEUl+VK4spDFTPxhHzFyqhEoVQz7wkSAGLMGuz6FSocFIUijE3pt7HlGBhnqejARXmq0QyZnnxE6jwVJBwFgA= +Location: /riak/test/doc +Content-Type: application/json +Link: ; rel="up", ; riaktag="next" +Etag: 3pvmY35coyWPxh8mh4uBQC +Last-Modified: Wed, 10 Mar 2010 20:14:13 GMT + +{"riak":"CAP"} +--OjZ8Km9J5vbsmxtcn1p48J91cJP-- + +--JZi8W8pB0Z3nO3odw11GUB4LQCN +Content-Type: multipart/mixed; boundary=RJKFlAs9PrdBNfd74HANycvbA8C + +--RJKFlAs9PrdBNfd74HANycvbA8C +X-Riak-Vclock: a85hYGBgzGDKBVIsbLvm1WYwJTLmsTLcjeE5ypcFAA== +Location: /riak/test/doc2 +Content-Type: application/json +Link: ; rel="up" +Etag: 6dQBm9oYA1mxRSH0e96l5W +Last-Modified: Wed, 10 Mar 2010 18:11:41 GMT + +{"foo":"bar"} +--RJKFlAs9PrdBNfd74HANycvbA8C-- + +--JZi8W8pB0Z3nO3odw11GUB4LQCN-- +* Connection #0 to host 127.0.0.1 left intact +* Closing connection #0 +``` diff --git a/content/riak/kv/2.2.6/developing/api/http/list-buckets.md b/content/riak/kv/2.2.6/developing/api/http/list-buckets.md new file mode 100644 index 0000000000..6d08c675c0 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/api/http/list-buckets.md @@ -0,0 +1,64 @@ +--- +title: "HTTP List Buckets" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "List Buckets" + identifier: "http_list_buckets" + weight: 103 + parent: "apis_http" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/references/http/list-buckets + - /riak-docs/riak/kv/2.2.6/dev/references/http/list-buckets +--- + +Lists all known buckets (ones that have keys stored in them). + +{{% note title="Not for production use" %}} +Similar to the list keys operation, this requires traversing all keys stored +in the cluster and should not be used in production. +{{% /note %}} + +## Request + +```bash +# Using the default bucket type +GET /buckets?buckets=true + +# Using a non-default bucket type +GET /types//buckets?buckets=true +``` + +Required query parameter: + +* **buckets=true** - required to invoke the list-buckets functionality + +## Response + +Normal status codes: + +* `200 OK` + +Important headers: + +* `Content-Type - application/json` + +The JSON object in the response will contain a single entry, "buckets", which +will be an array of bucket names. + +## Example + +```curl +$ curl -i http://localhost:8098/buckets?buckets=true +HTTP/1.1 200 OK +Vary: Accept-Encoding +Server: MochiWeb/1.1 WebMachine/1.9.0 (participate in the frantic) +Date: Fri, 30 Sep 2011 15:24:35 GMT +Content-Type: application/json +Content-Length: 21 + +{"buckets":["files"]} +``` diff --git a/content/riak/kv/2.2.6/developing/api/http/list-keys.md b/content/riak/kv/2.2.6/developing/api/http/list-keys.md new file mode 100644 index 0000000000..515548fe4f --- /dev/null +++ b/content/riak/kv/2.2.6/developing/api/http/list-keys.md @@ -0,0 +1,76 @@ +--- +title: "HTTP List Keys" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "List Keys" + identifier: "http_list_keys" + weight: 104 + parent: "apis_http" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/references/http/list-keys + - /riak-docs/riak/kv/2.2.6/dev/references/http/list-keys +--- + +Lists keys in a bucket. + +{{% note title="Not for production use" %}} +This operation requires traversing all keys stored in the cluster and should +not be used in production. +{{% /note %}} + +## Request + +```bash +# Using the default bucket type +GET /buckets/bucket/keys?keys=true # List all keys +GET /buckets/bucket/keys?keys=stream # Stream keys to the client + +# Using a non-default bucket type +GET /types//buckets/bucket/keys?keys=true +GET /types//buckets/bucket/keys?keys=stream +``` + +Required query parameters: + +* `keys` - defaults to `false`. When set to `true` all keys will be returned in +a single payload. When set to `stream`, keys will be returned in +chunked-encoding. + +## Response + +Normal response codes: + +* `200 OK` + +Important headers: + +* `Content-Type` - `application/json` +* `Transfer-Encoding` - `chunked` when the `keys` query parameter is set to +`stream`. + +The JSON object in the response will contain up to two entries, +`"props"` and `"keys"` which are present or missing according to the +query parameters and format used. If `keys=stream` in the query +parameters, multiple JSON objects in chunked-encoding will be returned +containing `"keys"` entries. + +## Example + +```curl +$ curl -i http://localhost:8098/buckets/jsconf/keys?keys=true +HTTP/1.1 200 OK +Vary: Accept-Encoding +Server: MochiWeb/1.1 WebMachine/1.9.0 (participate in the frantic) +Date: Fri, 30 Sep 2011 15:24:35 GMT +Content-Type: application/json +Content-Length: 239 + +{"keys":["challenge.jpg","puddi.png","basho.gif","puddikid.jpg","yay.png"," +thinking.png","victory.gif","slides","joyent.png","seancribbs-small.jpg"," +trollface.jpg","riak_logo_animated1.gif","victory.jpg","challenge.png"," +team_cribbs.png"]} +``` diff --git a/content/riak/kv/2.2.6/developing/api/http/list-resources.md b/content/riak/kv/2.2.6/developing/api/http/list-resources.md new file mode 100644 index 0000000000..a64137ac0f --- /dev/null +++ b/content/riak/kv/2.2.6/developing/api/http/list-resources.md @@ -0,0 +1,80 @@ +--- +title: "HTTP List Resources" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "List Resources" + identifier: "http_list_resources" + weight: 112 + parent: "apis_http" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/references/http/list-resources + - /riak-docs/riak/kv/2.2.6/dev/references/http/list-resources +--- + +List available HTTP resources for the Riak node. This can be used by clients to +automatically recognize the location of the resources for specific operations. + +The standard resources are: + +* `riak_kv_wm_buckets` - [Bucket Operations]({{}}riak/kv/2.2.6/developing/api/http/#bucket-operations) +* `riak_kv_wm_index` - [HTTP Secondary Indexes]({{}}riak/kv/2.2.6/developing/api/http/secondary-indexes) +* `riak_kv_wm_link_walker` - [HTTP Link Walking]({{}}riak/kv/2.2.6/developing/api/http/link-walking) +* `riak_kv_wm_mapred` - [HTTP MapReduce]({{}}riak/kv/2.2.6/developing/api/http/mapreduce) +* `riak_kv_wm_object`- [Object/Key Operations]({{}}riak/kv/2.2.6/developing/api/http/#object-key-operations) +* `riak_kv_wm_ping` - [HTTP Ping]({{}}riak/kv/2.2.6/developing/api/http/ping) +* `riak_kv_wm_props` - [HTTP Set Bucket Properties]({{}}riak/kv/2.2.6/developing/api/http/set-bucket-props) +* `riak_kv_wm_stats` - [HTTP Status]({{}}riak/kv/2.2.6/developing/api/http/status) + +## Request + +```bash +GET / +``` + +Headers: + +* `Accept` - `application/json` or `text/html` + +## Response + +Normal status codes: + +* `200 OK` + +Important headers: + +* `Link` - all resources that are described in the response body, but in Link +form + +## Example + +Request JSON response + +```curl +$ curl -i http://localhost:8098 -H "Accept: application/json" +HTTP/1.1 200 OK +Vary: Accept +Server: MochiWeb/1.1 WebMachine/1.10.0 (never breaks eye contact) +Link: ; rel="riak_kv_wm_buckets",; rel="riak_kv_wm_buckets",; rel="riak_kv_wm_counter",; rel="riak_kv_wm_index",; rel="riak_kv_wm_keylist",; rel="riak_kv_wm_link_walker",; rel="riak_kv_wm_link_walker",; rel="riak_kv_wm_mapred",; rel="riak_kv_wm_object",; rel="riak_kv_wm_object",; rel="riak_kv_wm_ping",; rel="riak_kv_wm_props",; rel="riak_kv_wm_stats" +Date: Wed, 27 Nov 2013 20:18:31 GMT +Content-Type: application/json +Content-Length: 398 + +{"riak_kv_wm_buckets":"/buckets","riak_kv_wm_buckets":"/riak","riak_kv_wm_counter":"/buckets","riak_kv_wm_index":"/buckets","riak_kv_wm_keylist":"/buckets","riak_kv_wm_link_walker":"/buckets","riak_kv_wm_link_walker":"/riak","riak_kv_wm_mapred":"/mapred","riak_kv_wm_object":"/buckets","riak_kv_wm_object":"/riak","riak_kv_wm_ping":"/ping","riak_kv_wm_props":"/buckets","riak_kv_wm_stats":"/stats"} + +# Request HTML response +curl -i http://localhost:8098 -H "Accept: text/html" +HTTP/1.1 200 OK +Vary: Accept +Server: MochiWeb/1.1 WebMachine/1.10.0 (never breaks eye contact) +Link: ; rel="riak_kv_wm_buckets",; rel="riak_kv_wm_buckets",; rel="riak_kv_wm_counter",; rel="riak_kv_wm_index",; rel="riak_kv_wm_keylist",; rel="riak_kv_wm_link_walker",; rel="riak_kv_wm_link_walker",; rel="riak_kv_wm_mapred",; rel="riak_kv_wm_object",; rel="riak_kv_wm_object",; rel="riak_kv_wm_ping",; rel="riak_kv_wm_props",; rel="riak_kv_wm_stats" +Date: Wed, 27 Nov 2013 20:20:05 GMT +Content-Type: text/html +Content-Length: 666 + + +``` diff --git a/content/riak/kv/2.2.6/developing/api/http/mapreduce.md b/content/riak/kv/2.2.6/developing/api/http/mapreduce.md new file mode 100644 index 0000000000..cce17cd099 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/api/http/mapreduce.md @@ -0,0 +1,70 @@ +--- +title: "HTTP MapReduce" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "MapReduce" + identifier: "http_mapreduce" + weight: 108 + parent: "apis_http" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/references/http/mapreduce + - /riak-docs/riak/kv/2.2.6/dev/references/http/mapreduce +--- + +[MapReduce]({{}}riak/kv/2.2.6/developing/usage/mapreduce) is a generic way to query Riak by specifying inputs and constructing a set of map, reduce, and link phases through which data will flow. + +## Request + +```bash +POST /mapred +``` + +Important headers: +* `Content-Type` - must always be `application/json`. The format of the request body is described in detail on the [MapReduce]({{}}riak/kv/2.2.6/developing/usage/mapreduce) page. + +Optional query parameters: +* `chunked` - when set to `true`, results will be returned as they are received in `multipart/mixed` format using chunked-encoding. + +_+This request must include an entity (body), which is the JSON form of the MapReduce query.+_ + +## Response + +Normal status codes: +* `200 OK` + +Typical error codes: +* `400 Bad Request` - if an invalid job is submitted. +* `500 Internal Server Error` - if there was an error in processing a map or reduce function +* `503 Service Unavailable` - if the job timed out before it could complete + +Important headers: +* `Content-Type` - `application/json` when `chunked` is not true, otherwise `multipart/mixed` with `application/json` sections. + +## Example + +```curl +$ curl -v -d '{"inputs":"test", "query":[{"link":{"bucket":"test"}},{"map":{"language":"javascript","name":"Riak.mapValuesJson"}}]}' -H "Content-Type: application/json" http://127.0.0.1:8098/mapred +* About to connect() to 127.0.0.1 port 8098 (#0) +* Trying 127.0.0.1... connected +* Connected to 127.0.0.1 (127.0.0.1) port 8098 (#0) +> POST /mapred HTTP/1.1 +> User-Agent: curl/7.19.4 (universal-apple-darwin10.0) libcurl/7.19.4 OpenSSL/0.9.8l zlib/1.2.3 +> Host: 127.0.0.1:8098 +> Accept: */* +> Content-Type: application/json +> Content-Length: 117 +> +< HTTP/1.1 200 OK +< Server: MochiWeb/1.1 WebMachine/1.9.0 (participate in the frantic) +< Date: Fri, 30 Sep 2011 15:24:35 GMT +< Content-Type: application/json +< Content-Length: 30 +< +* Connection #0 to host 127.0.0.1 left intact +* Closing connection #0 +[{"foo":"bar"},{"riak":"CAP"}] +``` diff --git a/content/riak/kv/2.2.6/developing/api/http/ping.md b/content/riak/kv/2.2.6/developing/api/http/ping.md new file mode 100644 index 0000000000..5d5402ed09 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/api/http/ping.md @@ -0,0 +1,53 @@ +--- +title: "HTTP Ping" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Ping" + identifier: "http_ping" + weight: 110 + parent: "apis_http" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/references/http/ping + - /riak-docs/riak/kv/2.2.6/dev/references/http/ping +--- + +Checks if the server is alive. This is useful for monitoring tools, load-balancers and automated scripts. + +## Request + +```bash +GET /ping +``` + +## Response + +Normal status codes: + +* `200 OK` + +## Example + +```curl +$ curl -v http://127.0.0.1:8098/ping +* About to connect() to 127.0.0.1 port 8098 (#0) +* Trying 127.0.0.1... connected +* Connected to 127.0.0.1 (127.0.0.1) port 8098 (#0) +> GET /ping HTTP/1.1 +> User-Agent: curl/7.19.7 (universal-apple-darwin10.0) libcurl/7.19.7 OpenSSL/0.9.8l zlib/1.2.3 +> Host: 127.0.0.1:8098 +> Accept: */* +> +< HTTP/1.1 200 OK +< Server: MochiWeb/1.1 WebMachine/1.9.0 (participate in the frantic) +< Date: Fri, 30 Sep 2011 15:24:35 GMT +< Content-Type: text/html +< Content-Length: 2 +< +* Connection #0 to host 127.0.0.1 left intact +* Closing connection #0 +OK +``` diff --git a/content/riak/kv/2.2.6/developing/api/http/reset-bucket-props.md b/content/riak/kv/2.2.6/developing/api/http/reset-bucket-props.md new file mode 100644 index 0000000000..37f87cd78c --- /dev/null +++ b/content/riak/kv/2.2.6/developing/api/http/reset-bucket-props.md @@ -0,0 +1,57 @@ +--- +title: "HTTP Reset Bucket Properties" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Reset Bucket Properties" + identifier: "http_reset_bucket_props" + weight: 102 + parent: "apis_http" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/references/http/reset-bucket-props + - /riak-docs/riak/kv/2.2.6/dev/references/http/reset-bucket-props +--- + +Resets bucket properties like `n_val` and `allow_mult` back to the +default settings. + +## Request + +```bash +DELETE /buckets/bucket/props +``` + +Resetting bucket properties is not available via the old API format. + +## Response + +Normal status codes: + +* `204 No Content` + +## Example + +```curl +$ curl -XDELETE -v localhost:8098/buckets/bucket/props {13:47} +* About to connect() to localhost port 8098 (#0) +* Trying 127.0.0.1... +* connected +* Connected to localhost (127.0.0.1) port 8098 (#0) +> DELETE /buckets/bucket/props HTTP/1.1 +> User-Agent: curl/7.24.0 (x86_64-apple-darwin12.0) libcurl/7.24.0 OpenSSL/0.9.8r zlib/1.2.5 +> Host: localhost:8098 +> Accept: */* +> +< HTTP/1.1 204 No Content +< Vary: Accept-Encoding +< Server: MochiWeb/1.1 WebMachine/1.9.2 (someone had painted it blue) +< Date: Tue, 06 Nov 2012 21:56:17 GMT +< Content-Type: application/json +< Content-Length: 0 +< +* Connection #0 to host localhost left intact +* Closing connection #0 +``` diff --git a/content/riak/kv/2.2.6/developing/api/http/search-index-info.md b/content/riak/kv/2.2.6/developing/api/http/search-index-info.md new file mode 100644 index 0000000000..611577f6db --- /dev/null +++ b/content/riak/kv/2.2.6/developing/api/http/search-index-info.md @@ -0,0 +1,52 @@ +--- +title: "HTTP Search Index Info" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Search Index Info" + identifier: "http_search_index_info" + weight: 114 + parent: "apis_http" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/references/http/search-index-info + - /riak-docs/riak/kv/2.2.6/dev/references/http/search-index-info +--- + +Retrieves information about all currently available [Search indexes]({{}}riak/kv/2.2.6/developing/usage/search) in JSON format. + +## Request + +``` +GET /search/index +``` + +## Response + +If there are no currently available Search indexes, a `200 OK` will be +returned but with an empty list as the response value. + +Below is the example output if there is one Search index, called +`test_index`, currently available: + +```json +[ + { + "n_val": 3, + "name": "test_index", + "schema": "_yz_default" + } +] +``` + +#### Normal Response Codes + +* `200 OK` + +#### Typical Error Codes + +* `404 Object Not Found` --- Typically returned if Riak Search is not + currently enabled on the node +* `503 Service Unavailable` --- The request timed out internally diff --git a/content/riak/kv/2.2.6/developing/api/http/search-query.md b/content/riak/kv/2.2.6/developing/api/http/search-query.md new file mode 100644 index 0000000000..44960b8db6 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/api/http/search-query.md @@ -0,0 +1,69 @@ +--- +title: "HTTP Search Query" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Search Query" + identifier: "http_search_query" + weight: 113 + parent: "apis_http" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/references/http/search-query + - /riak-docs/riak/kv/2.2.6/dev/references/http/search-query +--- + +Performs a [Riak KV Search]({{}}riak/kv/2.2.6/developing/usage/search) query. + +## Request + +``` +GET /search/query/ +``` + +## Optional Query Parameters + +* `wt` --- The [response + writer](https://cwiki.apache.org/confluence/display/solr/Response+Writers) + to be used when returning the Search payload. The currently + available options are `json` and `xml`. The default is `xml`. +* `q` --- The actual Search query itself. Examples can be found in + [Using Search]({{}}riak/kv/2.2.6/developing/usage/search). If a query is not specified, Riak will return + information about the index itself, e.g. the number of documents + indexed. + +## Normal Response Codes + +* `200 OK` + +## Typical Error Codes + +* `400 Bad Request` --- Returned when, for example, a malformed query is + supplied +* `404 Object Not Found` --- Returned if the Search index you are + attempting to query does not exist +* `503 Service Unavailable` --- The request timed out internally + +## Response + +If a `200 OK` is returned, then the Search query has been successful. +Below is an example JSON response from querying an index that currently +has no documents associated with it: + +```json +{ + "response": { + "docs": [], + "maxScore": 0.0, + "numFound": 0, + "start": 0 + }, + "responseHeader": { + "status": 0, + "QTime": 10, + "params": { /* internal info from the query */ } + } +} +``` diff --git a/content/riak/kv/2.2.6/developing/api/http/secondary-indexes.md b/content/riak/kv/2.2.6/developing/api/http/secondary-indexes.md new file mode 100644 index 0000000000..58228c6641 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/api/http/secondary-indexes.md @@ -0,0 +1,91 @@ +--- +title: "HTTP Secondary Indexes" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Secondary Indexes" + identifier: "http_2i" + weight: 109 + parent: "apis_http" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/references/http/secondary-indexes + - /riak-docs/riak/kv/2.2.6/dev/references/http/secondary-indexes +--- + +[Secondary Indexes]({{}}riak/kv/2.2.6/developing/usage/secondary-indexes) allows an application to tag a Riak object with one or more field/value pairs. The object is indexed under these field/value pairs, and the application can later query the index to retrieve a list of matching keys. + +## Request + +### Exact Match + +```bash +GET /buckets/mybucket/index/myindex_bin/value +``` + +### Range Query + +``` +GET /buckets/mybucket/index/myindex_bin/start/end +``` + +#### Range query with terms + +To see the index values matched by the range, use `return_terms=true`. + +``` +GET /buckets/mybucket/index/myindex_bin/start/end?return_terms=true +``` + +### Pagination + +Add the parameter `max_results` for pagination. This will limit the results and provide for the next request a `continuation` value. + +``` +GET /buckets/mybucket/index/myindex_bin/start/end?return_terms=true&max_results=500 +GET /buckets/mybucket/index/myindex_bin/start/end?return_terms=true&max_results=500&continuation=g2gCbQAAAAdyaXBqYWtlbQAAABIzNDkyMjA2ODcwNTcxMjk0NzM= +``` + +### Streaming + +``` +GET /buckets/mybucket/index/myindex_bin/start/end?stream=true +``` + +## Response + +Normal status codes: + ++ `200 OK` + +Typical error codes: + ++ `400 Bad Request` - if the index name or index value is invalid. ++ `500 Internal Server Error` - if there was an error in processing a map or reduce function, or if indexing is not supported by the system. ++ `503 Service Unavailable` - if the job timed out before it could complete + +## Example + +```curl +$ curl -v http://localhost:8098/buckets/mybucket/index/field1_bin/val1 +* About to connect() to localhost port 8098 (#0) +* Trying 127.0.0.1... connected +* Connected to localhost (127.0.0.1) port 8098 (#0) +> GET /buckets/mybucket/index/field1_bin/val1 HTTP/1.1 +> User-Agent: curl/7.19.7 (universal-apple-darwin10.0) libcurl/7.19.7 OpenSSL/0.9.8r zlib/1.2.3 +> Host: localhost:8098 +> Accept: */* +> +< HTTP/1.1 200 OK +< Vary: Accept-Encoding +< Server: MochiWeb/1.1 WebMachine/1.9.0 (participate in the frantic) +< Date: Fri, 30 Sep 2011 15:24:35 GMT +< Content-Type: application/json +< Content-Length: 19 +< +* Connection #0 to host localhost left intact +* Closing connection #0 +{"keys":["mykey1"]}% +``` diff --git a/content/riak/kv/2.2.6/developing/api/http/set-bucket-props.md b/content/riak/kv/2.2.6/developing/api/http/set-bucket-props.md new file mode 100644 index 0000000000..624deb7a90 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/api/http/set-bucket-props.md @@ -0,0 +1,101 @@ +--- +title: "HTTP Set Bucket Properties" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Set Bucket Properties" + identifier: "http_set_bucket_props" + weight: 101 + parent: "apis_http" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/references/http/set-bucket-props + - /riak-docs/riak/kv/2.2.6/dev/references/http/set-bucket-props +--- + +Sets bucket properties like "n_val" and "allow_mult". + +## Request + +```bash +PUT /buckets/bucket/props +``` + +Important headers: + +* `Content-Type` - `application/json` + +The body of the request should be a JSON object with a single entry "props". +Unmodified bucket properties may be omitted. + +Available properties: + +* `n_val` (integer > 0) - the number of replicas for objects in this bucket +* `allow_mult` (true or false) - whether to allow sibling objects to be created +(concurrent updates) +* `last_write_wins` (true or false) - whether to ignore object history (vector +clock) when writing +* `precommit` - [precommit hooks]({{}}riak/kv/2.2.6/developing/usage/commit-hooks) +* `postcommit` - [postcommit hooks]({{}}riak/kv/2.2.6/developing/usage/commit-hooks) +* `r, w, dw, rw` - default quorum values for operations on keys in the bucket. +Valid values are: + * `"all"` - all nodes must respond + * `"quorum"` - (n_val/2) + 1 nodes must respond. *This is the default.* + * `"one"` - equivalent to 1 + * *Any integer* - must be less than or equal to n_val +* `backend` - when using `riak_kv_multi_backend`, which named backend to use for +the bucket +* `node_confirms` - declares the number of diverse physical node acks required for a write +to be successful + +Other properties do exist but are not commonly modified. + +{{% note title="Property types" %}} +Make sure you use the proper types for attributes like **n_val** and +**allow_mult**. If you use strings instead of integers and booleans +respectively, you may see some odd errors in your logs, saying something like +`"{badarith,[{riak_kv_util,normalize_rw_value,2},]}"`. +{{% /note %}} + +## Response + +Normal status codes: + +* `204 No Content` + +Typical error codes: + +* `400 Bad Request` - if the submitted JSON is invalid +* `415 Unsupported Media Type` - if the Content-Type was not set to +application/json in the request + +If successful, no content will be returned in the response body. + +## Example + +```curl +$ curl -v -XPUT http://127.0.0.1:8098/buckets/test/props \ + -H "Content-Type: application/json" -d '{"props":{"n_val":5}}' +* About to connect() to 127.0.0.1 port 8098 (#0) +* Trying 127.0.0.1... connected +* Connected to 127.0.0.1 (127.0.0.1) port 8098 (#0) +> PUT /buckets/test/props HTTP/1.1 +> User-Agent: curl/7.19.4 (universal-apple-darwin10.0) libcurl/7.19.4 +OpenSSL/0.9.8l zlib/1.2.3 +> Host: 127.0.0.1:8098 +> Accept: */* +> Content-Type: application/json +> Content-Length: 21 +> +< HTTP/1.1 204 No Content +< Vary: Accept-Encoding +< Server: MochiWeb/1.1 WebMachine/1.9.0 (participate in the frantic) +< Date: Fri, 30 Sep 2011 15:24:35 GMT +< Content-Type: application/json +< Content-Length: 0 +< +* Connection #0 to host 127.0.0.1 left intact +* Closing connection #0 +``` diff --git a/content/riak/kv/2.2.6/developing/api/http/status.md b/content/riak/kv/2.2.6/developing/api/http/status.md new file mode 100644 index 0000000000..b1704cea6f --- /dev/null +++ b/content/riak/kv/2.2.6/developing/api/http/status.md @@ -0,0 +1,169 @@ +--- +title: "HTTP Status" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Status" + identifier: "http_status" + weight: 111 + parent: "apis_http" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/references/http/status + - /riak-docs/riak/kv/2.2.6/dev/references/http/status +--- + +Reports about the performance and configuration of the Riak node to which it was requested. You must have the `{riak_kv_stat,true}` configuration setting in app.config for this endpoint to be active. + +## Performance + +Repeated requests to the `/stats` endpoint do not have a negative +performance impact as the statistics are cached internally in Riak. + +## Request + +```bash +GET /stats +``` + +Important headers: + +* `Accept` - determines whether the response will be formatted in `application/json` or `text/plain`. + +## Response + +Normal status codes: +* `200 OK` + +Typical error codes: +* `404 Not Found` - if `riak_kv_stat` is not enabled + +Important headers: +* `Content-Type` - `application/json` or `text/plain` (JSON with added line-breaks) + +## Example + +```curl +$ curl -v http://127.0.0.1:8098/stats -H "Accept: text/plain" +* About to connect() to 127.0.0.1 port 8098 (#0) +* Trying 127.0.0.1... connected +* Connected to 127.0.0.1 (127.0.0.1) port 8098 (#0) +> GET /stats HTTP/1.1 +> User-Agent: curl/7.19.7 (universal-apple-darwin10.0) libcurl/7.19.7 OpenSSL/0.9.8l zlib/1.2.3 +> Host: 127.0.0.1:8098 +> Accept: text/plain +> +< HTTP/1.1 200 OK +< Vary: Accept, Accept-Encoding +< Server: MochiWeb/1.1 WebMachine/1.9.0 (participate in the frantic) +< Date: Fri, 30 Sep 2011 15:24:35 GMT +< Content-Type: text/plain +< Content-Length: 2102 +< +{ + "vnode_gets": 0, + "vnode_puts": 0, + "read_repairs": 0, + "vnode_gets_total": 0, + "vnode_puts_total": 0, + "node_gets": 0, + "node_gets_total": 0, + "node_get_fsm_time_mean": "undefined", + "node_get_fsm_time_median": "undefined", + "node_get_fsm_time_95": "undefined", + "node_get_fsm_time_99": "undefined", + "node_get_fsm_time_100": "undefined", + "node_puts": 0, + "node_puts_total": 0, + "node_put_fsm_time_mean": "undefined", + "node_put_fsm_time_median": "undefined", + "node_put_fsm_time_95": "undefined", + "node_put_fsm_time_99": "undefined", + "node_put_fsm_time_100": "undefined", + "read_repairs_total": 0, + "cpu_nprocs": 84, + "cpu_avg1": 251, + "cpu_avg5": 174, + "cpu_avg15": 110, + "mem_total": 7946684000.0, + "mem_allocated": 4340880000.0, + "nodename": "riak@127.0.0.1", + "connected_nodes": [ + + ], + "sys_driver_version": "1.5", + "sys_global_heaps_size": 0, + "sys_heap_type": "private", + "sys_logical_processors": 2, + "sys_otp_release": "R13B04", + "sys_process_count": 189, + "sys_smp_support": true, + "sys_system_version": "Erlang R13B04 (erts-5.7.5) [[source]] [[64-bit]] [[smp:2:2]] [[rq:2]] [[async-threads:5]] [[hipe]] [[kernel-poll:true]]", + "sys_system_architecture": "i386-apple-darwin10.3.0", + "sys_threads_enabled": true, + "sys_thread_pool_size": 5, + "sys_wordsize": 8, + "ring_members": [ + "riak@127.0.0.1" + ], + "ring_num_partitions": 64, + "ring_ownership": "[{'riak@127.0.0.1',64}]", + "ring_creation_size": 64, + "storage_backend": "riak_kv_bitcask_backend", + "pbc_connects_total": 0, + "pbc_connects": 0, + "pbc_active": 0, + "riak_kv_version": "0.11.0", + "riak_core_version": "0.11.0", + "bitcask_version": "1.0.1", + "luke_version": "0.1", + "webmachine_version": "1.7.1", + "mochiweb_version": "1.7.1", + "erlang_js_version": "0.4", + "runtime_tools_version": "1.8.3", + "crypto_version": "1.6.4", + "os_mon_version": "2.2.6", + "sasl_version": "2.1.9", + "stdlib_version": "1.16.5", + "kernel_version": "2.13.5" +} +* Connection #0 to host 127.0.0.1 left intact +* Closing connection #0 +``` + +## Output Explanation + +The output of `/stats` contains the output of `riak-admin status` detailed in the [Inspecting a Node]({{}}riak/kv/2.2.6/using/cluster-operations/inspecting-node) doc, plus the below stats generated by the Riak Core application. + +Stat | Description +------------------------------|--------------------------------------------------- +riak_core_stat_ts | The last time (in Epoch time) Riak Core stats were generated +ignored_gossip_total | Total number of ignored gossip messages since node was started +rings_reconciled_total | Total number of ring reconciliation operations since node was started +rings_reconciled | Number of ring reconciliation operations in the last minute +gossip_received | Number of gossip messages received in the last minute +rejected_handoffs | Total number of ownership handoff operations rejected by the node since it was started +handoff_timeouts | Total number of handoff timeouts encountered by this node since it was started +dropped_vnode_requests_total | Total number of requests dropped by local vnodes since the node was started +converge_delay_min | Minimum time in milliseconds describing time taken for the ring to converge after ring changes +converge_delay_max | Maximum time in milliseconds describing time taken for the ring to converge after ring changes +converge_delay_mean | Mean time in milliseconds describing time taken for the ring to converge after ring changes +converge_delay_last | Last observed histogram value in milliseconds describing time taken for the ring to converge after ring changes +rebalance_delay_min | Minimum time in milliseconds taken to calculate partition rebalance during a cluster membership change +rebalance_delay_max | Maximum time in milliseconds taken to calculate partition rebalance during a cluster membership change +rebalance_delay_mean | Mean time in milliseconds describing time taken for the ring to converge after ring changes +rebalance_delay_last | Last observed histogram value in milliseconds describing time taken for the ring to converge after ring changes +riak_kv_vnodes_running | Number of local Riak KV virtual nodes running +riak_kv_vnodeq_min | Minimum queue size of all local Riak KV virtual nodes in the last minute +riak_kv_vnodeq_median | Median queue size of all local Riak KV virtual nodes in the last minute +riak_kv_vnodeq_mean | Mean queue size of all local Riak KV virtual nodes in the last minute +riak_kv_vnodeq_max | Max queue size of all local Riak KV virtual nodes in the last minute +riak_kv_vnodeq_total | Total queue size of all local Riak KV virtual nodes in the last minute +riak_pipe_vnodes_running | Number of local Riak Pipe virtual nodes running +riak_pipe_vnodeq_min | Minimum queue size of local Riak Pipe virtual nodes in the last minute +riak_pipe_vnodeq_median | Median queue size of local Riak Pipe virtual nodes in the last minute +riak_pipe_vnodeq_mean | Mean queue size of local Riak Pipe virtual nodes in the last minute +riak_pipe_vnodeq_max | Max queue size of local Riak Pipe virtual nodes in the last minute +riak_pipe_vnodeq_total | Total queue size of all local Riak Pipe virtual nodes in the last minute diff --git a/content/riak/kv/2.2.6/developing/api/http/store-object.md b/content/riak/kv/2.2.6/developing/api/http/store-object.md new file mode 100644 index 0000000000..5941303c41 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/api/http/store-object.md @@ -0,0 +1,146 @@ +--- +title: "HTTP Store Object" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Store Object" + identifier: "http_store_object" + weight: 106 + parent: "apis_http" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/references/http/store-object + - /riak-docs/riak/kv/2.2.6/dev/references/http/store-object +--- + +Stores an object under the specified bucket / key. Storing an object comes in +two forms, depending on whether you want to use a key of your choosing, or let +Riak assign a key to a new object. + +## Request + +```bash +POST /types/type/buckets/bucket/keys # Riak-defined key +PUT /types/type/buckets/bucket/keys/key # User-defined key +POST /buckets/bucket/keys # Riak-defined key +PUT /buckets/bucket/keys/key # User-defined key +``` + +For the sake of compatibility with older clients, `POST` is also acceptable in +the form where the key is specified. + +Important headers: + +* `Content-Type` must be set for the stored object. Set what you expect to +receive back when next requesting it. +* `X-Riak-Vclock` if the object already exists, the vector clock attached to the +object when read. +* `X-Riak-Meta-*` - any additional metadata headers that should be stored with +the object. +* `X-Riak-Index-*` - index entries under which this object should be indexed. +[Read more about Secondary Indexing]({{}}riak/kv/2.2.6/developing/api/http/secondary-indexes) +* `Link` - user and system-defined links to other resources. [Read more about Links.]({{}}riak/kv/2.2.6/developing/api/http/link-walking) + +Optional headers (only valid on `PUT`): + +* `If-None-Match`, `If-Match`, `If-Modified-Since`, and `If-Unmodified-Since` +invoke conditional request semantics, matching on the `ETag` and `Last-Modified` +of the existing object. These can be used to prevent overwriting a modified +object. If the test fails, you will receive a `412 Precondition Failed` +response. This does not prevent concurrent writes; it is possible for the +condition to evaluate to true for multiple requests if the requests occur at the +same time. + +Optional query parameters: + +* `w` (write quorum) how many replicas to write to before returning a successful +response (default is defined by the bucket level) +* `dw` (durable write quorum) how many replicas to commit to durable storage +before returning a successful response (default is defined at the bucket level) +* `pw` how many primary replicas must be online to attempt a write (default is +defined at the bucket level) +* `returnbody=[true|false]` whether to return the contents of the stored object. + +*This request must include a body (entity).* + +## Response + +Normal status codes: + +* `201 Created` (when submitting without a key) +* `200 OK` +* `204 No Content` +* `300 Multiple Choices` + +Typical error codes: + +* `400 Bad Request` - e.g. when r, w, or dw parameters are invalid (> N) +* `412 Precondition Failed` if one of the conditional request headers failed to +match (see above) + +Important headers: + +* `Location` a relative URL to the newly-created object (when submitting without +a key) + +If `returnbody=true`, any of the response headers expected from [HTTP Fetch Object]({{}}riak/kv/2.2.6/developing/api/http/fetch-object) may be present. Like when fetching the object, `300 Multiple Choices` +may be returned if siblings existed or were created as part of the operation, +and the response can be dealt with similarly. + +## Example: Storing Without Key + +```curl +$ curl -v http://127.0.0.1:8098/buckets/test/keys \ + -H "Content-Type: text/plain" -d 'this is a test' +* About to connect() to 127.0.0.1 port 8098 (#0) +* Trying 127.0.0.1... connected +* Connected to 127.0.0.1 (127.0.0.1) port 8098 (#0) +> POST /buckets/test/keys HTTP/1.1 +> User-Agent: curl/7.19.4 (universal-apple-darwin10.0) libcurl/7.19.4 OpenSSL/0.9.8l zlib/1.2.3 +> Host: 127.0.0.1:8098 +> Accept: */* +> Content-Type: text/plain +> Content-Length: 14 +> +< HTTP/1.1 201 Created +< Vary: Accept-Encoding +< Server: MochiWeb/1.1 WebMachine/1.9.0 (participate in the frantic) +< Location: /buckets/test/keys/bzPygTesROPtGGVUKfyvp2RR49 +< Date: Fri, 30 Sep 2011 15:24:35 GMT +< Content-Type: application/json +< Content-Length: 0 +< +* Connection #0 to host 127.0.0.1 left intact +* Closing connection #0 +``` + +## Example: Storing With Key + +```curl +$ curl -v -XPUT -d '{"bar":"baz"}' -H "Content-Type: application/json" -H "X-Riak-Vclock: a85hYGBgzGDKBVIszMk55zKYEhnzWBlKIniO8mUBAA==" http://127.0.0.1:8098/buckets/test/keys/doc?returnbody=true +* About to connect() to 127.0.0.1 port 8098 (#0) +* Trying 127.0.0.1... connected +* Connected to 127.0.0.1 (127.0.0.1) port 8098 (#0) +> PUT /buckets/test/keys/doc?returnbody=true HTTP/1.1 +> User-Agent: curl/7.19.4 (universal-apple-darwin10.0) libcurl/7.19.4 OpenSSL/0.9.8l zlib/1.2.3 +> Host: 127.0.0.1:8098 +> Accept: */* +> Content-Type: application/json +> X-Riak-Vclock: a85hYGBgzGDKBVIszMk55zKYEhnzWBlKIniO8mUBAA== +> Content-Length: 13 +> +< HTTP/1.1 200 OK +< X-Riak-Vclock: a85hYGBgymDKBVIszMk55zKYEhnzWBlKIniO8kGF2TyvHYIKfwcJZwEA +< Vary: Accept-Encoding +< Server: MochiWeb/1.1 WebMachine/1.9.0 (participate in the frantic) +< Link: ; rel="up" +< Date: Fri, 30 Sep 2011 15:24:35 GMT +< Content-Type: application/json +< Content-Length: 13 +< +* Connection #0 to host 127.0.0.1 left intact +* Closing connection #0 +{"bar":"baz"} +``` diff --git a/content/riak/kv/2.2.6/developing/api/http/store-search-index.md b/content/riak/kv/2.2.6/developing/api/http/store-search-index.md new file mode 100644 index 0000000000..c9bcf7378a --- /dev/null +++ b/content/riak/kv/2.2.6/developing/api/http/store-search-index.md @@ -0,0 +1,52 @@ +--- +title: "HTTP Store Search Index" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Store Search Index" + identifier: "http_store_search_index" + weight: 115 + parent: "apis_http" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/references/http/store-search-index + - /riak-docs/riak/kv/2.2.6/dev/references/http/store-search-index +--- + +Creates a new Riak Search [index]({{}}riak/kv/2.2.6/developing/usage/search/#simple-setup). + +## Request + +``` +PUT /search/index/ +``` + +## Optional Request Body + +If you run a `PUT` request to this endpoint without a request body, Riak +will create a new Search index that uses the [default Search schema]({{}}riak/kv/2.2.6/developing/usage/search-schemas/#the-default-schema), i.e. `_yz_default`. + +To specify a different schema, however, you must pass Riak a JSON object +as the request body in which the `schema` field specifies the name of +the schema to use. If you've [stored a schema]({{}}riak/kv/2.2.6/developing/usage/search-schemas/#custom-schemas) called `my_custom_schema`, the following `PUT` +request would create an index called `my_index` that used that schema: + +```curl +curl -XPUT http://localhost:8098/search/index/my_index \ + -H "Content-Type: application/json" \ + -d '{"schema": "my_custom_schema"}' +``` + +More information can be found in [Using Search]({{}}riak/kv/2.2.6/developing/usage/search). + +## Normal Response Codes + +* `204 No Content` --- The index has been successfully created + +## Typical Error Codes + +* `409 Conflict` --- The index cannot be created because there is + already an index with that name +* `503 Service Unavailable` --- The request timed out internally diff --git a/content/riak/kv/2.2.6/developing/api/http/store-search-schema.md b/content/riak/kv/2.2.6/developing/api/http/store-search-schema.md new file mode 100644 index 0000000000..948a80e869 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/api/http/store-search-schema.md @@ -0,0 +1,50 @@ +--- +title: "HTTP Store Search Schema" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Store Search Schema" + identifier: "http_store_search_schema" + weight: 117 + parent: "apis_http" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/references/http/store-search-schema + - /riak-docs/riak/kv/2.2.6/dev/references/http/store-search-schema +--- + +Creates a new Riak [Search schema]({{}}riak/kv/2.2.6/developing/usage/search-schemas). + +## Request + +``` +PUT /search/schema/ +``` + +## Required Form Data + +In order to create a new Search schema, you must pass Riak a properly +formed XML schema. More information can be found in the [Search Schema]({{}}riak/kv/2.2.6/developing/usage/search-schemas) document. If you've created a schema and stored it in the filed +`my_schema.xml` and would like to create a new schema called +`my_custom_schema`, you would use the following HTTP request: + +```curl +curl -XPUT http://localhost:8098/search/schema/my_custom_schema \ + -H "Content-Type: application/xml" \ + --data-binary @my_schema.xml +``` + +## Normal Response + +* `204 No Content` --- The schema has been successfully created + +## Typical Error Codes + +* `400 Bad Request` --- The schema cannot be created because there is + something wrong with the schema itself, e.g. an XML formatting error + that makes Riak Search unable to parse the schema +* `409 Conflict` --- The schema cannot be created because there is + already a schema with that name +* `503 Service Unavailable` --- The request timed out internally diff --git a/content/riak/kv/2.2.6/developing/api/protocol-buffers.md b/content/riak/kv/2.2.6/developing/api/protocol-buffers.md new file mode 100644 index 0000000000..7b0a517c10 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/api/protocol-buffers.md @@ -0,0 +1,185 @@ +--- +title: "Protocol Buffers Client API" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Protocol Buffers API" + identifier: "apis_pbc" + weight: 103 + parent: "developing_apis" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/references/protocol-buffers + - /riak-docs/riak/kv/2.2.6/dev/references/protocol-buffers +--- + +This is an overview of the operations you can perform using the +[Protocol Buffers](https://code.google.com/p/protobuf/) Client (PBC) +interface to Riak, and can be used as a guide for developing a +PBC-compliant Riak client. + +## Protocol + +Riak listens on a TCP port (8087 by default) for incoming connections. +Once connected, the client can send a stream of requests on the same +connection. + +Each operation consists of a [request message](https://developers.google.com/protocol-buffers/docs/encoding) and one or more response messages. Messages are all encoded the same way, consisting of: + +* 32-bit length of message code + Protocol Buffers message in network + order +* 8-bit message code to identify the Protocol Buffers message +* N bytes of Protocol Buffers-encoded message + +### Example + +``` +00 00 00 07 09 0A 01 62 12 01 6B +|----Len---|MC|----Message-----| + +Len = 0x07 +Message Code (MC) = 0x09 = RpbGetReq +RpbGetReq Message = 0x0A 0x01 0x62 0x12 0x01 0x6B + +Decoded Message: +bucket: "b" +key: "k" +``` + +## Message Codes + +Code | Message | +:----|:--------| +0 | `RpbErrorResp` | +1 | `RpbPingReq` | +2 | `RpbPingResp` | +3 | `RpbGetClientIdReq` | +4 | `RpbGetClientIdResp` | +5 | `RpbSetClientIdReq` | +6 | `RpbSetClientIdResp` | +7 | `RpbGetServerInfoReq` | +8 | `RpbGetServerInfoResp` | +9 | `RpbGetReq` | +10 | `RpbGetResp` | +11 | `RpbPutReq` | +12 | `RpbPutResp` | +13 | `RpbDelReq` | +14 | `RpbDelResp` | +15 | `RpbListBucketsReq` | +16 | `RpbListBucketsResp` | +17 | `RpbListKeysReq` | +18 | `RpbListKeysResp` | +19 | `RpbGetBucketReq` | +20 | `RpbGetBucketResp` | +21 | `RpbSetBucketReq` | +22 | `RpbSetBucketResp` | +23 | `RpbMapRedReq` | +24 | `RpbMapRedResp` | +25 | `RpbIndexReq` | +26 | `RpbIndexResp` | +27 | `RpbSearchQueryReq` | +28 | `RbpSearchQueryResp` | +29 | `RpbResetBucketReq` | +30 | `RpbResetBucketResp` | +31 | `RpbGetBucketTypeReq` | +32 | `RpbSetBucketTypeResp` | +40 | `RpbCSBucketReq` | +41 | `RpbCSUpdateReq` | +50 | `RpbCounterUpdateReq` | +51 | `RpbCounterUpdateResp` | +52 | `RpbCounterGetReq` | +53 | `RpbCounterGetResp` | +54 | `RpbYokozunaIndexGetReq` | +55 | `RpbYokozunaIndexGetResp` | +56 | `RpbYokozunaIndexPutReq` | +57 | `RpbYokozunaIndexPutResp` | +58 | `RpbYokozunaSchemaGetReq` | +59 | `RpbYokozunaSchemaGetResp` | +60 | `RpbYokozunaSchemaPutReq` | +80 | `DtFetchReq` | +81 | `DtFetchResp` | +82 | `DtUpdateReq` | +83 | `DtUpdateResp` | +253 | `RpbAuthReq` | +254 | `RpbAuthResp` | +255 | `RpbStartTls` | + +{{% note title="Message Definitions" %}} +All Protocol Buffers messages are defined in the `riak.proto` and other +`.proto` files in the `/src` directory of the +RiakPB project. +{{% /note %}} + +### Error Response + +If the request does not result in an error, Riak will return one of a +variety of response messages, e.g. `RpbGetResp` or `RpbPutResp`, +depending on which request message is sent. + +If the server experiences an error processing a request, however, it +will return an `RpbErrorResp` message instead of the response expected +for the given request (e.g. `RbpGetResp` is the expected response to +`RbpGetReq`). Error messages contain an error string and an error code, +like this: + +```protobuf +message RpbErrorResp { + required bytes errmsg = 1; + required uint32 errcode = 2; +} +``` + +### Values + +* `errmsg` --- A string representation of what went wrong +* `errcode` --- A numeric code. Currently, only `RIAKC_ERR_GENERAL=1` + is defined. + +## Bucket Operations + +* [PBC List Buckets]({{}}riak/kv/2.2.6/developing/api/protocol-buffers/list-buckets) +* [PBC List Keys]({{}}riak/kv/2.2.6/developing/api/protocol-buffers/list-keys) +* [PBC Get Bucket Properties]({{}}riak/kv/2.2.6/developing/api/protocol-buffers/get-bucket-props) +* [PBC Set Bucket Properties]({{}}riak/kv/2.2.6/developing/api/protocol-buffers/set-bucket-props) +* [PBC Reset Bucket Properties]({{}}riak/kv/2.2.6/developing/api/protocol-buffers/reset-bucket-props) + +## Object/Key Operations + +* [PBC Fetch Object]({{}}riak/kv/2.2.6/developing/api/protocol-buffers/fetch-object) +* [PBC Store Object]({{}}riak/kv/2.2.6/developing/api/protocol-buffers/store-object) +* [PBC Delete Object]({{}}riak/kv/2.2.6/developing/api/protocol-buffers/delete-object) + +## Query Operations + +* [PBC MapReduce]({{}}riak/kv/2.2.6/developing/api/protocol-buffers/mapreduce) +* [PBC Secondary Indexes]({{}}riak/kv/2.2.6/developing/api/protocol-buffers/secondary-indexes) +* [PBC Search]({{}}riak/kv/2.2.6/developing/api/protocol-buffers/search) + +## Server Operations + +* [PBC Ping]({{}}riak/kv/2.2.6/developing/api/protocol-buffers/ping) +* [PBC Server Info]({{}}riak/kv/2.2.6/developing/api/protocol-buffers/server-info) + +## Bucket Type Operations + +* [PBC Get Bucket Type]({{}}riak/kv/2.2.6/developing/api/protocol-buffers/get-bucket-type) +* [PBC Set Bucket Type]({{}}riak/kv/2.2.6/developing/api/protocol-buffers/set-bucket-type) + +## Data Type Operations + +* [PBC Data Type Fetch]({{}}riak/kv/2.2.6/developing/api/protocol-buffers/dt-fetch) +* [PBC Data Type Union]({{}}riak/kv/2.2.6/developing/api/protocol-buffers/dt-union) +* [PBC Data Type Store]({{}}riak/kv/2.2.6/developing/api/protocol-buffers/dt-store) +* [PBC Data Type Counter Store]({{}}riak/kv/2.2.6/developing/api/protocol-buffers/dt-counter-store) +* [PBC Data Type Set Store]({{}}riak/kv/2.2.6/developing/api/protocol-buffers/dt-set-store) +* [PBC Data Type Map Store]({{}}riak/kv/2.2.6/developing/api/protocol-buffers/dt-map-store) + +## Yokozuna Operations + +* [PBC Yokozuna Index Get]({{}}riak/kv/2.2.6/developing/api/protocol-buffers/yz-index-get) +* [PBC Yokozuna Index Put]({{}}riak/kv/2.2.6/developing/api/protocol-buffers/yz-index-put) +* [PBC Yokozuna Index Delete]({{}}riak/kv/2.2.6/developing/api/protocol-buffers/yz-index-delete) +* [PBC Yokozuna Schema Get]({{}}riak/kv/2.2.6/developing/api/protocol-buffers/yz-schema-get) +* [PBC Yokozuna Schema Put]({{}}riak/kv/2.2.6/developing/api/protocol-buffers/yz-schema-put) diff --git a/content/riak/kv/2.2.6/developing/api/protocol-buffers/auth-req.md b/content/riak/kv/2.2.6/developing/api/protocol-buffers/auth-req.md new file mode 100644 index 0000000000..6fa010b164 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/api/protocol-buffers/auth-req.md @@ -0,0 +1,30 @@ +--- +title: "PBC Auth Request" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Auth Request" + identifier: "pbc_auth_request" + weight: 125 + parent: "apis_pbc" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/references/protocol-buffers/auth-req + - /riak-docs/riak/kv/2.2.6/dev/references/protocol-buffers/auth-req +--- + +Sends a username (`user`) and password (`password`) to Riak as part of +an authentication request. Both values are sent as binaries. + +## Request + +```protobuf +message RpbAuthReq { + required bytes user = 1; + required bytes password = 2; +} +``` + +For more on authentication, see our documentation on [Authentication and Authorization]({{}}riak/kv/2.2.6/using/security/basics). diff --git a/content/riak/kv/2.2.6/developing/api/protocol-buffers/delete-object.md b/content/riak/kv/2.2.6/developing/api/protocol-buffers/delete-object.md new file mode 100644 index 0000000000..89628f0cd6 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/api/protocol-buffers/delete-object.md @@ -0,0 +1,100 @@ +--- +title: "PBC Delete Object" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Delete Object" + identifier: "pbc_delete_object" + weight: 107 + parent: "apis_pbc" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/references/protocol-buffers/delete-object + - /riak-docs/riak/kv/2.2.6/dev/references/protocol-buffers/delete-object +--- + +Delete an object in the specified [bucket type]({{}}riak/kv/2.2.6/using/cluster-operations/bucket-types)/bucket/key location. + +## Request + +```protobuf +message RpbDelReq { + required bytes bucket = 1; + required bytes key = 2; + optional uint32 rw = 3; + optional bytes vclock = 4; + optional uint32 r = 5; + optional uint32 w = 6; + optional uint32 pr = 7; + optional uint32 pw = 8; + optional uint32 dw = 9; + optional uint32 timeout = 10; + optional bool sloppy_quorum = 11; + optional uint32 n_val = 12; + optional bytes type = 13; +} +``` + +#### Required Parameters + +Parameter | Description | +:---------|:------------| +`bucket` | The name of the bucket in which the object is stored +`key` | The key under which the object is stored + +#### Optional Parameters + +{{% note title="Note on defaults and special values" %}} +All of the optional parameters below have default values determined on a +per-bucket basis. Please refer to the documentation on [setting bucket properties](../set-bucket-props) for more information. + +Furthermore, you can assign an integer value to the `w`, `dw`, `pr`, and +`pw`, provided that that integer value is less than or equal to N, _or_ +a special value denoting `one` (`4294967295-1`), `quorum` +(`4294967295-2`), `all` (`4294967295-3`), or `default` (`4294967295-4`). +{{% /note %}} + +Parameter | Description | +:---------|:------------| +`rw` | How many replicas to delete before returning a successful response +`r` | Read quorum, i.e. how many replicas need to agree when retrieving the object +`w` | Write quorum, i.e. how many replicas to write to before returning a successful response +`pr` | Primary read quorum, i.e. how many primary replicas need to be available when retrieving the object +`pw` | Primary write quorum, i.e. how many primary nodes must be up when the write is attempted +`dw` | Durable write quorum, i.e. how many replicas to commit to durable storage before returning a successful response +`timeout` | The timeout duration, in milliseconds, after which Riak will return an error message +`vclock` | Opaque vector clock provided by an earlier `RpbGetResp` message Used to prevent deleting of objects that have been modified since the last GET request (sent as a byte array) +`sloppy_quorum` | If this parameter is set to `true`, the next available node in the ring will accept requests if any primary node is unavailable +`n_val` | The number of nodes to which the delete request will be sent +`type` | The bucket types associated with the object. If the bucket type is not specified, the `default` bucket type will be used, as is the case for all messages sent to Riak that have the bucket type as an optional parameter. + +## Response + +Only the message code is returned. + +## Example + +#### Request + +``` +Hex 00 00 00 12 0D 0A 0A 6E 6F 74 61 62 75 63 6B 65 + 74 12 01 6B 18 01 +Erlang <<0,0,0,18,13,10,10,110,111,116,97,98,117,99,107,101,116,18,1,107,24,1>> + +RpbDelReq protoc decode: +bucket: "notabucket" +key: "k" +rw: 1 + +``` + +#### Response + +``` +Hex 00 00 00 01 0E +Erlang <<0,0,0,1,14>> + +RpbDelResp - only message code defined +``` diff --git a/content/riak/kv/2.2.6/developing/api/protocol-buffers/dt-counter-store.md b/content/riak/kv/2.2.6/developing/api/protocol-buffers/dt-counter-store.md new file mode 100644 index 0000000000..b4a479e31b --- /dev/null +++ b/content/riak/kv/2.2.6/developing/api/protocol-buffers/dt-counter-store.md @@ -0,0 +1,31 @@ +--- +title: "PBC Data Type Counter Store" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Data Type Counter Store" + identifier: "pbc_dt_counter_store" + weight: 117 + parent: "apis_pbc" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/references/protocol-buffers/dt-counter-store + - /riak-docs/riak/kv/2.2.6/dev/references/protocol-buffers/dt-counter-store +--- + +An operation to update a [counter]({{}}riak/kv/2.2.6/developing/data-types). + +## Request + +```protobuf +message CounterOp { + optional sint64 increment = 1; +} +``` + +The `increment` value specifies how much the counter will be incremented +or decremented, depending on whether the `increment` value is positive +or negative. This operation can be used to update counters that are +stored on their own in a key or [within a map]({{}}riak/kv/2.2.6/developing/api/protocol-buffers/dt-map-store). diff --git a/content/riak/kv/2.2.6/developing/api/protocol-buffers/dt-fetch.md b/content/riak/kv/2.2.6/developing/api/protocol-buffers/dt-fetch.md new file mode 100644 index 0000000000..1ab7027041 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/api/protocol-buffers/dt-fetch.md @@ -0,0 +1,127 @@ +--- +title: "PBC Data Type Fetch" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Data Type Fetch" + identifier: "pbc_dt_fetch" + weight: 114 + parent: "apis_pbc" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/references/protocol-buffers/dt-fetch + - /riak-docs/riak/kv/2.2.6/dev/references/protocol-buffers/dt-fetch +--- + +The equivalent of [`RpbGetReq`]({{}}riak/kv/2.2.6/developing/api/protocol-buffers/fetch-object) for [Riak Data Types]({{}}riak/kv/2.2.6/developing/data-types). This request results in a `DtFetchResp` +message (explained in the **Response** section below). + +## Request + +```protobuf +message DtFetchReq { + required bytes bucket = 1; + required bytes key = 2; + required bytes type = 3; + optional uint32 r = 4; + optional uint32 pr = 5; + optional bool basic_quorum = 6; + optional bool notfound_ok = 7; + optional uint32 timeout = 8; + optional bool sloppy_quorum = 9; + optional uint32 n_val = 10; + optional bool include_context = 11 [default=true]; +} +``` + +#### Required Parameters + +Parameter | Description +:---------|:----------- +`bucket` | The name of the bucket in which the Data Type is stored +`key` | The key where the Data Type is stored +`type` | The [Using Bucket Types]({{}}riak/kv/2.2.6/using/cluster-operations/bucket-types) of the bucket in which the Data Type is stored, _not_ the type of Data Type (i.e. counter, set, or map) + +#### Optional Parameters + +> **Note on defaults and special values** +> +> All of the optional parameters below have default values determined on a +per-bucket basis. Please refer to the documentation on [setting bucket properties]({{}}riak/kv/2.2.6/developing/api/protocol-buffers/set-bucket-props) for more information. + +Furthermore, you can assign an integer value to the `r` and +`pr`, provided that that integer value is less than or equal +to N, _or_ a special value denoting `one` +(`4294967295-1`), `quorum` +(`4294967295-2`), `all` +(`4294967295-3`), or `default` +(`4294967295-4`). + +Parameter | Description +:---------|:----------- +`r` | Read quorum, i.e. how many replicas need to agree when retrieving the object +`pr` | Primary read quorum, i.e. how many primary replicas need to be available when retrieving the object +`basic_quorum` | Whether to return early in some failure cases, e.g. when `r=1` and you get 2 errors and a success basic_quorum=true would return an error +`notfound_ok` | Whether to treat `not found` responses as successful reads for the purposes of R +`timeout` | The timeout duration, in milliseconds, after which Riak will return an error message +`sloppy_quorum` | If this parameter is set to `true`, the next available node in the ring will accept requests if any primary node is unavailable +`n_val` | The number of nodes to which the delete request will be sent +`include_context` | If this parameter is set to `true`, the Data Type's opaque "context" will be returned to the client + +## Response + +The response to a fetch request ([`DtFetchReq`]({{}}riak/kv/2.2.6/developing/api/protocol-buffers/dt-fetch)) is a `DtFetchResp` message. + +```protobuf +message DtFetchResp { + enum DataType { + COUNTER = 1; + SET = 2; + MAP = 3; + } + + optional bytes context = 1; + required DataType type = 2; + optional DtValue value = 3; +} +``` + +If the `include_context` option is specified, an opaque "context" value +will be returned along with the user-readable data. When sending an +update request, the client should send this context as well, just as one +would send a [vclock]({{}}riak/kv/2.2.6/learn/glossary/#vector-clock) for standard KV updates. + +The type of the Data Type is specified in the `type` field, and must be +one of the three possible values of the `DataType` enum (`COUNTER`, +`SET`, or `MAP`). + +The current value of the Data Type is contained in the `value` field, +which itself contains a `DtValue` message. This message will have the +following structure: + +```protobuf +message DtValue { + optional sint64 counter_value = 1; + repeated bytes set_value = 2; + repeated MapEntry map_value = 3; +} +``` + +If the Data Type queried is a counter, it will return an integer value +for the counter; it a set, it will return the sets current value, in +bytes, if a map it will return a `MapEntry` message. `MapEntry` messages +are structured as follows: + +```protobuf +message MapEntry { + required MapField field = 1; + optional sint64 counter_value = 2; + repeated bytes set_value = 3; + optional bytes register_value = 4; + optional bool flag_value = 5; + repeated MapEntry map_value = 6; +} +``` + diff --git a/content/riak/kv/2.2.6/developing/api/protocol-buffers/dt-map-store.md b/content/riak/kv/2.2.6/developing/api/protocol-buffers/dt-map-store.md new file mode 100644 index 0000000000..364b731514 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/api/protocol-buffers/dt-map-store.md @@ -0,0 +1,73 @@ +--- +title: "PBC Data Type Map Store" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Data Type Map Store" + identifier: "pbc_dt_map_store" + weight: 119 + parent: "apis_pbc" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/references/protocol-buffers/dt-map-store + - /riak-docs/riak/kv/2.2.6/dev/references/protocol-buffers/dt-map-store +--- + +An operation to be applied to a value stored in a map (the contents of an update operation). The operation field that is present depends on the type of the field to which it is applied. All operations apply to individual fields nested in the map, i.e. counter-specific operations apply to specified counters in the map, set-specific operations to sets, etc. + +## Request + +Operations on maps are requested using a `MapOp` message, which has the following structure: + +```protobuf +message MapOp { + repeated MapField adds = 1; + repeated MapField removes = 2; + repeated MapUpdate updates = 3; +} +``` + +In a `MapOp` message, you can either add or remove fields (sets, counters, or maps) to or from the map or update a field or multiple fields. You can include as many field additions or removals and/or field updates as you wish. + +Adding or removing a field involves including a `MapField` message in your `MapOp` operation: + +```protobuf +message MapField { + enum MapFieldType { + COUNTER = 1; + SET = 2; + REGISTER = 3; + FLAG = 4; + MAP = 5; + } + required bytes name = 1; + required MapFieldType type = 2; +} +``` + +The `MapFieldType` specifies which type of field is being updated, and must be one of the possible values of the `MapFieldType` enum (either `COUNTER`, `SET`, `REGISTER`, `FLAG`, or `MAP`). The `name` parameter specifies the name of the field that will be updated. + +If you wish to update a map field, you can do so using a `MapUpdate` message, which has the following structure: + +```protobuf +message MapUpdate { + enum FlagOp { + ENABLE = 1; + DISABLE = 2; + } + required MapField field = 1; + optional CounterOp counter_op = 2; + optional SetOp set_op = 3; + optional bytes register_op = 4; + optional FlagOp flag_op = 5; + optional MapOp map_op = 6; +} +``` + +The `MapField` parameter is explained above. The operations used to update fields depend on the Data Type in that field, i.e. `CounterOp` messages to update counters, `SetOp` messages to update sets, etc. Updating counters is covered in [PBC Data Type Counter Store]({{}}riak/kv/2.2.6/developing/api/protocol-buffers/dt-counter-store) while updating sets is covered in [PBC Data Type Set Store]({{}}riak/kv/2.2.6/developing/api/protocol-buffers/dt-set-store). + +If you are updating a flag, you do so by including a `FlagOp` message. As shown in the `MapUpdate` message above, this operation takes one of two values: `ENABLE` and `DISABLE` (`1` and `2`, respectively). + +Updating a register does not involve sending a special message type. Instead, you must set the register to a desired value by specifying a binary for the `register_op` parameter. diff --git a/content/riak/kv/2.2.6/developing/api/protocol-buffers/dt-set-store.md b/content/riak/kv/2.2.6/developing/api/protocol-buffers/dt-set-store.md new file mode 100644 index 0000000000..633b724110 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/api/protocol-buffers/dt-set-store.md @@ -0,0 +1,32 @@ +--- +title: "PBC Data Type Set Store" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Data Type Set Store" + identifier: "pbc_dt_set_store" + weight: 118 + parent: "apis_pbc" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/references/protocol-buffers/dt-set-store + - /riak-docs/riak/kv/2.2.6/dev/references/protocol-buffers/dt-set-store +--- + +An operation to update a set, either on its own (at the bucket/key +level) or [inside of a map]({{}}riak/kv/2.2.6/developing/api/protocol-buffers/dt-map-store). + +## Request + +```protobuf +message SetOp { + repeated bytes adds = 1; + repeated bytes removes = 2; +} +``` + +Set members are binary values that can only be added (`adds`) or removed +(`removes`) from a set. You can add and/or remove as many members of a +set in a single message as you would like. diff --git a/content/riak/kv/2.2.6/developing/api/protocol-buffers/dt-store.md b/content/riak/kv/2.2.6/developing/api/protocol-buffers/dt-store.md new file mode 100644 index 0000000000..1b62e83c31 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/api/protocol-buffers/dt-store.md @@ -0,0 +1,128 @@ +--- +title: "PBC Data Type Store" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Data Type Store" + identifier: "pbc_dt_store" + weight: 116 + parent: "apis_pbc" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/references/protocol-buffers/dt-store + - /riak-docs/riak/kv/2.2.6/dev/references/protocol-buffers/dt-store +--- + +A request to update the value of a [Riak Data Type]({{}}riak/kv/2.2.6/developing/data-types). + +## Request + +A `DtUpdateReq` message requires that you specify the location of the +Data Type in Riak, which operations are to be performed, and whether the +Data Type's opaque context should be returned in the resulting +`DtUpdateResp`. + +The `DtOp` value specifies which Data Type-specific operation is being +performed. More on that in the [PBC Data Type Union]({{}}riak/kv/2.2.6/developing/api/protocol-buffers/dt-union) document. + +```protobuf +message DtUpdateReq { + required bytes bucket = 1; + optional bytes key = 2; + required bytes type = 3; + optional bytes context = 4; + required DtOp op = 5; + optional uint32 w = 6; + optional uint32 dw = 7; + optional uint32 pw = 8; + optional bool return_body = 9 [default=false]; + optional uint32 timeout = 10; + optional bool sloppy_quorum = 11; + optional uint32 n_val = 12; + optional bool include_context = 13 [default=true]; +} +``` + +#### Required Parameters + +Parameter | Description +:---------|:----------- +`bucket` | The name of the bucket in which the Data Type is stored +`type` | The bucket type of the bucket in which the Data Type is stored, _not_ the type of Data Type (i.e. counter, set, or map). Learn more about [using bucket types]({{}}riak/kv/2.2.6/using/cluster-operations/bucket-types). + +Also required is a `DtOp` message that specifies which operation is to +be performed, depending on whether the Data Type being updated is a +[counter]({{}}riak/kv/2.2.6/developing/api/protocol-buffers/dt-counter-store), [set]({{}}riak/kv/2.2.6/developing/api/protocol-buffers/dt-set-store), or [map]({{}}riak/kv/2.2.6/developing/api/protocol-buffers/dt-map-store). + +```protobuf +message DtOp { + optional CounterOp counter_op = 1; + optional SetOp set_op = 2; + optional MapOp map_op = 3; +} +``` + +#### Optional Parameters + +{{% note title="Note on defaults and special values" %}} +All of the optional parameters below have default values determined on a +per-bucket basis. Please refer to the documentation on [setting bucket properties](../set-bucket-props) for more information. + +Furthermore, you can assign an integer value to the `w`, `dw`, `pr`, and +`pw`, provided that that integer value is less than or equal to N, _or_ +a special value denoting `one` (`4294967295-1`), `quorum` +(`4294967295-2`), `all` (`4294967295-3`), or `default` (`4294967295-4`). +{{% /note %}} + +Parameter | Description +:---------|:----------- +`key` | The key where the Data Type is stored. If not specified, Riak will assign a random key and return that key to the client if `return_body` is set to `true`. +`context` | The opaque binary "context" that informs Riak which version of a data type the client has seen, analogous to [vector clocks]({{}}riak/kv/2.2.6/learn/glossary/#vector-clock) +`w` | Write quorum, i.e. how many replicas to write to before returning a successful response +`dw` | Durable write quorum, i.e. how many replicas to commit to durable storage before returning a successful response +`pw` | Primary write quorum, i.e. how many primary nodes must be up when the write is attempted +`return_body` | Whether to return the contents of the stored object. Defaults to `false`. +`timeout` | The timeout duration, in milliseconds, after which Riak will return an error message +`sloppy_quorum` | If this parameter is set to `true`, the next available node in the ring will accept requests if any primary node is unavailable +`n_val` | The number of nodes on which the value is to be stored +`include_context` | If `return_body` is set to `true`, the Data Type's opaque "context" will be returned to the client when the `DtUpdateResp` is sent to the client. + +## Response + +The response to a Data Type update request is analogous to +[`RpbPutResp`]({{}}riak/kv/2.2.6/developing/api/protocol-buffers/store-object) for KV operations. If the +`return_body` is set in the update request message (as explained above), +the message will include the opaque context of the Data Type (`context`) +and the new value of the Data Type _after_ the update has completed +(depending on whether the Data Type is a counter, set, or map). If no +key was specified in the update request, it will include the +Riak-assigned key (`key`). + +```protobuf +message DtUpdateResp { + optional bytes key = 1; + optional bytes context = 2; + optional sint64 counter_value = 3; + repeated bytes set_value = 4; + repeated MapEntry map_value = 5; +} +``` + +Assuming `return_body` is set to `true`: if a counter is updated, the +response will include an integer as the `counter_value`; if a set is +updated, a list of binaries will be return as the `set_value`; and if a +map is updated, the returned `map_value` will be a `MapEntry` message. +That message takes the following form: + +```protobuf +message MapEntry { + required MapField field = 1; + optional sint64 counter_value = 2; + repeated bytes set_value = 3; + optional bytes register_value = 4; + optional bool flag_value = 5; + repeated MapEntry map_value = 6; +} +``` diff --git a/content/riak/kv/2.2.6/developing/api/protocol-buffers/dt-union.md b/content/riak/kv/2.2.6/developing/api/protocol-buffers/dt-union.md new file mode 100644 index 0000000000..c0553d9785 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/api/protocol-buffers/dt-union.md @@ -0,0 +1,31 @@ +--- +title: "PBC Data Type Union" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Data Type Union" + identifier: "pbc_dt_union" + weight: 115 + parent: "apis_pbc" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/references/protocol-buffers/dt-union + - /riak-docs/riak/kv/2.2.6/dev/references/protocol-buffers/dt-union +--- + +A "union" type for update operations. + +## Request + +```protobuf +message DtOp { + optional CounterOp counter_op = 1; + optional SetOp set_op = 2; + optional MapOp map_op = 3; +} +``` + +The included operation depends on the Data Type that is being updated. +`DtOp` messages are sent only as part of a [`DtUpdateReq`]({{}}riak/kv/2.2.6/developing/api/protocol-buffers/dt-store) message. diff --git a/content/riak/kv/2.2.6/developing/api/protocol-buffers/fetch-object.md b/content/riak/kv/2.2.6/developing/api/protocol-buffers/fetch-object.md new file mode 100644 index 0000000000..55e34886f0 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/api/protocol-buffers/fetch-object.md @@ -0,0 +1,181 @@ +--- +title: "PBC Fetch Object" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Fetch Object" + identifier: "pbc_fetch_object" + weight: 105 + parent: "apis_pbc" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/references/protocol-buffers/fetch-object + - /riak-docs/riak/kv/2.2.6/dev/references/protocol-buffers/fetch-object +--- + +Fetch an object from the specified bucket type/bucket/key location +(specified by `bucket`, `type`, and `key`, respectively). If the bucket +type is not specified, the `default` bucket type will be used, as is the +case for all messages sent to Riak that have the bucket type as an +optional parameter. + +## Request + +```protobuf +message RpbGetReq { + required bytes bucket = 1; + required bytes key = 2; + optional uint32 r = 3; + optional uint32 pr = 4; + optional bool basic_quorum = 5; + optional bool notfound_ok = 6; + optional bytes if_modified = 7; + optional bool head = 8; + optional bool deletedvclock = 9; + optional uint32 timeout = 10; + optional bool sloppy_quorum = 11; + optional uint32 n_val = 12; + optional bytes type = 13; +} +``` + + +## Optional Parameters + +> **Note on defaults and special values** +> +> All of the optional parameters below have default values determined on a +per-bucket basis. Please refer to the documentation on [setting bucket properties]({{}}riak/kv/2.2.6/developing/api/protocol-buffers/set-bucket-props) for more information. +> +> Furthermore, you can assign an integer value to the `r` and +`pr` parameters, provided that that integer value is less than or +equal to N, or a special value denoting `one` +(`4294967295-1`), `quorum` (`4294967295-2`), +`all` (`4294967295-3`), or `default` +(`4294967295-4`). + +Parameter | Description | +:---------|:------------| +`basic_quorum` | Whether to return early in some failure cases, e.g. when `r=1` and you get 2 errors and a success basic_quorum=true would return an error +`notfound_ok` | Whether to treat `not found` responses as successful reads for the purposes of R +`if_modified` | When a vclock is supplied as this option, the response will only return the object if the vclocks don't match +`head` | If set to `true`, Riak will return the object with the value(s) set as empty, which allows you to get the metadata without a potentially large value accompanying it +`deletedvclock` | If set to `true`, Riak will return the tombstone's vclock, if applicable +`timeout` | The timeout duration, in milliseconds, after which Riak will return an error message +`sloppy_quorum` | If this parameter is set to `true`, the next available node in the ring will accept requests if any primary node is unavailable + +## Response + +```protobuf +message RpbGetResp { + repeated RpbContent content = 1; + optional bytes vclock = 2; + optional bool unchanged = 3; +} +``` + +#### Values + +Value | Description +:-----|:----------- +`content` | The value plus metadata entries for the object. If there are siblings, there will be more than one entry. If the key is not found, the content will be empty. +`vclock` | The opaque vector clock that must be included in the `RpbPutReq` to resolve the siblings +`unchanged` | If `if_modified` was specified in the GET request but the object has not been modified, this will be set to `true` + +The content entries hold the object value and any metadata. +Below is the structure of a RpbContent message, which is +included in GET/PUT responses (`RpbGetResp` (above) and +[`RpbPutResp`]({{}}riak/kv/2.2.6/developing/api/protocol-buffers/store-object), respectively): + +```protobuf +message RpbContent { + required bytes value = 1; + optional bytes content_type = 2; + optional bytes charset = 3; + optional bytes content_encoding = 4; + optional bytes vtag = 5; + repeated RpbLink links = 6; + optional uint32 last_mod = 7; + optional uint32 last_mod_usecs = 8; + repeated RpbPair usermeta = 9; + repeated RpbPair indexes = 10; + optional bool deleted = 11; +} +``` + +From the above, we can see that an `RpbContent` message will always +contain the binary `value` of the object. But it could also contain any +of the following optional parameters: + +* `content_type` --- The content type of the object, e.g. `text/plain` + or `application/json` +* `charset` --- The character encoding of the object, e.g. `utf-8` +* `content_encoding` --- The content encoding of the object, e.g. + `video/mp4` +* `vtag` --- The object's [vtag]({{}}riak/kv/2.2.6/learn/glossary/#vector-clock) +* `links` --- This parameter is associated with the now-deprecated link + walking feature and should not be used by Riak clients +* `last_mod` --- A timestamp for when the object was last modified, in + [ISO 8601 time](http://en.wikipedia.org/wiki/ISO_8601) +* `last_mod_usecs` --- A timestamp for when the object was last modified, + in [Unix time](http://en.wikipedia.org/wiki/Unix_time) +* `usermeta` --- This field stores user-specified key/value metadata + pairs to be associated with the object. `RpbPair` messages used to + send metadata of this sort are structured like this: + + ```protobuf + message RpbPair { + required bytes key = 1; + optional bytes value = 2; + } + ``` + Notice that both a key and value can be stored or just a key. + `RpbPair` messages are also used to attach [secondary indexes]({{}}riak/kv/2.2.6/developing/usage/secondary-indexes) to objects (in the optional + `indexes` field). +* `deleted` --- Whether the object has been deleted (i.e. whether a + tombstone for the object has been found under the specified key) + +{{% note title="Note on missing keys" %}} +Remember: if a key is not stored in Riak, an `RpbGetResp` response without the +`content` and `vclock` fields will be returned. This should be mapped to +whatever convention the client language uses to return not found. The Erlang +client, for example, returns the atom `{error, notfound}`. +{{% /note %}} + +## Example + +#### Request + +``` +Hex 00 00 00 07 09 0A 01 62 12 01 6B +Erlang <<0,0,0,7,9,10,1,98,18,1,107>> + +RpbGetReq protoc decode: +bucket: "b" +key: "k" +``` + +#### Response + +``` +Hex 00 00 00 4A 0A 0A 26 0A 02 76 32 2A 16 33 53 44 + 6C 66 34 49 4E 4B 7A 38 68 4E 64 68 79 49 6D 4B + 49 72 75 38 BB D7 A2 DE 04 40 E0 B9 06 12 1F 6B + CE 61 60 60 60 CC 60 CA 05 52 2C AC C2 5B 3F 65 + 30 25 32 E5 B1 32 EC 56 B7 3D CA 97 05 00 +Erlang <<0,0,0,74,10,10,38,10,2,118,50,42,22,51,83,68,108,102,52,73,78,75,122, + 56,104,78,100,104,121,73,109,75,73,114,117,56,187,215,162,222,4,64, + 224,185,6,18,31,107,206,97,96,96,96,204,96,2.2.6,82,44,172,194,91,63, + 101,48,37,50,229,177,50,236,86,183,61,202,151,5,0>> + +RpbGetResp protoc decode: +content { + value: "v2" + vtag: "3SDlf4INKz8hNdhyImKIru" + last_mod: 1271442363 + last_mod_usecs: 105696 +} +vclock: "k316a```314`312005R,254302[?e0%23452612354V267=312227005000" +``` diff --git a/content/riak/kv/2.2.6/developing/api/protocol-buffers/get-bucket-props.md b/content/riak/kv/2.2.6/developing/api/protocol-buffers/get-bucket-props.md new file mode 100644 index 0000000000..d064289c74 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/api/protocol-buffers/get-bucket-props.md @@ -0,0 +1,110 @@ +--- +title: "PBC Get Bucket Properties" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Get Bucket Properties" + identifier: "pbc_get_bucket_props" + weight: 102 + parent: "apis_pbc" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/references/protocol-buffers/get-bucket-props + - /riak-docs/riak/kv/2.2.6/dev/references/protocol-buffers/get-bucket-props +--- + +Fetch a bucket's properties. + +## Request + +```protobuf +message RpbGetBucketReq { + required bytes bucket = 1; + optional bytes type = 2; +} +``` + +The bucket's name (`bucket`) must be specified. The [bucket type]({{}}riak/kv/2.2.6/using/cluster-operations/bucket-types) parameter (`type`) is optional. If it is not specified, +the `default` bucket type will be used. + +## Response + +When an `RpbGetBucketReq` message is sent to Riak, it will respond with +an `RpbGetBucketResp` message, which returns the bucket's properties: + +```protobuf +message RpbGetBucketResp { + required RpbBucketProps props = 1; +} +``` + +The `RpbBucketProps` value itself is structured as follows: + +```protobuf +message RpbBucketProps { + optional uint32 n_val = 1; + optional bool allow_mult = 2; + optional bool last_write_wins = 3; + repeated RpbCommitHook precommit = 4; + optional bool has_precommit = 5 [default = false]; + repeated RpbCommitHook postcommit = 6; + optional bool has_postcommit = 7 [default = false]; + optional RpbModFun chash_keyfun = 8; + optional RpbModFun linkfun = 9; + optional uint32 old_vclock = 10; + optional uint32 young_vclock = 11; + optional uint32 big_vclock = 12; + optional uint32 small_vclock = 13; + optional uint32 pr = 14; + optional uint32 r = 15; + optional uint32 w = 16; + optional uint32 pw = 17; + optional uint32 dw = 18; + optional uint32 rw = 19; + optional bool basic_quorum = 20; + optional bool notfound_ok = 21; + optional bytes backend = 22; + optional bool search = 23; + enum RpbReplMode { + FALSE = 0; + REALTIME = 1; + FULLSYNC = 2; + TRUE = 3; + } + optional RpbReplMode repl = 24; + optional bytes search_index = 25; + optional bytes datatype = 26; + optional bool consistent = 27; +} +``` + +#### Optional Response Values + +Each `RpbBucketProps` message returns all of the properties associated +with a particular bucket. Default values for bucket properties, as well +as descriptions of all of the above properties, can be found in the +[configuration file]({{}}riak/kv/2.2.6/configuring/reference/#default-bucket-properties) documentation. + +It should be noted that the value of an `RpbBucketProps` message may +include other message types, such as `RpbModFun` (specifying +module-function pairs for bucket properties that require them) and +`RpbCommitHook` (specifying the module-function pair and name of a +commit hook). Those message types are structured like this: + +```protobuf +message RpbModFun { + required bytes module = 1; + required bytes function = 2; +} + +message RpbCommitHook { + optional RpbModFun modfun = 1; + optional bytes name = 2; +} +``` + +{{% note title="Note on `RpbReplMode`" %}} +The `RpbReplMode` is of use only to users of Riak CS's [Multi-Datacenter Replication capabilities](http://docs.basho.com/riak/cs/2.1.1/cookbooks/multi-datacenter-overview/) +{{% /note %}} diff --git a/content/riak/kv/2.2.6/developing/api/protocol-buffers/get-bucket-type.md b/content/riak/kv/2.2.6/developing/api/protocol-buffers/get-bucket-type.md new file mode 100644 index 0000000000..78c7736c99 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/api/protocol-buffers/get-bucket-type.md @@ -0,0 +1,33 @@ +--- +title: "PBC Get Bucket Type" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Get Bucket Type" + identifier: "pbc_get_bucket_type" + weight: 112 + parent: "apis_pbc" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/references/protocol-buffers/get-bucket-type + - /riak-docs/riak/kv/2.2.6/dev/references/protocol-buffers/get-bucket-type +--- + +Gets the bucket properties associated with a [bucket type]({{}}riak/kv/2.2.6/using/cluster-operations/bucket-types). + +## Request + +```protobuf +message RpbGetBucketTypeReq { + required bytes type = 1; +} +``` + +Only the name of the bucket type needs to be specified (under `name`). + +## Response + +A bucket type's properties will be sent to the client as part of an +[`RpbBucketProps`]({{}}riak/kv/2.2.6/developing/api/protocol-buffers/get-bucket-props) message. diff --git a/content/riak/kv/2.2.6/developing/api/protocol-buffers/get-client-id.md b/content/riak/kv/2.2.6/developing/api/protocol-buffers/get-client-id.md new file mode 100644 index 0000000000..ee397cdf32 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/api/protocol-buffers/get-client-id.md @@ -0,0 +1,61 @@ +--- +title: "PBC Get Client ID" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Get Client ID" + identifier: "pbc_get_client_id" + weight: 127 + parent: "apis_pbc" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/references/protocol-buffers/get-client-id + - /riak-docs/riak/kv/2.2.6/dev/references/protocol-buffers/get-client-id +--- + +{{% note title="Deprecation notice" %}} +The use of client IDs in conflict resolution is now deprecated in Riak. If you +are building or maintaining a Riak client that is intended to be compatible +with Riak 1.4 or later, you can safely ignore client IDs. +{{% /note %}} + +Get the client id used for this connection. Client ids are used for +conflict resolution and each unique actor in the system should be +assigned one. A client id is assigned randomly when the socket is +connected and can be changed using [Set Client ID]({{}}riak/kv/2.2.6/developing/api/protocol-buffers/set-client-id). + +## Request + +Just the `RpbGetClientIdReq` message code. No request message defined. + +## Response + +```protobuf +// Get ClientId Request - no message defined, just send RpbGetClientIdReq +message code +message RpbGetClientIdResp { + required bytes client_id = 1; // Client id in use for this connection +} +``` + +## Example + +Request + +``` +Hex 00 00 00 01 03 +Erlang <<0,0,0,1,3>> +``` + + +Response + +``` +Hex 00 00 00 07 04 0A 04 01 65 01 B5 +Erlang <<0,0,0,7,4,10,4,1,101,1,181>> + +RpbGetClientIdResp protoc decode: +client_id: "001e001265" +``` diff --git a/content/riak/kv/2.2.6/developing/api/protocol-buffers/list-buckets.md b/content/riak/kv/2.2.6/developing/api/protocol-buffers/list-buckets.md new file mode 100644 index 0000000000..6267865674 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/api/protocol-buffers/list-buckets.md @@ -0,0 +1,76 @@ +--- +title: "PBC List Buckets" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "List Buckets" + identifier: "pbc_list_buckets" + weight: 100 + parent: "apis_pbc" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/references/protocol-buffers/list-buckets + - /riak-docs/riak/kv/2.2.6/dev/references/protocol-buffers/list-buckets +--- + +List all of the bucket names available. + +{{% note title="Caution" %}} +This call can be expensive for the server. Do not use in performance-sensitive +code. +{{% /note %}} + + +## Request + +Only the message code is required. + +## Response + + +```protobuf +message RpbListBucketsResp { + repeated bytes buckets = 1; +} +``` + + +Values + +* `buckets` --- Buckets on the server + +## Example + +#### Request + +```bash +Hex 00 00 00 01 0F +Erlang <<0,0,0,1,15>> + +RpbListBucketsReq - only message code defined +``` + + +#### Response + +```bash +Hex 00 00 00 2A 10 0A 02 62 31 0A 02 62 35 0A 02 62 + 34 0A 02 62 38 0A 02 62 33 0A 03 62 31 30 0A 02 + 62 39 0A 02 62 32 0A 02 62 36 0A 02 62 37 +Erlang <<0,0,0,42,16,10,2,98,49,10,2,98,53,10,2,98,52,10,2,98,56,10,2,98,51,10, + 3,98,49,48,10,2,98,57,10,2,98,50,10,2,98,54,10,2,98,55>> + +RpbListBucketsResp protoc decode: +buckets: "b1" +buckets: "b5" +buckets: "b4" +buckets: "b8" +buckets: "b3" +buckets: "b10" +buckets: "b9" +buckets: "b2" +buckets: "b6" +buckets: "b7" +``` diff --git a/content/riak/kv/2.2.6/developing/api/protocol-buffers/list-keys.md b/content/riak/kv/2.2.6/developing/api/protocol-buffers/list-keys.md new file mode 100644 index 0000000000..66c21c3599 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/api/protocol-buffers/list-keys.md @@ -0,0 +1,97 @@ +--- +title: "PBC List Keys" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "List Keys" + identifier: "pbc_list_keys" + weight: 101 + parent: "apis_pbc" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/references/protocol-buffers/list-keys + - /riak-docs/riak/kv/2.2.6/dev/references/protocol-buffers/list-keys +--- + +List all of the keys in a bucket. This is a streaming call, with +multiple response messages sent for each request. + +{{% note title="Not for production use" %}} +This operation requires traversing all keys stored in the cluster and should +not be used in production. +{{% /note %}} + +## Request + +```protobuf +message RpbListKeysReq { + required bytes bucket = 1; +} +``` + +Optional Parameters + +* `bucket` --- bucket to get keys from + +## Response + +```protobuf +message RpbListKeysResp { + repeated bytes keys = 1; + optional bool done = 2; +} +``` + +#### Values + +* **keys** - batch of keys in the bucket. +* **done** - set true on the last response packet + +## Example + +#### Request + +```bash +Hex 00 00 00 0B 11 0A 08 6C 69 73 74 6B 65 79 73 +Erlang <<0,0,0,11,17,10,8,108,105,115,116,107,101,121,115>> + +RpbListKeysReq protoc decode: +bucket: "listkeys" + +``` + +#### Response Packet 1 + +```bash +Hex 00 00 00 04 12 0A 01 34 +Erlang <<0,0,0,4,18,10,1,52>> + +RpbListKeysResp protoc decode: +keys: "4" + +``` + +#### Response Packet 2 + +```bash +Hex 00 00 00 08 12 0A 02 31 30 0A 01 33 +Erlang <<0,0,0,8,18,10,2,49,48,10,1,51>> + +RpbListKeysResp protoc decode: +keys: "10" +keys: "3" +``` + + +#### Response Packet 3 + +```bash +Hex 00 00 00 03 12 10 01 +Erlang <<0,0,0,3,18,16,1>> + +RpbListKeysResp protoc decode: +done: true + +``` diff --git a/content/riak/kv/2.2.6/developing/api/protocol-buffers/mapreduce.md b/content/riak/kv/2.2.6/developing/api/protocol-buffers/mapreduce.md new file mode 100644 index 0000000000..8160f7b8ea --- /dev/null +++ b/content/riak/kv/2.2.6/developing/api/protocol-buffers/mapreduce.md @@ -0,0 +1,149 @@ +--- +title: "PBC MapReduce" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "MapReduce" + identifier: "pbc_mapreduce" + weight: 107 + parent: "apis_pbc" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/references/protocol-buffers/mapreduce + - /riak-docs/riak/kv/2.2.6/dev/references/protocol-buffers/mapreduce +--- + +Execute a MapReduce job. + +## Request + + +```protobuf +message RpbMapRedReq { + required bytes request = 1; + required bytes content_type = 2; +} +``` + + +Required Parameters + +* `request` --- MapReduce job +* `content_type` - Encoding for MapReduce job + +Mapreduce jobs can be encoded in two different ways + +* `application/json` --- JSON-encoded MapReduce job +* `application/x-erlang-binary` --- Erlang external term format + +The JSON encoding is the same as [REST API]({{}}riak/kv/2.2.6/developing/usage/mapreduce/#rest) and +the external term format is the same as the [local Erlang API]({{}}riak/kv/2.2.6/developing/app-guide/advanced-mapreduce/#erlang) + +## Response + +The results of the MapReduce job is returned for each phase that +generates a result, encoded in the same format the job was submitted in. +Multiple response messages will be returned followed by a final message +at the end of the job. + +```protobuf +message RpbMapRedResp { + optional uint32 phase = 1; + optional bytes response = 2; + optional bool done = 3; +} +``` + + +Values + +* `phase` --- Phase number of the MapReduce job +* `response` - Response encoded with the content_type submitted +* `done` - Set `true` on the last response packet + +## Example + +Here is how submitting a JSON encoded job to sum up a bucket full of +JSON encoded values. + +``` +{"inputs": "bucket_501653", + "query": + [{"map": {"arg": null, + "name": "Riak.mapValuesJson", + "language": "javascript", + "keep": false}}, + {"reduce": {"arg": null, + "name": "Riak.reduceSum", + "language": "javascript", + "keep": true}}]}" +``` + +Request + +```bash +Hex 00 00 00 F8 17 0A E2 01 7B 22 69 6E 70 75 74 73 + 22 3A 20 22 62 75 63 6B 65 74 5F 35 30 31 36 35 + 33 22 2C 20 22 71 75 65 72 79 22 3A 20 5B 7B 22 + 6D 61 70 22 3A 20 7B 22 61 72 67 22 3A 20 6E 75 + 6C 6C 2C 20 22 6E 61 6D 65 22 3A 20 22 52 69 61 + 6B 2E 6D 61 70 56 61 6C 75 65 73 4A 73 6F 6E 22 + 2C 20 22 6C 61 6E 67 75 61 67 65 22 3A 20 22 6A + 61 76 61 73 63 72 69 70 74 22 2C 20 22 6B 65 65 + 70 22 3A 20 66 61 6C 73 65 7D 7D 2C 20 7B 22 72 + 65 64 75 63 65 22 3A 20 7B 22 61 72 67 22 3A 20 + 6E 75 6C 6C 2C 20 22 6E 61 6D 65 22 3A 20 22 52 + 69 61 6B 2E 72 65 64 75 63 65 53 75 6D 22 2C 20 + 22 6C 61 6E 67 75 61 67 65 22 3A 20 22 6A 61 76 + 61 73 63 72 69 70 74 22 2C 20 22 6B 65 65 70 22 + 3A 20 74 72 75 65 7D 7D 5D 7D 12 10 61 70 70 6C + 69 63 61 74 69 6F 6E 2F 6A 73 6F 6E +Erlang <<0,0,0,248,23,10,226,1,123,34,105,110,112,117,116,115,34,58,32,34,98, + 117,99,107,101,116,95,53,48,49,54,53,51,34,44,32,34,113,117,101,114, + 121,34,58,32,91,123,34,109,97,112,34,58,32,123,34,97,114,103,34,58,32, + 110,117,108,108,44,32,34,110,97,109,101,34,58,32,34,82,105,97,107,46, + 109,97,112,86,97,108,117,101,115,74,115,111,110,34,44,32,34,108,97, + 110,103,117,97,103,101,34,58,32,34,106,97,118,97,115,99,114,105,112, + 116,34,44,32,34,107,101,101,112,34,58,32,102,97,108,115,101,125,125, + 44,32,123,34,114,101,100,117,99,101,34,58,32,123,34,97,114,103,34,58, + 32,110,117,108,108,44,32,34,110,97,109,101,34,58,32,34,82,105,97,107, + 46,114,101,100,117,99,101,83,117,109,34,44,32,34,108,97,110,103,117, + 97,103,101,34,58,32,34,106,97,118,97,115,99,114,105,112,116,34,44,32, + 34,107,101,101,112,34,58,32,116,114,117,101,125,125,93,125,18,16,97, + 112,112,108,105,99,97,116,105,111,110,47,106,115,111,110>> + +RpbMapRedReq protoc decode: +request: "{"inputs": "bucket_501653", "query": [{"map": {"arg": null, +"name": "Riak.mapValuesJson", "language": "javascript", "keep": false}}, + {"reduce": {"arg": null, "name": "Riak.reduceSum", "language": +"javascript", "keep": true}}]}" +content_type: "application/json" + +``` + + +Response 1 - result from phase 1 + +```bash +Hex 00 00 00 08 18 08 01 12 03 5B 39 5D +Erlang <<0,0,0,8,24,8,1,18,3,91,57,93>> + +RpbMapRedResp protoc decode: +phase: 1 +response: "[[9]]" + +``` + + +Response 2 - end of MapReduce job + +```bash +Hex 00 00 00 03 18 18 01 +Erlang <<0,0,0,3,24,24,1>> + +RpbMapRedResp protoc decode: +done: true + +``` diff --git a/content/riak/kv/2.2.6/developing/api/protocol-buffers/ping.md b/content/riak/kv/2.2.6/developing/api/protocol-buffers/ping.md new file mode 100644 index 0000000000..53e3e92565 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/api/protocol-buffers/ping.md @@ -0,0 +1,42 @@ +--- +title: "PBC Ping" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Ping" + identifier: "pbc_ping" + weight: 110 + parent: "apis_pbc" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/references/protocol-buffers/ping + - /riak-docs/riak/kv/2.2.6/dev/references/protocol-buffers/ping +--- + +Check if the server is alive + +## Request + +Just the `RpbPingReq` message code. No request message defined. + +## Response + +Just the `RpbPingResp` message code. No response message defined. + +## Example + +Request + +```bash +Hex 00 00 00 01 01 +Erlang <<0,0,0,1,1>> +``` + +Response + +```bash +Hex 00 00 00 01 02 +Erlang <<0,0,0,1,2>> +``` diff --git a/content/riak/kv/2.2.6/developing/api/protocol-buffers/reset-bucket-props.md b/content/riak/kv/2.2.6/developing/api/protocol-buffers/reset-bucket-props.md new file mode 100644 index 0000000000..755b20e35a --- /dev/null +++ b/content/riak/kv/2.2.6/developing/api/protocol-buffers/reset-bucket-props.md @@ -0,0 +1,59 @@ +--- +title: "PBC Reset Bucket Properties" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Reset Bucket Properties" + identifier: "pbc_reset_bucket_props" + weight: 104 + parent: "apis_pbc" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/references/protocol-buffers/reset-bucket-props + - /riak-docs/riak/kv/2.2.6/dev/references/protocol-buffers/reset-bucket-props +--- + +Request to reset the properties of a given bucket or bucket type. + +## Request + +```protobuf +message RpbResetBucketReq { + required bytes bucket = 1; + optional bytes type = 2; +} +``` + +You must specify the name of the bucket (`bucket`) and optionally a +[bucket type]({{}}riak/kv/2.2.6/developing/usage/bucket-types) using the `type` value. If you do not +specify a bucket type, the `default` bucket type will be used by Riak. + +## Response + +Only the message code is returned. + +## Example + +Request to reset the properties for the bucket `friends`: + +#### Request + +```bash +Hex 00 00 00 0A 1D 0A 07 66 72 69 65 6E 64 73 +Erlang <<0,0,0,10,29,10,7,102,114,105,101,110,100,115>> + +RpbResetBucketReq protoc decode: +bucket: "friends" + +``` + +#### Response + +```bash +Hex 00 00 00 01 1E +Erlang <<0,0,0,1,30>> + +RpbResetBucketResp - only message code defined +``` diff --git a/content/riak/kv/2.2.6/developing/api/protocol-buffers/search.md b/content/riak/kv/2.2.6/developing/api/protocol-buffers/search.md new file mode 100644 index 0000000000..b169aa8757 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/api/protocol-buffers/search.md @@ -0,0 +1,148 @@ +--- +title: "PBC Search" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Search" + identifier: "pbc_search" + weight: 109 + parent: "apis_pbc" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/references/protocol-buffers/search + - /riak-docs/riak/kv/2.2.6/dev/references/protocol-buffers/search +--- + +Send a Search request to retrieve a list of documents, along with a few +stats. + +## Request + + +```protobuf +message RpbSearchQueryReq { + required bytes q = 1; + required bytes index = 2; + optional uint32 rows = 3; + optional uint32 start = 4; + optional bytes sort = 5; + optional bytes filter = 6; + optional bytes df = 7; + optional bytes op = 8; + repeated bytes fl = 9; + optional bytes presort = 10; +} +``` + +Required Parameters + +* `q` --- The contents of the query +* `index` --- The name of the index to search + +Optional Parameters + +* `rows` --- The maximum number of rows to return +* `start` --- A start offset, i.e. the number of keys to skip before + returning values +* `sort` --- How the search results are to be sorted +* `filter` --- Filters search with additional query scoped to inline + fields +* `df` --- Override the `default_field` setting in the schema file +* `op` --- `and` or `or`, to override the `default_op` operation setting + in the schema file +* `fl` --- Return the fields limit +* `presort` --- Presort. The options are `key` or `score` + + +## Response + +The results of a search query are returned as a repeating list of 0 or +more `RpbSearchDoc`s. `RpbSearchDoc`s themselves are composed of 0 or +more key/value pairs (`RpbPair`) that match the given request +parameters. It also returns the maximum search score and the number of +results. + + +```protobuf +// RbpPair is a generic key/value pair datatype used for +// other message types +message RpbPair { + required bytes key = 1; + optional bytes value = 2; +} + +message RpbSearchDoc { + repeated RpbPair fields = 1; +} + +message RpbSearchQueryResp { + repeated RpbSearchDoc docs = 1; + optional float max_score = 2; + optional uint32 num_found = 3; +} +``` + +Values + +* `docs` --- A list of docs that match the search request +* `max_score` --- The top score returned +* `num_found` --- Returns the total number of values matched by this + search + + +## Example + +Request + +Here we search for any animals that being with the string `pig`. We only +want the first 100, and sort the values by a `name` field. + +```bash +RpbSearchQueryReq protoc decode: +q: "pig*" +index: "animals" +rows: 100 +start: 0 +sort: "name" + +Hex 00 00 00 1A 1B 0A 04 70 69 67 2A 12 07 61 6E + 69 6D 61 6C 73 18 64 20 00 2A 04 6E 61 6D 65 +Erlang <<0,0,0,26,27,10,4,112,105,103,42,18,7,97,110, + 105,109,97,108,115,24,100,32,0,42,4,110,97, + 109,101>> +``` + +Response + +```bash +Hex 00 00 00 36 1B 0A 1D 0A 0D 0A 06 61 6E 69 6D + 61 6C 12 03 70 69 67 0A 0C 0A 04 6E 61 6D 65 + 12 04 66 72 65 64 0A 12 0A 10 0A 06 61 6E 69 + 6D 61 6C 12 06 70 69 67 65 6F 6E 18 02 +Erlang <<0,0,0,54,27,10,29,10,13,10,6,97,110,105,109, + 97,108,18,3,112,105,103,10,12,10,4,110,97, + 109,101,18,4,102,114,101,100,10,18,10,16,10, + 6,97,110,105,109,97,108,18,6,112,105,103, + 101,111,110,24,2>> + +RpbSearchQueryResp protoc decode: +docs { + fields { + key: "animal" + value: "pig" + } + fields { + key: "name" + value: "fred" + } +} +docs { + fields { + key: "animal" + value: "pigeon" + } +} +num_found: 2 +``` diff --git a/content/riak/kv/2.2.6/developing/api/protocol-buffers/secondary-indexes.md b/content/riak/kv/2.2.6/developing/api/protocol-buffers/secondary-indexes.md new file mode 100644 index 0000000000..65b9d845fe --- /dev/null +++ b/content/riak/kv/2.2.6/developing/api/protocol-buffers/secondary-indexes.md @@ -0,0 +1,121 @@ +--- +title: "PBC Secondary Indexes" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Secondary Indexes" + identifier: "pbc_secondary_indexes" + weight: 108 + parent: "apis_pbc" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/references/protocol-buffers/secondary-indexes + - /riak-docs/riak/kv/2.2.6/dev/references/protocol-buffers/secondary-indexes +--- + +Request a set of keys that match a secondary index query. + +## Request + +```protobuf +message RpbIndexReq { + enum IndexQueryType { + eq = 0; + range = 1; + } + required bytes bucket = 1; + required bytes index = 2; + required IndexQueryType qtype = 3; + optional bytes key = 4; + optional bytes range_min = 5; + optional bytes range_max = 6; + optional bool return_terms = 7; + optional bool stream = 8; + optional uint32 max_results = 9; + optional bytes continuation = 10; + optional uint32 timeout = 11; + optional bytes type = 12; + optional bytes term_regex = 13; + optional bool pagination_sort = 14; +} +``` + +#### Required Parameters + +Parameter | Description +:---------|:----------- +`bucket` | The name of the bucket in which the Data Type is stored +`index` | The name of the index to be queried +`qtype` | The type of index query to be performed. This can take either of the two possible values of the `IndexQueryType` enum: `eq` for an exact index match for the given `key` or `range` for a range query + +#### Optional Parameters + +Parameter | Description +:---------|:----------- +`key` | The name of the index to be queried if `qtype` is set to `eq` +`range_min` and `range_max` | The minimum and maximum values for a range query if `qtype` is set to `range` +`return_terms` | If set to `true`, the response will include matched indexed values (for range queries only) +`stream` | If set to `true`, keys matching the index query will be streamed to the client instead of waiting for `max_results` or the full result to be tabulated +`max_results` | If pagination is turned on, the number of results to be returned to the client +`continuation` | If set to `true`, values are returned in a paginated response +`timeout` | The timeout duration, in milliseconds, after which Riak will return an error message +`type` | The bucket type of the bucket that is being queried. If not set, the bucket type `default` will be used. Learn more about [using bucket types]({{}}riak/kv/2.2.6/developing/usage/bucket-types). +`term_regex` | If set to a regular expression (as a binary), a term filter will be applied to the index query +`pagination_sort` | If set to `true`, paginated results will be sorted, first by index value, then by key + +## Response + +The results of a Secondary Index query are returned as a repeating list +of 0 or more keys that match the given request parameters. + +```protobuf +message RpbIndexResp { + repeated bytes keys = 1; + repeated RpbPair results = 2; + optional bytes continuation = 3; + optional bool done = 4; +} +``` + +#### Values + +Parameter | Description +:---------|:----------- +`keys` | A list of keys that match the index request +`results` | If `return_terms` is specified with range queries, used to return matched index values as key/value pairs in `RpbPair` messages. More on `RpbPair` messages can be found in [PBC Fetch Object]({{}}riak/kv/2.2.6/developing/api/protocol-buffers/fetch-object). +`continuation` | Used for paginated responses +`done` | Used for streaming. The value will be `true` when the current stream is done (either `max_results` has been reached or there are no more results). + +## Example + +#### Request + +Here we look for any exact matches of `chicken` on an `animal_bin` index +for a bucket named `farm`. + +```bash +RpbIndexReq protoc decode: +bucket: "farm" +index: "animal_bin" +qtype: 0 +key: "chicken" + +Hex 00 00 00 1E 19 0A 04 66 61 72 6D 12 0A 61 6E 69 + 6D 61 6C 5F 62 69 6E 18 00 22 07 63 68 69 63 6B 65 6E +Erlang <<0,0,0,30,25,10,10,4,102,97,114,109,18,10,97,110,105, + 109,97,108,95,98,105,110,24,0,34,7,99,104,105,99,107, + 101,110>> +``` + +#### Response + +```bash +Hex 00 00 00 0F 1A 0A 03 68 65 6E 0A 07 72 6F 6F 73 74 65 72 +Erlang <<0,0,0,15,26,10,3,104,101,110,10,7,114,111,111,115,116,101,114>> + +RpbIndexResp protoc decode: +keys: "hen" +keys: "rooster" +``` diff --git a/content/riak/kv/2.2.6/developing/api/protocol-buffers/server-info.md b/content/riak/kv/2.2.6/developing/api/protocol-buffers/server-info.md new file mode 100644 index 0000000000..8e36044df6 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/api/protocol-buffers/server-info.md @@ -0,0 +1,58 @@ +--- +title: "PBC Server Info" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Server Info" + identifier: "pbc_server_info" + weight: 111 + parent: "apis_pbc" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/references/protocol-buffers/server-info + - /riak-docs/riak/kv/2.2.6/dev/references/protocol-buffers/server-info +--- + +A message from Riak that contains two pieces of information about the +server: the name of the node and the version of Riak in use on that +node. + +## Request + +A request consists only of the `RpbGetServerInfoReq` message code. No +request message is defined. + +## Response + +```protobuf +message RpbGetServerInfoResp { + optional bytes node = 1; + optional bytes server_version = 2; +} +``` + +## Example + +#### Request + +```bash +Hex 00 00 00 01 07 +Erlang <<0,0,0,1,7>> + +RpbGetServerInfoReq - only message code defined +``` + +#### Response + +```bash +Hex 00 00 00 17 08 0A 0E 72 69 61 6B 40 31 32 37 2E + 30 2E 30 2E 31 12 04 30 2E 31 30 +Erlang <<0,0,0,23,8,10,14,114,105,97,107,64,49,50,55,46,48,46,48,46,49,18,4,48, + 46,49,48>> + +RpbGetServerInfoResp protoc decode: +node: "riak@127.0.0.1" +server_version: "0.10" +``` diff --git a/content/riak/kv/2.2.6/developing/api/protocol-buffers/set-bucket-props.md b/content/riak/kv/2.2.6/developing/api/protocol-buffers/set-bucket-props.md new file mode 100644 index 0000000000..de7e06a405 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/api/protocol-buffers/set-bucket-props.md @@ -0,0 +1,68 @@ +--- +title: "PBC Set Bucket Properties" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Set Bucket Properties" + identifier: "pbc_set_bucket_props" + weight: 103 + parent: "apis_pbc" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/references/protocol-buffers/set-bucket-props + - /riak-docs/riak/kv/2.2.6/dev/references/protocol-buffers/set-bucket-props +--- + +Sets the properties for a bucket. + +## Request + +```protobuf +message RpbSetBucketReq { + required bytes bucket = 1; + required RpbBucketProps props = 2; + optional bytes type = 3; +} +``` + +You must specify the name of the bucket (`bucket`) and include an +`RpbBucketProps` message. More on that message type can be found in the +[PBC Get Bucket Properties]({{}}riak/kv/2.2.6/developing/api/protocol-buffers/get-bucket-props) documentation. + +You can also specify a [bucket type]({{}}riak/kv/2.2.6/developing/usage/bucket-types) using the +`type` value. If you do not specify a bucket type, the `default` bucket +type will be used by Riak. + +## Response + +Only the message code is returned. + +## Example + +Change `allow_mult` to true for the bucket `friends`: + +#### Request + +```bash +Hex 00 00 00 0E 15 0A 07 66 72 69 65 6E 64 73 12 02 + 10 01 +Erlang <<0,0,0,14,21,10,7,102,114,105,101,110,100,115,18,2,16,1>> + +RpbSetBucketReq protoc decode: +bucket: "friends" +props { + allow_mult: true +} + +``` + +#### Response + +```bash +Hex 00 00 00 01 16 +Erlang <<0,0,0,1,22>> + +RpbSetBucketResp - only message code defined +``` diff --git a/content/riak/kv/2.2.6/developing/api/protocol-buffers/set-bucket-type.md b/content/riak/kv/2.2.6/developing/api/protocol-buffers/set-bucket-type.md new file mode 100644 index 0000000000..ab3b28fbbb --- /dev/null +++ b/content/riak/kv/2.2.6/developing/api/protocol-buffers/set-bucket-type.md @@ -0,0 +1,31 @@ +--- +title: "PBC Set Bucket Type" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Set Bucket Type" + identifier: "pbc_set_bucket_type" + weight: 113 + parent: "apis_pbc" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/references/protocol-buffers/set-bucket-type + - /riak-docs/riak/kv/2.2.6/dev/references/protocol-buffers/set-bucket-type +--- + +Assigns a set of [bucket properties]({{}}riak/kv/2.2.6/developing/api/protocol-buffers/set-bucket-props) to a +[bucket type]({{}}riak/kv/2.2.6/developing/usage/bucket-types). + +## Request + +```protobuf +message RpbSetBucketTypeReq { + required bytes type = 1; + required RpbBucketProps props = 2; +} +``` + +The `type` field specifies the name of the bucket type as a binary. The +`props` field contains an [`RpbBucketProps`]({{}}riak/kv/2.2.6/developing/api/protocol-buffers/get-bucket-props). diff --git a/content/riak/kv/2.2.6/developing/api/protocol-buffers/set-client-id.md b/content/riak/kv/2.2.6/developing/api/protocol-buffers/set-client-id.md new file mode 100644 index 0000000000..b0817a8dff --- /dev/null +++ b/content/riak/kv/2.2.6/developing/api/protocol-buffers/set-client-id.md @@ -0,0 +1,62 @@ +--- +title: "PBC Set Client ID" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Set Client ID" + identifier: "pbc_set_client_id" + weight: 126 + parent: "apis_pbc" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/references/protocol-buffers/set-client-id + - /riak-docs/riak/kv/2.2.6/dev/references/protocol-buffers/set-client-id +--- + +{{% note title="Deprecation notice" %}} +The use of client IDs in conflict resolution is now deprecated in Riak. If you +are building or maintaining a Riak client that is intended to be compatible +with Riak 1.4 or later, you can safely ignore client IDs. +{{% /note %}} + +Set the client ID for this connection. A library may want to set the +client ID if it has a good way to uniquely identify actors across +reconnects. This will reduce vector clock bloat. + +## Request + +```protobuf +message RpbSetClientIdReq { + required bytes client_id = 1; // Client id to use for this connection +} +``` + + +## Response + +Just the `RpbSetClientIdResp` message code. + +## Example + +Request + +``` +Hex 00 00 00 07 05 0A 04 01 65 01 B6 +Erlang <<0,0,0,7,5,10,4,1,101,1,182>> + +RpbSetClientIdReq protoc decode: +client_id: "001e001266" + +``` + + +Response + +``` +Hex 00 00 00 01 06 +Erlang <<0,0,0,1,6>> + +RpbSetClientIdResp - only message code defined +``` diff --git a/content/riak/kv/2.2.6/developing/api/protocol-buffers/store-object.md b/content/riak/kv/2.2.6/developing/api/protocol-buffers/store-object.md new file mode 100644 index 0000000000..07e99942a8 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/api/protocol-buffers/store-object.md @@ -0,0 +1,150 @@ +--- +title: "PBC Store Object" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Store Object" + identifier: "pbc_store_object" + weight: 106 + parent: "apis_pbc" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/references/protocol-buffers/store-object + - /riak-docs/riak/kv/2.2.6/dev/references/protocol-buffers/store-object +--- + +Stores an object under the specified location, as determined by the +intended [key]({{}}riak/kv/2.2.6/learn/concepts/keys-and-objects), [bucket]({{}}riak/kv/2.2.6/learn/concepts/buckets), and [bucket type]({{}}riak/kv/2.2.6/developing/usage/bucket-types). A bucket must always be specified (via +`bucket`), whereas key (`key`) and bucket type (`type`) are optional. If +no key is specified, Riak will assign a random key to the object. If no +[bucket type]({{}}riak/kv/2.2.6/developing/usage/bucket-types) is assigned, Riak will assign +`default`, which means that the [default bucket configuration]({{}}riak/kv/2.2.6/configuring/reference/#default-bucket-properties) will be used. + +#### Request + +```protobuf +message RpbPutReq { + required bytes bucket = 1; + optional bytes key = 2; + optional bytes vclock = 3; + required RpbContent content = 4; + optional uint32 w = 5; + optional uint32 dw = 6; + optional bool return_body = 7; + optional uint32 pw = 8; + optional bool if_not_modified = 9; + optional bool if_none_match = 10; + optional bool return_head = 11; + optional uint32 timeout = 12; + optional bool asis = 13; + optional bool sloppy_quorum = 14; + optional uint32 n_val = 15; + optional bytes type = 16; +} +``` + +#### Required Parameters + +Parameter | Description +:---------|:----------- +`bucket` | The name of the bucket, in bytes, in which the key/value is to reside +`content` | The new or updated contented of the object. Uses the same `RpbContent` message returned as part of an `RpbGetResp` message, documented in [PBC Fetch Object]({{}}riak/kv/2.2.6/developing/api/protocol-buffers/fetch-object) + +#### Optional Parameters + +{{% note title="Note on defaults and special values" %}} +All of the optional parameters below have default values determined on a +per-bucket basis. Please refer to the documentation on [setting bucket properties](../set-bucket-props) for more information. + +Furthermore, you can assign an integer value to the `w`, `dw`, `pr`, and +`pw`, provided that that integer value is less than or equal to N, _or_ +a special value denoting `one` (`4294967295-1`), `quorum` +(`4294967295-2`), `all` (`4294967295-3`), or `default` (`4294967295-4`). +{{% /note %}} + +Parameter | Description +:---------|:----------- +`key` | The key to create/update. If not specified, Riak will generate a random key and return that key as part of the response to that request. +`vclock` | Opaque vector clock provided by an earlier RpbGetResp message. Omit if this is a new key or if you deliberately want to create a sibling. +`w` | Write quorum, i.e. how many replicas to write to before returning a successful response +`dw` | Durable write quorum, i.e. how many replicas to commit to durable storage before returning a successful response +`return_body` | Whether to return the contents of the now-stored object. Defaults to `false`. +`pw` | Primary write quorum, i.e. how many primary nodes must be up when the write is attempted +`return_head` | Return the metadata for the now-stored object without returning the value of the object +`timeout` | The timeout duration, in milliseconds, after which Riak will return an error message +`sloppy_quorum` | If this parameter is set to `true`, the next available node in the ring will accept requests if any primary node is unavailable +`n_val` | The number of nodes on which the value is to be stored + +The `if_not_modified`, `if_none_match`, and `asis` parameters are set +only for messages sent between nodes in a Riak cluster and should not be +set by Riak clients. + +#### Response + +```bash +message RpbPutResp { + repeated RpbContent contents = 1; + optional bytes vclock = 2; + optional bytes key = 3; +} +``` + +If `return_body` is set to `true` on the PUT request, the `RpbPutResp` +will contain the current object after the PUT completes, in `contents`, +as well as the object's [causal context]({{}}riak/kv/2.2.6/learn/concepts/causal-context), in the `vclock` +field. The `key` will be sent only if the server generated a random key +for the object. + +If `return_body` is not set and no key is generated, the PUT response +will be empty. + +## Example + +#### Request + +``` +Hex 00 00 00 1C 0B 0A 01 62 12 01 6B 22 0F 0A 0D 7B + 22 66 6F 6F 22 3A 22 62 61 72 22 7D 28 02 38 01 +Erlang <<0,0,0,28,11,10,1,98,18,1,107,34,15,10,13,123,34,102,111,111,34,58,34, + 98,97,114,34,125,40,2,56,1>> + +RpbPutReq protoc decode: +bucket: "b" +key: "k" +content { + value: "{"foo":"bar"}" +} +w: 2 +return_body: true + +``` + +#### Response + +``` +Hex 00 00 00 62 0C 0A 31 0A 0D 7B 22 66 6F 6F 22 3A + 22 62 61 72 22 7D 2A 16 31 63 61 79 6B 4F 44 39 + 36 69 4E 41 68 6F 6D 79 65 56 6A 4F 59 43 38 AF + B0 A3 DE 04 40 90 E7 18 12 2C 6B CE 61 60 60 60 + CA 60 CA 05 52 2C 2C E9 0C 86 19 4C 89 8C 79 AC + 0C 5A 21 B6 47 F9 20 C2 6C CD 49 AC 0D 77 7C A0 + 12 FA 20 89 2C 00 +Erlang <<0,0,0,98,12,10,49,10,13,123,34,102,111,111,34,58,34,98,97,114,34,125, + 42,22,49,99,97,121,107,79,68,57,54,105,78,65,104,111,109,121,101,86, + 106,79,89,67,56,175,176,163,222,4,64,144,231,24,18,44,107,206,97,96, + 96,96,202,96,2.2.6,82,44,44,233,12,134,25,76,137,140,121,172,12,90,33, + 182,71,249,32,194,108,205,73,172,13,119,124,160,18,250,32,137,44,0>> + +RpbPutResp protoc decode: +contents { + value: "{"foo":"bar"}" + vtag: "1caykOD96iNAhomyeVjOYC" + last_mod: 1271453743 + last_mod_usecs: 406416 +} +vclock: "k316a```312`312005R,,351014206031L211214y254014Z!266G371 +302l315I254rw|240022372 211,000" + +``` diff --git a/content/riak/kv/2.2.6/developing/api/protocol-buffers/yz-index-delete.md b/content/riak/kv/2.2.6/developing/api/protocol-buffers/yz-index-delete.md new file mode 100644 index 0000000000..319e4c6b93 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/api/protocol-buffers/yz-index-delete.md @@ -0,0 +1,33 @@ +--- +title: "PBC Yokozuna Index Delete" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Yokozuna Index Delete" + identifier: "pbc_yz_index_delete" + weight: 122 + parent: "apis_pbc" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/references/protocol-buffers/yz-index-delete + - /riak-docs/riak/kv/2.2.6/dev/references/protocol-buffers/yz-index-delete +--- + +Delete a search index. + +## Request + +The `name` parameter is the name of the index to delete, as a binary. + +```protobuf +message RpbYokozunaIndexDeleteReq { + required bytes name = 1; +} +``` + +## Response + +Returns a [RpbDelResp]({{}}riak/kv/2.2.6/developing/api/protocol-buffers/#message-codes) code with no data on success. + diff --git a/content/riak/kv/2.2.6/developing/api/protocol-buffers/yz-index-get.md b/content/riak/kv/2.2.6/developing/api/protocol-buffers/yz-index-get.md new file mode 100644 index 0000000000..5d167ba86c --- /dev/null +++ b/content/riak/kv/2.2.6/developing/api/protocol-buffers/yz-index-get.md @@ -0,0 +1,59 @@ +--- +title: "PBC Yokozuna Index Get" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Yokozuna Index Get" + identifier: "pbc_yz_index_get" + weight: 120 + parent: "apis_pbc" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/references/protocol-buffers/yz-index-get + - /riak-docs/riak/kv/2.2.6/dev/references/protocol-buffers/yz-index-get +--- + +Retrieve a search index from Riak Search. + +## Request + +The `name` parameter is the name of the index to fetch as a binary. + +```protobuf +message RpbYokozunaIndexGetReq { + optional bytes name = 1; +} +``` + +## Response + +If a `name` is passed through the `RpbYokozunaIndexGetReq` request, zero +or one `index` objects are returned. If `name` is empty, then a list of +all indexes will be returned. + +Both requests will return a response of this form. + +```protobuf +message RpbYokozunaIndexGetResp { + repeated RpbYokozunaIndex index = 1; +} +``` + +This message will contain any number of `RpbYokozunaIndex` messages, +depending on how many indexes are returned. + +```protobuf +message RpbYokozunaIndex { + required bytes name = 1; + optional bytes schema = 2; + optional uint32 n_val = 3; +} +``` + +Each message specifying an index must include the index's name as a +binary (as `name`). Optionally, you can specify a [`schema`]({{}}riak/kv/2.2.6/developing/usage/search-schemas) name and/or an `n_val`, i.e. the number of nodes on which the +index is stored (for GET requests) or on which you wish the index to be +stored (for PUT requests). An index's `n_val` must match the associated +bucket's `n_val`. diff --git a/content/riak/kv/2.2.6/developing/api/protocol-buffers/yz-index-put.md b/content/riak/kv/2.2.6/developing/api/protocol-buffers/yz-index-put.md new file mode 100644 index 0000000000..bc832e3c8d --- /dev/null +++ b/content/riak/kv/2.2.6/developing/api/protocol-buffers/yz-index-put.md @@ -0,0 +1,45 @@ +--- +title: "PBC Yokozuna Index Put" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Yokozuna Index Put" + identifier: "pbc_yz_index_put" + weight: 121 + parent: "apis_pbc" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/references/protocol-buffers/yz-index-put + - /riak-docs/riak/kv/2.2.6/dev/references/protocol-buffers/yz-index-put +--- + +Create a new index or modify an existing index. + +## Request + +```protobuf +message RpbYokozunaIndexPutReq { + required RpbYokozunaIndex index = 1; +} +``` + +Each message must contain a `RpbYokozunaIndex` message providing +information about the index being stored. + +```protobuf +message RpbYokozunaIndex { + required bytes name = 1; + optional bytes schema = 2; + optional uint32 n_val = 3; +} +``` + +Each message specifying an index must include the index's name as a +binary (as `name`). Optionally, you can specify a [`schema`]({{}}riak/kv/2.2.6/developing/usage/search-schemas) name and/or an `n_val`, i.e. the number of nodes on which the index is stored (for GET requests) or on which you wish the index to be stored (for PUT requests). An index's `n_val` must match the associated bucket's `n_val`. + +## Response + +Returns a [RpbPutResp]({{}}riak/kv/2.2.6/developing/api/protocol-buffers/#message-codes) code with no data on success. + diff --git a/content/riak/kv/2.2.6/developing/api/protocol-buffers/yz-schema-get.md b/content/riak/kv/2.2.6/developing/api/protocol-buffers/yz-schema-get.md new file mode 100644 index 0000000000..2921008968 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/api/protocol-buffers/yz-schema-get.md @@ -0,0 +1,48 @@ +--- +title: "PBC Yokozuna Schema Get" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Yokozuna Schema Get" + identifier: "pbc_yz_schema_get" + weight: 123 + parent: "apis_pbc" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/references/protocol-buffers/yz-schema-get + - /riak-docs/riak/kv/2.2.6/dev/references/protocol-buffers/yz-schema-get +--- + +Fetch a [search schema]({{}}riak/kv/2.2.6/developing/usage/search-schemas) from Riak Search. + +## Request + +In a request message, you only need to specify the name of the schema as +a binary (under `name`); + +```protobuf +message RpbYokozunaSchemaGetReq { + required bytes name = 1; // Schema name +} +``` + +## Response + +```protobuf +message RpbYokozunaSchemaGetResp { + required RpbYokozunaSchema schema = 1; +} +``` + +The response message will include a `RpbYokozunaSchema` structure. + +```protobuf +message RpbYokozunaSchema { + required bytes name = 1; + optional bytes content = 2; +} +``` + +This message includes the schema `name` and its xml `content`. diff --git a/content/riak/kv/2.2.6/developing/api/protocol-buffers/yz-schema-put.md b/content/riak/kv/2.2.6/developing/api/protocol-buffers/yz-schema-put.md new file mode 100644 index 0000000000..a89201b3e8 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/api/protocol-buffers/yz-schema-put.md @@ -0,0 +1,41 @@ +--- +title: "PBC Yokozuna Schema Put" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Yokozuna Schema Put" + identifier: "pbc_yz_schema_put" + weight: 124 + parent: "apis_pbc" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/references/protocol-buffers/yz-schema-put + - /riak-docs/riak/kv/2.2.6/dev/references/protocol-buffers/yz-schema-put +--- + +Create a new Solr [search schema]({{}}riak/kv/2.2.6/developing/usage/search-schemas). + +## Request + +```protobuf +message RpbYokozunaSchemaPutReq { + required RpbYokozunaSchema schema = 1; +} +``` + +Each message must contain a `RpbYokozunaSchema` object structure. + +```protobuf +message RpbYokozunaSchema { + required bytes name = 1; + optional bytes content = 2; +} +``` + +This message *must* include both the schema `name` and its Solr [search schema]({{}}riak/kv/2.2.6/developing/usage/search-schemas) `content` as XML. + +## Response + +Returns a [RpbPutResp]({{}}riak/kv/2.2.6/developing/api/protocol-buffers/#message-codes) code with no data on success. diff --git a/content/riak/kv/2.2.6/developing/api/repl-hooks.md b/content/riak/kv/2.2.6/developing/api/repl-hooks.md new file mode 100644 index 0000000000..c4acbacb4e --- /dev/null +++ b/content/riak/kv/2.2.6/developing/api/repl-hooks.md @@ -0,0 +1,192 @@ +--- +title_supertext: "Riak Multi-Datacenter Replication:" +title: "Hooks API" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Multi-Datacenter REPL Hooks API" + identifier: "apis_repl_hooks" + weight: 100 + parent: "developing_apis" +toc: true +commercial_offering: true +aliases: + - /riak-docs/riak/2.2.6/ops/mdc/v2/hooks + - /riak-docs/riak/kv/2.2.6/ops/mdc/v2/hooks +--- +[object]: https://github.com/basho/riak_kv/blob/master/src/riak_object.erl + +This document is a guide to developing extensions for Riak's +Multi-Datacenter Replication feature. + +## Replication Hooks + +Riak allows applications to register replication hooks to control +either of the following: + +* when extra objects need to be replicated along with the current object +* when an object should _not_ be replicated. + +To register a hook, you must call the following function in an +application-specific Erlang module, where `MyMod` is to be replaced +with the name of your custom module: + +```erlang +riak_core:register([{repl_helper, MyMod}]). +``` + +## Replication Hook API + +A replication hook must implement the following functions: + +### send_realtime/2 + +```erlang +(riak_object, RiakClient) -> ok | cancel | [riak_object] +``` + +This hook controls whether an [object][object] +replicated in realtime should be sent. To send this object, return `ok`; +to prevent the object from being sent, return `cancel`. You can also +return a list of Riak objects to be replicated immediately *before* the +current object. This is useful when you have an object that refers to +other objects, e.g. a chunked file, and want to ensure that all of the +dependency objects are replicated before the dependent object. + +### send/2 + +```erlang +(riak_object, RiakClient) -> ok | cancel | [riak_object] +``` + +This hook is used in fullsync replication. To send this +[object][object], +return `ok`; to prevent the object from being sent, return `cancel`. You +can also return a list of Riak objects to be replicated immediately +*before* the current object. This is useful for when you have an object +that refers to other objects, e.g. a chunked file, and want ensure that +all the dependency objects are replicated before the dependent object. + +### recv/1 + +```erlang +(riak_object) -> ok | cancel +``` + +When an [object][object] +is received by the client site, this hook is run. You can use it to +update metadata or to deny the object. + +## Implementing a Sample Replication Hook + +The following is a simple replication hook that will log when an object +is received via replication. For more information about the functions in +the sample, see the [Replication Hook API](#replication-hook-api) section below. + +Here is the relevant Erlang code: + +```erlang +%% Riak Enterprise MDC replication hook sample + +-module(riak_replication_hook_sample). +-export([register/0]). +-export([recv/1, send/2, send_realtime/2]). + +register() -> + riak_core:wait_for_service(riak_repl), + lager:log(info, self(), + "Automatically registering ~p hook with riak_core", + [?MODULE_STRING]), + riak_core:register([{repl_helper, ?MODULE}]), + case lists:member({undefined,?MODULE}, + app_helper:get_env(riak_core,repl_helper, [])) of + true -> + lager:log(info, self(), + "Successfully registered ~p hook with riak_core", + [?MODULE_STRING]); + false -> + lager:log(info, self(), + "Failed to register ~p hook with riak_core", + [?MODULE_STRING]) + end, + ok. + +recv(Object) -> + % This is a BLOCKING function. + % Longer-running processes should be handled asynchronously. + lager:log(info, self(), "Called recv(~p)", [riak_object:key(Object)]), + ok. + +send_realtime(_Object, _RiakClient) -> + % Do Nothing function -- These hooks are called in predictable + % but complex ways especially as the number of replication + % sites (Version 2 Replication) or sinks (Version 3 Replication) + % increase. + ok. + +send(_Object, _RiakClient) -> + % Do Nothing function -- These hooks are called in predictable + % but complex ways especially as the number of replication + % sites (Version 2 Replication) or sinks (Version 3 Replication) + % increase. + ok. +``` + +Save the above code as `riak_replication_hook_sample.erl`. + +To install the sample hook, compile `riak_replication_hook_sample.erl`. + +{{% note title="Note on the Erlang compiler" %}} + +[erlc]: http://erlang.org/doc/man/erlc.html +You must use the Erlang compiler [`erlc`][erlc] +associated with the Riak installation or the version of Erlang used when +compiling Riak from source. For packaged Riak installations, you can +consult **Table 1** (below) for the default location of +Riak’s `erlc` for each supported platform. If you compiled +from source, use the `erlc` from the Erlang version you used +to compile Riak. +{{% /note %}} + +Distribution | Path +:------------|:---- +CentOS & RHEL Linux | `/usr/lib64/riak/erts-5.10.3/bin/erlc` | +Debian & Ubuntu Linux | `/usr/lib/riak/erts-5.10.3/bin/erlc` | +FreeBSD | `/usr/local/lib/riak/erts-5.10.3/bin/erlc` | +SmartOS | `/opt/local/lib/riak/erts-5.10.3/bin/erlc` +Solaris 10 | `/opt/riak/lib/erts-5.10.3/bin/erlc` + +**Table 1**: Erlang compiler executable location for packaged Riak +installations on supported platforms + +Once you have determined the location of the Erlang compiler, e.g. on +Ubuntu, compiling is as simple as: + +```bash +/usr/lib/riak/erts-5.10.3/bin/erlc riak_replication_hook_sample.erl +``` + +This will create a `riak_replication_hook_sample.beam` file in the same +directory as the corresponding `.erl` file. Copy this `.beam` file into +the subdirectory where you want to store the custom hook: + +```bash +cp riak_replication_hook_sample.beam /path/to/replication/hook +``` + +Add a `-pa` argument to your `vm.args` file to specify the path where +your compiled `.beam` file lives: + +```bash +-pa /path/to/replication/hook +``` + +Finally, add a `-run` argument to your `vm.args` file to register the +hook: + +```bash +-run riak_replication_hook_sample register +``` + diff --git a/content/riak/kv/2.2.6/developing/app-guide.md b/content/riak/kv/2.2.6/developing/app-guide.md new file mode 100644 index 0000000000..64296ff990 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/app-guide.md @@ -0,0 +1,415 @@ +--- +title: "Riak KV Application Guide" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Application Guide" + identifier: "developing_app_guide" + weight: 105 + parent: "developing" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/using/application-guide/ + - /riak-docs/riak/kv/2.2.6/dev/using/application-guide/ +--- + +[usage conflict resolution]: {{}}riak/kv/2.2.6/developing/usage/conflict-resolution +[dev data model#log]: {{}}riak/kv/2.2.6/developing/data-modeling/#log-data +[dev data model#sensor]: {{}}riak/kv/2.2.6/developing/data-modeling/#sensor-data +[concept eventual consistency]: {{}}riak/kv/2.2.6/learn/concepts/eventual-consistency +[dev data model#user]: {{}}riak/kv/2.2.6/developing/data-modeling/#user-data +[dev kv model]: {{}}riak/kv/2.2.6/developing/key-value-modeling +[dev data types]: {{}}riak/kv/2.2.6/developing/data-types +[dev data types#counters]: {{}}riak/kv/2.2.6/developing/data-types/#counters +[dev data types#sets]: {{}}riak/kv/2.2.6/developing/data-types/#sets +[dev data types#maps]: {{}}riak/kv/2.2.6/developing/data-types/#maps +[usage create objects]: {{}}riak/kv/2.2.6/developing/usage/creating-objects +[usage search]: {{}}riak/kv/2.2.6/developing/usage/search +[use ref search]: {{}}riak/kv/2.2.6/using/reference/search +[usage 2i]: {{}}riak/kv/2.2.6/developing/usage/secondary-indexes +[dev client libraries]: {{}}riak/kv/2.2.6/developing/client-libraries +[concept crdts]: {{}}riak/kv/2.2.6/learn/concepts/crdts +[dev data model]: {{}}riak/kv/2.2.6/developing/data-modeling +[usage mapreduce]: {{}}riak/kv/2.2.6/developing/usage/mapreduce +[apps mapreduce]: {{}}riak/kv/2.2.6/developing/app-guide/advanced-mapreduce +[use ref 2i]: {{}}riak/kv/2.2.6/using/reference/secondary-indexes +[plan backend leveldb]: {{}}riak/kv/2.2.6/setup/planning/backend/leveldb +[plan backend bitcask]: {{}}riak/kv/2.2.6/setup/planning/backend/bitcask +[plan backend memory]: {{}}riak/kv/2.2.6/setup/planning/backend/memory +[obj model java]: {{}}riak/kv/2.2.6/developing/getting-started/java/object-modeling +[obj model ruby]: {{}}riak/kv/2.2.6/developing/getting-started/ruby/object-modeling +[obj model python]: {{}}riak/kv/2.2.6/developing/getting-started/python/object-modeling +[obj model csharp]: {{}}riak/kv/2.2.6/developing/getting-started/csharp/object-modeling +[obj model nodejs]: {{}}riak/kv/2.2.6/developing/getting-started/nodejs/object-modeling +[obj model erlang]: {{}}riak/kv/2.2.6/developing/getting-started/erlang/object-modeling +[obj model golang]: {{}}riak/kv/2.2.6/developing/getting-started/golang/object-modeling +[concept strong consistency]: {{}}riak/kv/2.2.6/using/reference/strong-consistency +[use ref strong consistency]: {{}}riak/kv/2.2.6/using/reference/strong-consistency +[cluster ops strong consistency]: {{}}riak/kv/2.2.6/using/cluster-operations/strong-consistency +[config strong consistency]: {{}}riak/kv/2.2.6/configuring/strong-consistency +[apps strong consistency]: {{}}riak/kv/2.2.6/developing/app-guide/strong-consistency +[usage update objects]: {{}}riak/kv/2.2.6/developing/usage/updating-objects +[apps replication properties]: {{}}riak/kv/2.2.6/developing/app-guide/replication-properties +[install index]: {{}}riak/kv/2.2.6/setup/installing +[getting started]: {{}}riak/kv/2.2.6/developing/getting-started +[usage index]: {{}}riak/kv/2.2.6/developing/usage +[glossary]: {{}}riak/kv/2.2.6/learn/glossary + +So you've decided to build an application using Riak as a data store. We +think that this is a wise choice for a broad variety of use cases. But +using Riak isn't always straightforward, especially if you're used to +developing with relational databases like like MySQL or PostgreSQL or +non-persistent key/value stores like Redis. So in this guide, we'll walk +you through a set of questions that should be asked about your use case +before getting started. The answer to those questions may inform +decisions about which Riak features you should use, what kind of +replication and conflict resolution strategies you should employ, and +perhaps even how parts of your application should be built. + +## What Kind of Data Are You Storing? + +This is an important initial question for two reasons: + +1. Not all data is a good fit for Riak. If your data isn't a good fit, +we would advise that you seek out a storage system that better suits +your needs. +2. The kinds of data that you're storing should guide your decision both +about _how_ to store and access your data in Riak and about which Riak +features would be helpful (and which ones might even be harmful). + +### Good Fits for Riak + +Riak tends to be an excellent choice if you're dealing with any of the +following: + +* **Immutable data** --- While Riak provides several means of + [resolving conflicts][usage conflict resolution] between different replicas + of objects, those processes can lead to slower performance in some + cases. Storing immutable data means that you can avoid those processes + altogether and get the most out of Riak. +* **Small objects** --- Riak was not built as a store for large objects + like video files or other + [BLOB](http://en.wikipedia.org/wiki/Binary_large_object)s. We built + [Riak CS](http://basho.com/riak-cloud-storage/) for that. Riak is + great, however, for JSON, [log files][dev data model#log], [sensor data][dev data model#sensor], HTML files, and other objects that tend + to run smaller than 1 MB. +* **Independent objects** --- Objects that do not have interdependencies + on other objects are a good fit for Riak's [eventually consistent][concept eventual consistency] nature. +* **Objects with "natural" keys** --- It is almost always advisable to + build keys for objects out of timestamps, [usernames][dev data model#user], + or other ["natural" markers][dev kv model] that distinguish + that object from other objects. Data that can be modeled this way fits + nicely with Riak because Riak emphasizes extremely fast object lookup. +* **Data compatible with [Riak Data Types][dev data types]** --- If + you're working with mutable data, one option is to run basic CRUD + operations on that data in a standard key/value fashion and either + manage conflict resolution yourself or allow Riak to do so. But if + your data can be modeled as a [counter][dev data types#counters], + [set][dev data types#sets], or [map][dev data types#maps], you + should seriously consider using [Riak Data Types][dev data types], + which can speed application development and transfer a great deal of + complexity away from the application and to Riak itself. + +### Not-so-good Fits for Riak + +Riak may not such be a good choice if you use it to store: + +* **Objects that exceed 1-2MB in size** --- If you will be + storing a lot of objects over that size, we would recommend checking + out [Riak CS](http://docs.basho.com/riakcs/latest/) instead, as Riak + CS was built to solve this problem. Storing large objects in Riak will + typically lead to substandard performance. +* **Objects with complex interdependencies** --- If your data cannot be + easily denormalized or if it requires that objects can be easily + assembled into and accessible as larger wholes---think columns or + tables---then you might want to consider a relational database + instead. + +### Conclusion + +If it sounds like Riak is a good choice for some or all of your +application's data needs, move on to the next sections, where you can +find out more about which Riak features are recommendable for your use +case, how you should model your data, and what kinds of data modeling +and development strategies we recommend. + +## Which Features Should You Consider? + +Basic CRUD key/value operations are almost always the most performant +operations when using Riak. If your needs can be served using CRUD +operations, we recommend checking out our tutorial on [key/value modeling][dev kv model] for some basic guidelines. But if basic CRUD key/value +operations don't quite suffice for your use case, Riak offers a variety +of features that may be just what you're looking for. In the sections +immediately below, you can find brief descriptions of those features as +well as relevant links to Basho documentation. + +## Search + +Riak Search provides you with [Apache +Solr](http://lucene.apache.org/solr/)-powered full-text indexing and +querying on top of the scalability, fault tolerance, and operational +simplicity of Riak. Our motto for Riak Search: **Write it like Riak. +Query it like Solr**. That is, you can store objects in Riak [like normal][usage create objects] and run full-text queries on those objects later on +using the Solr API. + +* [Using Search][usage search] --- Getting started with Riak Search +* [Search Details][use ref search] --- A detailed overview of the concepts and design + consideration behind Riak Search +* [Search Schema][usage search schema] --- How to create custom schemas for extracting data + from Riak Search + +### When to Use Search + +* **When you need a rich querying API** --- Riak Search gives you access + to the entirety of [Solr](http://lucene.apache.org/solr/)'s extremely + broad API, which enables you to query on the basis of wildcards, + strings, booleans, geolocation, ranges, language-specific fulltext, + and far more. You can even use Search in conjunction with [Riak Data Types][dev data types] \(documentation coming soon). + +> **Search is preferred for querying** +> +> In general, you should consider Search to be the default choice for +nearly all querying needs that go beyond basic CRUD/KV operations. If +your use case demands some sort of querying mechanism and you're in +doubt about what to use, you should assume that Search is the right tool +for you. + +### When Not to Use Search + +* **When deep pagination is needed** --- At the moment, you should + consider [secondary indexes][usage 2i] instead of + Search if your use case requires deep pagination. This will be + changed, however, in a future release of Riak, at which point you + should consider Search the default choice for _all_ querying needs. +* **In large clusters** --- In clusters larger than 8-10 nodes, you may + experience slower performance when using Search. In clusters of that + size, we would recommend using Search in a limited fashion, setting + up a separate, dedicated cluster for Search data, or finding another + solution. + +## Riak Data Types + +When performing basic K/V operations, Riak is agnostic toward the actual +data stored within objects. Beginning with Riak 2.0, however, you now +have access to operations-based objects based on academic research on +[CRDTs](http://hal.upmc.fr/docs/00/55/55/88/PDF/techreport.pdf). Riak +Data Types enable you to update and read [counters][dev data types#counters], +[sets][dev data types#sets], and [maps][dev data types#maps] directly in Riak, as well as [registers][dev data types#maps] and [flags][dev data types#maps] inside of Riak maps. + +The beauty of Riak Data Types is that all convergence logic is handled +by Riak itself according to deterministic, Data Type-specific rules, +which means that your application doesn't need to reason about +[siblings][usage conflict resolution]. In many cases, this can +unburden applications of the need to handle object convergence on their +own. + +* [Using Data Types][dev data types] --- A guide to setting up Riak to use Data Types, + including a variety of code samples for all of the Basho's official + [client libraries][dev client libraries] +* [Data Types][concept crdts] --- A theoretical treatment of Riak Data Types, along + with implementation details +* [Data Modeling with Riak Data Types][dev data model] --- An object modeling example that relies on Riak Data Types. + +> **Note**: +> +> Riak Data Types can be used in conjunction with Riak Search, +meaning that the data stored in counters, sets, and maps can be indexed +and searched just like any other data in Riak. Documentation on Data +Types and Search is coming soon. + +### When to Use Riak Data Types + +* **When your data fits** --- If the data that you're storing can be + modeled as one of the five available types, Riak Data Types could be a + very good option. Please note that in many cases there may not be a + 1:1 correspondence between the five available types and the data that + you'd like to store, but there may be workarounds to close the gap. + Most things that can be stored as JSON, for example, can be stored as + maps (though with modifications). +* **When you don't need to reason about siblings** --- If your use case + doesn't require that your application have access to siblings and + allows for sibling convergence logic to take place at the Riak level + rather than at the application level, then Riak Data Types are well + worth exploring. + +### When Not to Use Riak Data Types + +* **When you need to provide your own convergence logic** --- If your + application needs to have access to all sibling values, then Riak Data + Types are not a good choice because they by definition do not produce + siblings. +* **When your data just doesn't fit** --- While the five existing Data + Types allow for a great deal of flexibility and a wide range of use + cases, they don't cover all use cases. If you have data that requires + a modeling solution that can't be covered, you should stick to + standard K/V operations. +* **When object size is of significant concern** --- Riak Data Types + behave much like other Riak objects, but they tend to carry more + metadata than normal Riak objects, especially maps. In most cases the + metadata payload will be a small percentage of the object's total + size, but if you want to keep objects as lean as possible, it may be + better to stick to normal K/V operations. + +## MapReduce + +Riak's MapReduce feature enables you to perform batch processing jobs in +a way that leverages Riak's distributed nature. When a MapReduce job is +sent to Riak, Riak automatically distributes the processing work to +where the target data lives, which can reduce network bandwidth. Riak +comes equipped with a set of default MapReduce jobs that you can employ, +or you can write and run your own MapReduce jobs in +[Erlang](http://www.erlang.org/). + +* [Using MapReduce][usage mapreduce] --- A general guide to using MapReduce +* [Advanced MapReduce][apps mapreduce] --- A more in-depth guide to MapReduce, + including code samples and implementation details + +### When to Use MapReduce + +* **Batch processing only** --- You should use MapReduce only when truly + truly necessary. MapReduce jobs are very computationally expensive and + can degrade performance in production clusters. You should restrict + MapReduce usage to infrequent batch processing operations, preferably + carried out at times when your cluster is experiencing load that is + well below average. + +### When Not to Use MapReduce + +* **When another Riak feature will do** --- Before even considering + using MapReduce, you should thoroughly investigate [Riak Search][usage search] or [secondary indexes][usage 2i] as possible + solutions to your needs. + +In general, you should not think of MapReduce as, for example, Hadoop +within Riak. While it can be useful for certain types of +non-primary-key-based queries, it is neither a "Big Data" processing +tool nor an indexing mechanism nor a replacement for [Riak Search][usage search]. If you do need a tool like Hadoop or Apache Spark, you should +consider using Riak in conjunction with a more suitable data processing +tool. + +## Secondary Indexes (2i) + +Using basic key/value operations in Riak sometimes leads to the +following problem: how do I know which keys I should look for? Secondary +indexes (2i) provide a solution to this problem, enabling you to tag +objects with either binary or integer metadata and then query Riak for +all of the keys that share specific tags. 2i is especially useful if +you're storing binary data that is opaque to features like [Riak Search][usage search]. + +* [Using Secondary Indexes][usage 2i] --- A general guide to using 2i, along + with code samples and information on 2i features like pagination, + streaming, and sorting +* [Advanced Secondary Indexes][use ref 2i] --- Implementation details behind 2i + +### When to Use Secondary Indexes + +* **When you require deep pagination** --- At the moment, 2i's + deep pagination capabilities are more performant than those offered + by Search if you require pagination of more than 3-5 pages. This + will change, however, in the future, at which point we will + recommend using Search instead. + +### When Not to Use Secondary Indexes + +* **For most querying purposes** --- If your use case does not + involve deep pagination, we recommend Search over 2i for _all_ + querying purposes. +* **If you're using Bitcask** --- 2i is available only in the + [LevelDB][plan backend leveldb] backend. If you'd like to use [Bitcask][plan backend bitcask] or the [Memory][plan backend memory] backend, you will not be able to use 2i. + +## Mixed Approach + +One thing to always bear in mind is that Riak enables you to mix and +match a wide variety of approaches in a single cluster. You can use +basic CRUD operations for some of your data, index some of your data to +be queried by Riak Search, use Riak Data Types for another subset, etc. +You are always free to use a wide array of Riak features---or you can +use none at all and stick to key/value operations. + +## How Should You Model Your Data? + +It's difficult to offer universally applicable data modeling guidelines +because data models differ so markedly from use case to use case. What +works when storing [user data][dev data model#user], for example, might +be a poor fit when working with [sensor data][dev data model#sensor]. +Nonetheless, there's a variety of material in our documentation that +might be helpful when thinking about data modeling: + +* Object Modeling in Riak KV: + - [Java][obj model java] + - [Ruby][obj model ruby] + - [Python][obj model python] + - [C#][obj model csharp] + - [NodeJS][obj model nodejs] + - [Erlang][obj model erlang] + - [Go][obj model golang] +* [Key/Value Modeling][dev kv model] + +### Data Types + +One feature to always bear in mind when using Riak is [Riak Data Types][dev data types]. If some or all of your data can be modeled in +accordance with one of the available Data Types---flags (similar to +Booleans), registers (good for storing small binaries or text snippets), +[counters][dev data types#counters], [sets][dev data types#sets], +or [maps][dev data types#maps]---you might be able to streamline +application development by using them as an alternative to key/value +operations. In some cases, it might even be worthwhile to transform your +data modeling strategy in accordance with To see if this feature might +be a good fit for your application, we recommend checking out the +following documentation: + +* [Data Types][concept crdts] +* [Using Data Types][dev data types] +* [Data Modeling with Riak Data Types][dev data model] + +## What are Your Consistency Requirements? + +Riak has traditionally been thought of as an [eventually consistent][concept eventual consistency], AP system, i.e. as a system that +favors availability and partition tolerance over data consistency. In +Riak versions 2.0 and later, the option of applying strong consistency +guarantees is available to developers that want to use Riak as a strict +CP system. One of the advantages of Riak's approach to strong +consistency is that you don't need to store all of your data in a +strongly consistent fashion if you use this feature. Instead, you can +mix and match a CP approach with an AP approach in a single cluster in +any way you wish. + +If you need some or all of your data to be subject to strong consistency +requirements, we recommend checking out the following documentation: + +* [Strong Consistency][use ref strong consistency] +* [Using Strong Consistency][apps strong consistency] +* [Managing Strong Consistency][cluster ops strong consistency] + +## Are Your Objects Mutable? + +Although Riak always performs best when storing and retrieving immutable +data, Riak also handles mutable objects very ably using a variety of +eventual consistency principles. Storing mutable data in Riak, however, +can get tricky because it requires you to choose and implement a +conflict resolution strategy for when object conflicts arise, which is a +normal occurrence in Riak. For more implementation details, we recommend +checking out the following docs: + +* [Conflict Resolution][usage conflict resolution] +* [Object Updates][usage update objects] +* [Replication Properties][apps replication properties] + +## Getting Started + +If you have a good sense of how you will be using Riak for your +application (or if you just want to experiment), the following guides +will help you get up and running: + +* [Installing Riak KV][install index] --- Install Riak KV and start up a 5-node Riak + cluster +* [Client Libraries][dev client libraries] --- A listing of official and non-official client + libraries for building applications with Riak +* [Getting Started with Client Libraries][getting started] --- How to + get up and going with one of Basho's official client libraries (Java, + Ruby, Python, and Erlang) +* [Developing with Riak KV: Usage][usage index] --- A guide to basic key/value operations and other common tasks in Riak KV. +* [Riak KV Glossary][glossary] --- A listing of frequently used terms in Riak's + documentation + diff --git a/content/riak/kv/2.2.6/developing/app-guide/advanced-mapreduce.md b/content/riak/kv/2.2.6/developing/app-guide/advanced-mapreduce.md new file mode 100644 index 0000000000..8c4b5ab4ce --- /dev/null +++ b/content/riak/kv/2.2.6/developing/app-guide/advanced-mapreduce.md @@ -0,0 +1,798 @@ +--- +title: "Advanced MapReduce" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Advanced MapReduce" + identifier: "app_guide_mapreduce" + weight: 103 + parent: "developing_app_guide" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/advanced/mapreduce/ + - /riak-docs/riak/kv/2.2.6/dev/advanced/mapreduce/ +--- + +[usage 2i]: {{}}riak/kv/2.2.6/developing/usage/secondary-indexes +[apps replication properties]: {{}}riak/kv/2.2.6/developing/app-guide/replication-properties +[use ref custom code]: {{}}riak/kv/2.2.6/using/reference/custom-code +[usage bucket types]: {{}}riak/kv/2.2.6/developing/usage/bucket-types +[glossary vnode]: {{}}riak/kv/2.2.6/learn/glossary/#vnode +[config reference]: {{}}riak/kv/2.2.6/configuring/reference +[google mr]: http://research.google.com/archive/mapreduce.html +[mapping list]: http://hackage.haskell.org/package/base-4.7.0.0/docs/Prelude.html#v:map +[function contrib]: https://github.com/basho/riak_function_contrib +[erlang client]: https://github.com/basho/riak-erlang-client +[`set-union`]: http://en.wikipedia.org/wiki/Union_(set_theory)#Definition + + +> **Use MapReduce sparingly** +> +> In Riak KV, MapReduce is the primary method for non-primary-key-based +querying. Although useful for tasks such as batch +processing jobs, MapReduce operations can be very computationally +expensive, to the extent that they can degrade performance in +production clusters operating under load. Because of this potential for performance degradation, we recommend running +MapReduce operations in a controlled, rate-limited fashion and never for +realtime querying purposes. + +MapReduce, the data processing paradigm popularized by +[Google][google mr], is provided by Riak KV to aggregate +results as background batch processes. + +## MapReduce + +In Riak KV, MapReduce is one of the primary methods for +non-primary-key-based querying alongside +[secondary indexes][usage 2i]. Riak KV allows you to +run MapReduce jobs using Erlang or JavaScript. + +{{% note title="Deprecation Warning" %}} +Javascript MapReduce is deprecated and will be removed in a future version. +{{% /note %}} + + +### Why Do We Use MapReduce for Querying Riak KV? + +Key/value stores like Riak KV generally do not offer the kinds of complex +querying capabilities found in other data storage systems, such as +relational databases. MapReduce enables you to perform powerful queries +over the data stored in Riak KV but should be used with caution. + +The main goal of MapReduce is to spread the processing of a query across +many systems to take advantage of parallel processing power. This is +generally done by dividing the query into several steps, i.e. dividing +the dataset into several chunks and then running those step/chunk pairs +on separate physical hosts. Riak KV's MapReduce has an additional goal: +increasing data locality. When processing a large dataset, it's often +much more efficient to take the computation to the data than it is to +bring the data to the computation. + +"Map" and "Reduce" are phases in the query process. Map functions take +one piece of data as input and produce zero or more results as output. +If you're familiar with [mapping over a list][mapping list] +in functional programming languages, you're already familiar with the +"Map" steps in a MapReduce query. + +## MapReduce caveats + +MapReduce should generally be treated as a fallback rather than a +standard part of an application. There are often ways to model data +such that dynamic queries become single key retrievals, which are +dramatically faster and more reliable in Riak KV, and tools such as Riak +search and 2i are simpler to use and may place less strain on a +cluster. + +### R=1 + +One consequence of Riak KV's processing model is that MapReduce queries +have an effective `R` value of 1. The queries are distributed +to a representative sample of the cluster where the data is expected to +be found, and if one server lacks a copy of data it's supposed to have, +a MapReduce job will not attempt to look for it elsewhere. + +For more on the value of `R`, see our documentation on [replication properties][apps replication properties]. + +### Key lists + +Asking Riak KV to generate a list of all keys in a production environment +is generally a bad idea. It's an expensive operation. + +Attempting to constrain that operation to a bucket (e.g., +`mapred_bucket` as used below) does not help because Riak KV must still +pull all keys from storage to determine which ones are in the +specified bucket. + +If at all possible, run MapReduce against a list of known keys. + +### Code distribution + +As we'll discuss in this document, the functions invoked from Erlang +MapReduce must be available on all servers in the cluster unless +using the client library from an Erlang shell. + +### Security restrictions + +If Riak's security functionality is enabled, there are two +restrictions on MapReduce that come into play: + +* The `riak_kv.mapreduce` permission must be granted to the user (or + via the user's groups) +* Other than the module `riak_kv_mapreduce`, any Erlang modules + distributed with Riak KV will **not** be accessible to custom MapReduce + code unless made available via the `add_path` mechanism documented + in [Installing Custom Code][use ref custom code]. + +## How Riak KV's MapReduce Queries Are Specified + +MapReduce queries in Riak KV have two components: (1) a list of inputs and +(2) a list of "steps," or "phases." + +Each element of the input list is an object location, as specified by +[bucket type][usage bucket types], bucket, and key. This location may +also be annotated with "key-data," which will be passed as an +argument to a map function when evaluated on the object stored under +that bucket-key pair. + +Each element of the phases list is a description of a map function, a +reduce function, or a link function. The description includes where to +find the code for the phase function (for map and reduce phases), static +data passed to the function every time it is executed during that phase, +and a flag indicating whether or not to include the results of that +phase in the final output of the query. + +The phase list describes the chain of operations through which each +input will flow. That is, the initial inputs will be fed to the first +phase in the list and the output of that phase will be fed as input to +the next phase in the list. This stream will continue through the final +phase. + +## How Phases Work + +### Map Phase + +The input list to a map phase must be a list of (possibly annotated) +bucket-key pairs. For each pair, Riak KV will send the request to evaluate +the map function to the partition that is responsible for storing the +data for that bucket-key. The [vnode][glossary vnode] hosting that partition +will look up the object stored under that bucket-key and evaluate the +map function with the object as an argument. The other arguments to the +function will be the annotation, if any is included, with the +bucket-key, and the static data for the phase, as specified in the +query. + +{{% note title="Tombstones" %}} +Be aware that most Riak KV clusters will retain deleted objects for some +period of time (3 seconds by default), and the MapReduce framework does +not conceal these from submitted jobs. These tombstones can be +recognized and filtered out by looking for `X-Riak-Deleted` +in the object metadata with a value of `true`. +{{% /note %}} + +### Reduce Phase + +Reduce phases accept any list of data as input, and produce any list of +data as output. They also receive a phase-static value, specified in the +query definition. + +The most important thing to understand is that the function defining the +reduce phase may be evaluated multiple times, and the input of later +evaluations will include the output of earlier evaluations. + +For example, a reduce phase may implement the +[`set-union`] function. In that case, the first set of inputs might be `[1,2.2.6]`, +and the output would be `[1,2,3]`. When the phase receives more inputs, +say `[3,4,5]`, the function will be called with the concatenation of the +two lists: `[1,2,3,3,4,5]`. + +Other systems refer to the second application of the reduce function as +a "re-reduce." There are at least a few reduce-query implementation +strategies that work with Riak KV's model. + +One strategy is to implement the phase preceding the reduce phase such +that its output is "the same shape" as the output of the reduce phase. +This is how the examples in this document are written, and the way that +we have found produces the cleanest code. + +An alternative strategy is to make the output of a reduce phase +recognizable such that it can be extracted from the input list on +subsequent applications. For example, if inputs from the preceding phase +are numbers, outputs from the reduce phase could be objects or strings. +This would allow the function to find the previous result and apply new +inputs to it. + +### How a Link Phase Works in Riak KV + +Link phases find links matching patterns specified in the query +definition. The patterns specify which buckets and tags links must have. + +"Following a link" means adding it to the output list of this phase. The +output of this phase is often most useful as input to a map phase or to +another reduce phase. + +## Invoking MapReduce + +To illustrate some key ideas, we'll define a simple module that +implements a map function to return the key value pairs contained in a +bucket and use it in a MapReduce query via Riak KV's HTTP API. + +Here is our example MapReduce function: + +```erlang +-module(mr_example). + +-export([get_keys/3]). + +% Returns bucket and key pairs from a map phase +get_keys(Value,_Keydata,_Arg) -> + [{riak_object:bucket(Value),riak_object:key(Value)}]. +``` + +Save this file as `mr_example.erl` and proceed to compiling the module. + +{{% note title="Note on the Erlang Compiler" %}} +You must use the Erlang compiler (`erlc`) associated with the +Riak KV installation or the version of Erlang used when compiling Riak KV from +source. +{{% /note %}} + +Compiling the module is a straightforward process: + +```bash +erlc mr_example.erl +``` + +Successful compilation will result in a new `.beam` file, `mr_example.beam`. + +Send this file to your operator, or read about [installing custom code][use ref custom code] +on your Riak KV nodes. Once your file has been installed, all that +remains is to try the custom function in a MapReduce query. For +example, let's return keys contained within a bucket named `messages` +(please pick a bucket which contains keys in your environment). + +```curl +curl -XPOST localhost:8098/mapred \ + -H 'Content-Type: application/json' \ + -d '{"inputs":"messages","query":[{"map":{"language":"erlang","module":"mr_example","function":"get_keys"}}]}' +``` + +The result should be a JSON map of bucket and key names expressed as key/value pairs. + +{{% note %}} +Be sure to install the MapReduce function as described above on all of +the nodes in your cluster to ensure proper operation. +{{% /note %}} + + +## Phase functions + +MapReduce phase functions have the same properties, arguments, and +return values whether you write them in Javascript or Erlang. + +### Map phase functions + +Map functions take three arguments (in Erlang, arity-3 is required). +Those arguments are: + + 1. `Value`: the value found at a key. This will be a Riak object, which + in Erlang is defined and manipulated by the `riak_object` module. + In Javascript, a Riak object looks like this: + + ```javascript + { + "bucket_type" : BucketTypeAsString, + "bucket" : BucketAsString, + "key" : KeyAsString, + "vclock" : VclockAsString, + "values" : [ + { + "metadata" : { + "X-Riak-VTag":VtagAsString, + "X-Riak-Last-Modified":LastModAsString, + "Links":[...List of link objects], + // ...other metadata... + }, + "data" : ObjectData + }, + // ...other metadata/data values (siblings)... + ] + } + ``` + 2. *KeyData* : key data that was submitted with the inputs to the query or phase. + 3. *Arg* : a static argument for the entire phase that was submitted with the query. + +A map phase should produce a list of results. You will see errors if +the output of your map function is not a list. Return the empty list if +your map function chooses not to produce output. If your map phase is +followed by another map phase, the output of the function must be +compatible with the input to a map phase - a list of bucket-key pairs or +`bucket-key-keydata` triples. + +#### Map function examples + +These map functions return the value (data) of the object being mapped: + +```erlang +fun(Value, _KeyData, _Arg) -> + [riak_object:get_value(Value)] +end. +``` + +These map functions filter their inputs based on the arg and return bucket-key pairs for a subsequent map phase: + +```erlang +fun(Value, _KeyData, Arg) -> + Key = riak_object:key(Value), + Bucket = riak_object:bucket(Value), + case erlang:byte_size(Key) of + L when L > Arg -> + [{Bucket,Key}]; + _ -> [] + end +end. +``` + +### Reduce phase functions + +Reduce functions take two arguments. Those arguments are: + +1. *ValueList*: the list of values produced by the preceding phase in the MapReduce query. +2. *Arg* : a static argument for the entire phase that was submitted with the query. + +A reduce function should produce a list of values, but it must also be +true that the function is commutative, associative, and idempotent. That +is, if the input list `[a,b,c,d]` is valid for a given F, then all of +the following must produce the same result: + + +```erlang + F([a,b,c,d]) + F([a,d] ++ F([c,b])) + F([F([a]),F([c]),F([b]),F([d])]) +``` + +#### Reduce function examples + +These reduce functions assume the values in the input are numbers and +sum them: + +```erlang +fun(Values, _Arg) -> + [lists:foldl(fun erlang:'+'/2, 0, Values)] +end. +``` + +These reduce functions sort their inputs: + +```erlang +fun(Values, _Arg) -> + lists:sort(Values) +end. +``` + +## MapReduce Examples + +Riak KV supports describing MapReduce queries in Erlang syntax through the +Protocol Buffers API. This section demonstrates how to do so using the +Erlang client. + +{{% note title="Distributing Erlang MapReduce Code" %}} +Any modules and functions you use in your Erlang MapReduce calls must be +available on all nodes in the cluster. Please read about +[installing custom code]({{}}riak/kv/2.2.6/using/reference/custom-code). +{{% /note %}} + +### Erlang Example + +Before running some MapReduce queries, let's create some objects to +run them on. Unlike the first example when we compiled +`mr_example.erl` and distributed it across the cluster, this time +we'll use the [Erlang client library][erlang client] and shell. + +```erlang +1> {ok, Client} = riakc_pb_socket:start("127.0.0.1", 8087). +2> Mine = riakc_obj:new(<<"groceries">>, <<"mine">>, + term_to_binary(["eggs", "bacon"])). +3> Yours = riakc_obj:new(<<"groceries">>, <<"yours">>, + term_to_binary(["bread", "bacon"])). +4> riakc_pb_socket:put(Client, Yours, [{w, 1}]). +5> riakc_pb_socket:put(Client, Mine, [{w, 1}]). +``` + +Now that we have a client and some data, let's run a query and count how +many occurrences of groceries. + +```erlang +6> Count = fun(G, undefined, none) -> + [dict:from_list([{I, 1} + || I <- binary_to_term(riak_object:get_value(G))])] + end. +7> Merge = fun(Gcounts, none) -> + [lists:foldl(fun(G, Acc) -> + dict:merge(fun(_, X, Y) -> X+Y end, + G, Acc) + end, + dict:new(), + Gcounts)] + end. +8> {ok, [{1, [R]}]} = riakc_pb_socket:mapred( + Client, + [{<<"groceries">>, <<"mine">>}, + {<<"groceries">>, <<"yours">>}], + [{map, {qfun, Count}, none, false}, + {reduce, {qfun, Merge}, none, true}]). +9> L = dict:to_list(R). +``` + +{{% note title="Riak Object Representations" %}} +Note how the `riak_object` module is used in the MapReduce +function but the `riakc_obj` module is used on the client. +Riak objects are represented differently internally to the cluster than +they are externally. +{{% /note %}} + +Given the lists of groceries we created, the sequence of commands above +would result in L being bound to `[{"bread",1},{"eggs",1},{"bacon",2}]`. + +### Erlang Query Syntax + +`riakc_pb_socket:mapred/3` takes a client and two lists as arguments. +The first list contains bucket-key pairs. The second list contains +the phases of the query. + +`riakc_pb_socket:mapred_bucket/3` replaces the first list of +bucket-key pairs with the name of a bucket; see the warnings above +about using this in a production environment. + +#### Inputs + +The `mapred/3` input objects are given as a list of tuples in the +format `{Bucket, Key}` or `{{Bucket, Key}, KeyData}`. `Bucket` and +`Key` should be binaries, and `KeyData` can be any Erlang term. The +former form is equivalent to `{{Bucket,Key},undefined}`. + +#### Query + +The query is given as a list of map, reduce and link phases. Map and +reduce phases are each expressed as tuples in the following form: + + +```erlang +{Type, FunTerm, Arg, Keep} +``` + +`Type` is an atom, either `map` or `reduce`. `Arg` is a static argument +(any Erlang term) to pass to each execution of the phase. `Keep` is +either `true` or `false` and determines whether results from the phase +will be included in the final value of the query. Riak KV assumes that the +final phase will return results. + +`FunTerm` is a reference to the function that the phase will execute and +takes any of the following forms: + +* `{modfun, Module, Function}` where `Module` and `Function` are atoms + that name an Erlang function in a specific module +* `{qfun,Fun}` where `Fun` is a callable fun term (closure or anonymous + function) +* `{jsfun,Name}` where `Name` is a binary that, when evaluated in + Javascript, points to a built-in Javascript function +* `{jsanon, Source}` where `Source` is a binary that, when evaluated in + Javascript is an anonymous function +* `{jsanon, {Bucket, Key}}` where the object at `{Bucket, Key}` contains + the source for an anonymous Javascript function + +{{% note title="qfun Note" %}} +Using `qfun` in compiled applications can be a fragile +operation. Please keep the following points in mind: + +1. The module in which the function is defined must be present and +exactly the same version on both the client and Riak KV nodes. + +2. Any modules and functions used by this function (or any function in +the resulting call stack) must also be present on the Riak KV nodes. + +Errors about failures to ensure both 1 and 2 are often surprising, +usually seen as opaque missing-function or function-clause +errors. Especially in the case of differing module versions, this can be +difficult to diagnose without expecting the issue and knowing of +`Module:info/0`. + +When using the Erlang shell, anonymous MapReduce functions can be +defined and sent to Riak KV instead of deploying them to all servers in +advance, but condition #2 above still holds. +{{% /note %}} + +Link phases are expressed in the following form: + + +```erlang +{link, Bucket, Tag, Keep} +``` + + +`Bucket` is either a binary name of a bucket to match, or the atom `_`, +which matches any bucket. `Tag` is either a binary tag to match, or the +atom `_`, which matches any tag. `Keep` has the same meaning as in map +and reduce phases. + + +> There are a small group of prebuilt Erlang MapReduce functions available +with Riak KV. Check them out [on GitHub](https://github.com/basho/riak_kv/blob/master/src/riak_kv_mapreduce.erl). + +## Bigger Data Examples + +### Loading Data + +This Erlang script will load historical stock-price data for Google +(ticker symbol "GOOG") into your existing Riak KV cluster so we can use it. +Paste the code below into a file called `load_data.erl` inside the `dev` +directory (or download it below). + +```erlang +#!/usr/bin/env escript +%% -*- erlang -*- +main([]) -> + io:format("Requires one argument: filename with the CSV data~n"); +main([Filename]) -> + {ok, Data} = file:read_file(Filename), + Lines = tl(re:split(Data, "\r?\n", [{return, binary},trim])), + lists:foreach(fun(L) -> LS = re:split(L, ","), format_and_insert(LS) end, Lines). + +format_and_insert(Line) -> + JSON = io_lib:format("{\"Date\":\"~s\",\"Open\":~s,\"High\":~s,\"Low\":~s,\"Close\":~s,\"Volume\":~s,\"Adj. Close\":~s}", Line), + Command = io_lib:format("curl -XPUT http://127.0.0.1:8098/buckets/goog/keys/~s -d '~s' -H 'content-type: application/json'", [hd(Line),JSON]), + io:format("Inserting: ~s~n", [hd(Line)]), + os:cmd(Command). +``` + +Make the script executable: + +```bash +chmod +x load_data.erl +``` + +Download the CSV file of stock data linked below and place it in the +`dev` directory where we've been working. + +* [goog.csv](https://github.com/basho/basho_docs/raw/master/extras/data/goog.csv) --- Google historical stock data +* [load_stocks.rb](https://github.com/basho/basho_docs/raw/master/extras/code-examples/load_stocks.rb) --- Alternative script in Ruby to load the data +* [load_data.erl](https://github.com/basho/basho_docs/raw/master/extras/code-examples/load_data.erl) --- Erlang script to load data (as shown in snippet) + +Now load the data into Riak KV. + +```bash +./load_data.erl goog.csv +``` + + +### Map only: find the days on which the high was over $600.00 + +From the Erlang shell with the client library loaded, let's define a +function which will check each value in our `goog` bucket to see if +the stock's high for the day was above $600. + +```erlang +> HighFun = fun(O, _, LowVal) -> +> {struct, Map} = mochijson2:decode(riak_object:get_value(O)), +> High = proplists:get_value(<<"High">>, Map, -1.0), +> case High > LowVal of +> true -> [riak_object:key(O)]; +> false -> [] +> end end. +#Fun +``` + +Now we'll use `mapred_bucket/3` to send that function to the cluster. + +```erlang +> riakc_pb_socket:mapred_bucket(Riak, <<"goog">>, [{map, {qfun, HighFun}, 600, true}]). + {ok,[{0, + [<<"2007-11-29">>,<<"2008-01-02">>,<<"2008-01-17">>, + <<"2010-01-08">>,<<"2007-12-05">>,<<"2007-10-24">>, + <<"2007-10-26">>,<<"2007-10-11">>,<<"2007-11-09">>, + <<"2007-12-06">>,<<"2007-12-19">>,<<"2007-11-01">>, + <<"2007-11-07">>,<<"2007-11-16">>,<<"2009-12-28">>, + <<"2007-12-26">>,<<"2007-11-05">>,<<"2008-01-16">>, + <<"2007-11-13">>,<<"2007-11-08">>,<<"2007-12-07">>, + <<"2008-01-"...>>,<<"2007"...>>,<<...>>|...]}]} +``` + +#### Map only: find the days on which the close is lower than open + +This example is slightly more complicated: instead of comparing a +single field against a fixed value, we're looking for days when the +stock declined. + +```erlang +> CloseLowerFun = fun(O, _, _) -> +> {struct, Map} = mochijson2:decode(riak_object:get_value(O)), +> Close = proplists:get_value(<<"Close">>, Map, -1.0), +> Open = proplists:get_value(<<"Open">>, Map, -2.0), +> case Close < Open of +> true -> [riak_object:key(O)]; +> false -> [] +> end end. +#Fun + +> riakc_pb_socket:mapred_bucket(Riak, <<"goog">>, [{map, {qfun, CloseLowerFun}, none, true}]). +{ok,[{0, + [<<"2008-05-13">>,<<"2008-12-19">>,<<"2009-06-10">>, + <<"2006-07-06">>,<<"2006-07-07">>,<<"2009-02-25">>, + <<"2009-07-17">>,<<"2005-10-05">>,<<"2006-08-18">>, + <<"2008-10-30">>,<<"2009-06-18">>,<<"2006-10-26">>, + <<"2008-01-17">>,<<"2010-04-16">>,<<"2007-06-29">>, + <<"2005-12-12">>,<<"2008-08-20">>,<<"2007-03-30">>, + <<"2006-07-20">>,<<"2006-10-24">>,<<"2006-05-26">>, + <<"2007-02-"...>>,<<"2008"...>>,<<...>>|...]}]} +``` + +#### Map and Reduce: find the maximum daily variance in price by month + +Here things start to get tricky. We'll use map to determine each day's +rise or fall, and our reduce phase will identify each month's largest +variance. + +```erlang +DailyMap = fun(O, _, _) -> + {struct, Map} = mochijson2:decode(riak_object:get_value(O)), + Date = binary_to_list(proplists:get_value(<<"Date">>, Map, "0000-00-00")), + High = proplists:get_value(<<"High">>, Map, 0.0), + Low = proplists:get_value(<<"Low">>, Map, 0.0), + Month = string:substr(Date, 1, 7), + [{Month, abs(High - Low)}] +end. + +MonthReduce = fun(List, _) -> + {Highs, _} = lists:foldl( + fun({Month, _Value}=Item, {Accum, PrevMonth}) -> + case Month of + PrevMonth -> + %% Highest value is always first in the list, so + %% skip over this one + {Accum, PrevMonth}; + _ -> + {[Item] ++ Accum, Month} + end + end, + {[], ""}, + List), + Highs + end. +> riakc_pb_socket:mapred_bucket(Riak, <<"goog">>, [{map, {qfun, DailyMap}, none, false}, {reduce, {qfun, MonthReduce}, none, true}]). +{ok,[{1, + [{"2010-02",10.099999999999909}, + {"2006-02",11.420000000000016}, + {"2004-08",8.100000000000009}, + {"2008-08",14.490000000000009}, + {"2006-05",11.829999999999984}, + {"2005-10",4.539999999999964}, + {"2006-06",7.300000000000011}, + {"2008-06",9.690000000000055}, + {"2006-03",11.770000000000039}, + {"2006-12",4.880000000000052}, + {"2005-09",9.050000000000011}, + {"2008-03",15.829999999999984}, + {"2008-09",14.889999999999986}, + {"2010-04",9.149999999999977}, + {"2008-06",14.909999999999968}, + {"2008-05",13.960000000000036}, + {"2005-05",2.780000000000001}, + {"2005-07",6.680000000000007}, + {"2008-10",21.390000000000043}, + {"2009-09",4.180000000000007}, + {"2006-08",8.319999999999993}, + {"2007-08",5.990000000000009}, + {[...],...}, + {...}|...]}]} +``` + +#### A MapReduce Challenge + +Here is a scenario involving the data you already have loaded. + +MapReduce Challenge: Find the largest day for each month in terms of +dollars traded, and subsequently the largest overall day. + +*Hint*: You will need at least one each of map and reduce phases. + +## Streaming MapReduce + +Because Riak KV distributes the map phases across the cluster to increase +data locality, you can gain access to the results of those individual +computations as they finish via streaming. Streaming can be very +helpful when getting access to results from a high latency MapReduce job +that only contains map phases. Streaming of results from reduce phases +isn't as useful, but if your map phases return data (keep: true), they +will be returned to the client even if the reduce phases haven't +executed. This will let you use streaming with a reduce phase to collect +the results of the map phases while the jobs are run and then get the +result to the reduce phase at the end. + +### Streaming via the HTTP API + +You can enable streaming with MapReduce jobs submitted to the `/mapred` +resource by adding `?chunked=true` to the url. The response will be sent +using HTTP 1.1 chunked transfer encoding with `Content-Type: multipart/mixed`. +Be aware that if you are streaming a set of serialized objects (like +JSON objects), the chunks are not guaranteed to be separated along the +same boundaries that your serialized objects are. For example, a chunk +may end in the middle of a string representing a JSON object, so you +will need to decode and parse your responses appropriately in the +client. + +### Streaming via the Erlang API + +You can use streaming with Erlang via the Riak KV local client or the +Erlang Protocol Buffers API. In either case, you will provide the call +to `mapred_stream` with a `Pid` that will receive the streaming results. + +For examples, see [MapReduce pbstream.erl]({{}}data/MapReduceExamples/pbstream.erl) + + +## Troubleshooting MapReduce, illustrated + +The most important advice: when developing Erlang MapReduce against +Riak KV, prototype against a development environment using the Erlang +shell. The shell allows for rapid feedback and iteration; once code +needs to be deployed to a server for production use, changing it is +more time-consuming. + +### Module not in path + +```bash +$ curl -XPOST localhost:8098/mapred \ +> -H 'Content-Type: application/json' \ +> -d '{"inputs":"messages","query":[{"map":{"language":"erlang","module":"mr_example","function":"get_keys"}}]}' + +{"phase":0,"error":"invalid module named in PhaseSpec function:\n must be a valid module name (failed to load mr_example: nofile)"} +``` + +### Node in process of starting + +```bash +$ curl -XPOST localhost:8098/mapred -H 'Content-Type: application/json' -d '{"inputs":"messages","query":[{"map":{"language":"erlang","module":"mr_example","function":"get_keys"}}]}' + +500 Internal Server Error

Internal Server Error

The server encountered an error while processing this request:
{error,{error,function_clause,
+              [{chashbin,itr_value,
+                         [done],
+                         [{file,"src/chashbin.erl"},{line,139}]},
+               {chashbin,itr_next_while,2,
+                         [{file,"src/chashbin.erl"},{line,183}]},
+...
+```
+
+### Erlang errors
+
+```erlang
+> riakc_pb_socket:mapred_bucket(Riak, <<"goog">>, [{map, {qfun, DailyFun}, none, true}]).
+{error,<<"{\"phase\":0,\"error\":\"function_clause\",\"input\":\"{ok,{r_object,<<\\\"goog\\\">>,<<\\\"2009-06-10\\\">>,[{r_content,{dic"...>>}
+```
+
+The Erlang shell truncates error messages; when using MapReduce, typically the information you need is buried more deeply within the stack.
+
+We can get a longer error message this way:
+
+```erlang
+> {error, ErrorMsg} = riakc_pb_socket:mapred_bucket(Riak, <<"goog">>, [{map, {qfun, DailyFun}, none, true}]).
+{error,<<"{\"phase\":0,\"error\":\"function_clause\",\"input\":\"{ok,{r_object,<<\\\"goog\\\">>,<<\\\"2009-06-10\\\">>,[{r_content,{dic"...>>}
+
+> io:format("~p~n", [ErrorMsg]).
+<<"{\"phase\":0,\"error\":\"function_clause\",\"input\":\"{ok,{r_object,<<\\\"goog\\\">>,<<\\\"2009-06-10\\\">>,[{r_content,{dict,6,16,16,8,80,48,{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},{{[],[],[[<<\\\"Links\\\">>]],[],[],[],[],[],[],[],[[<<\\\"content-type\\\">>,97,112,112,108,105,99,97,116,105,111,110,47,106,115,111,110],[<<\\\"X-Riak-VTag\\\">>,55,87,101,79,53,120,65,121,50,67,49,77,72,104,54,100,89,65,67,74,55,70]],[[<<\\\"index\\\">>]],[],[[<<\\\"X-Riak-Last-Modified\\\">>|{1405,709865,48668}]],[],[[<<\\\"X-Riak-Meta\\\">>]]}}},<<\\\"{\\\\\\\"Date\\\\\\\":\\\\\\\"2009-06-10\\\\\\\",\\\\\\\"Open\\\\\\\":436.23,\\\\\\\"High\\\\\\\":437.89,\\\\\\\"L...\\\">>}],...},...}\",\"type\":\"error\",\"stack\":\"[{string,substr,[\\\"2009-06-10\\\",0,7],[{file,\\\"string.erl\\\"},{line,207}]},{erl_eval,do_apply,6,[{file,\\\"erl_eval.erl\\\"},{line,573}]},{erl_eval,expr,5,[{file,\\\"erl_eval.erl\\\"},{line,364}]},{erl_eval,exprs,5,[{file,\\\"erl_eval.erl\\\"},{line,118}]},{riak_kv_mrc_map,map,3,[{file,\\\"src/riak_kv_mrc_map.erl\\\"},{line,172}]},{riak_kv_mrc_map,process,3,[{file,\\\"src/riak_kv_mrc_map.erl\\\"},{line,144}]},{riak_pipe_vnode_worker,process_input,3,[{file,\\\"src/riak_pipe_vnode_worker.erl\\\"},{line,446}]},{riak_pipe_vnode_worker,wait_for_input,...}]\"}">>
+```
+
+Still truncated, but this provides enough context to see the problem:
+`string,substr,[\\\"2009-06-10\\\",0,7]`. Erlang's `string:substr`
+function starts indexing strings at 1, not 0.
+
+### Exceptional tip
+
+When experimenting with MapReduce from the Erlang shell, it is helpful
+to avoid breaking the connection to Riak KV when an exception is trapped
+by the shell. Use `catch_exception`:
+
+```erlang
+> catch_exception(true).
+false
+```
diff --git a/content/riak/kv/2.2.6/developing/app-guide/cluster-metadata.md b/content/riak/kv/2.2.6/developing/app-guide/cluster-metadata.md
new file mode 100644
index 0000000000..ca5e528a1c
--- /dev/null
+++ b/content/riak/kv/2.2.6/developing/app-guide/cluster-metadata.md
@@ -0,0 +1,67 @@
+---
+title: "Cluster Metadata"
+description: ""
+project: "riak_kv"
+project_version: "2.2.6"
+menu:
+  riak_kv-2.2.6:
+    name: "Cluster Metadata"
+    identifier: "app_guide_cluster_metadata"
+    weight: 104
+    parent: "developing_app_guide"
+toc: true
+---
+
+Cluster metadata is a subsystem inside of Riak that enables systems
+built on top of
+[`riak_core`](https://github.com/basho/riak_core/blob/develop/src/riak_core_metadata.erl)
+to work with information that is stored cluster wide and can be read
+without blocking on communication over the network.
+
+One notable example of a subsystem of Riak relying on cluster metadata
+is Riak's [bucket types]({{}}riak/kv/2.2.6/using/reference/bucket-types) feature. This feature
+requires that a particular form of key/value pairs, namely bucket type
+names (the key) and their associated bucket properties (the value), be
+asynchronously broadcast to all nodes in a Riak cluster.
+
+Though it is different in crucial respects,
+[etcd](https://coreos.com/docs/cluster-management/setup/getting-started-with-etcd/)
+is a roughly analogous cluster metadata key/value store developed for
+use in [CoreOS](https://coreos.com/) clusters.
+
+## How Cluster Metadata Works
+
+Cluster metadata is different from other Riak data in two essential
+respects:
+
+1. Cluster metadata is intended only for internal Riak applications that
+   require metadata shared on a system-wide basis. Regular stored data,
+   on the other hand, is intended for use outside of Riak.
+2. Because it is intended for use only by applications internal to Riak,
+   cluster metadata can be accessed only internally, via the Erlang
+   interface provided by the
+   [`riak_core_metadata`](https://github.com/basho/riak_core/blob/develop/src/riak_core_metadata.erl)
+   module; it cannot be accessed externally via HTTP or Protocol Buffers.
+
+The storage system backing cluster metadata is a simple key/value store
+that is capable of asynchronously replicating information to all nodes
+in a cluster when it is stored or modified. Writes require
+acknowledgment from only a single node (equivalent to `w=1` in normal
+Riak), while reads return values only from the local node (equivalent to
+`r=1`). All updates are eventually consistent and propagated to all
+nodes, including nodes that join the cluster after the update has
+already reached all nodes in the previous set of members.
+
+All cluster metadata is eventually stored both in memory and on disk,
+but it should be noted that reads are only from memory, while writes are
+made both to memory and to disk. Logical clocks, namely [dotted version vectors]({{}}riak/kv/2.2.6/learn/concepts/causal-context/#dotted-version-vectors), are used in place of [vector clocks]({{}}riak/kv/2.2.6/learn/concepts/causal-context/#vector-clocks) or timestamps to resolve value conflicts. Values stored as cluster metadata are opaque Erlang
+terms addressed by both prefix and a key.
+
+## Erlang Code Interface
+
+If you'd like to use cluster metadata for an internal Riak application,
+the Erlang interface is defined in the
+[`riak_core_metadata`](https://github.com/basho/riak_core/blob/develop/src/riak_core_metadata.erl)
+module, which allows you to perform a variety of cluster metadata
+operations, including retrieving, modifying, and deleting metadata and
+iterating through metadata keys.
diff --git a/content/riak/kv/2.2.6/developing/app-guide/reference.md b/content/riak/kv/2.2.6/developing/app-guide/reference.md
new file mode 100644
index 0000000000..3c040fd9ac
--- /dev/null
+++ b/content/riak/kv/2.2.6/developing/app-guide/reference.md
@@ -0,0 +1,16 @@
+---
+draft: true
+title: "Reference"
+description: ""
+project: "riak_kv"
+project_version: "2.2.6"
+#menu:
+#  riak_kv-2.2.6:
+#    name: "Reference"
+#    identifier: "app_guide_reference"
+#    weight: 104
+#    parent: "developing_app_guide"
+toc: true
+---
+
+**TODO: Add content**
diff --git a/content/riak/kv/2.2.6/developing/app-guide/replication-properties.md b/content/riak/kv/2.2.6/developing/app-guide/replication-properties.md
new file mode 100644
index 0000000000..4abbeeb4d3
--- /dev/null
+++ b/content/riak/kv/2.2.6/developing/app-guide/replication-properties.md
@@ -0,0 +1,580 @@
+---
+title: "Replication Properties"
+description: ""
+project: "riak_kv"
+project_version: "2.2.6"
+menu:
+  riak_kv-2.2.6:
+    name: "Replication Properties"
+    identifier: "app_guide_replication_properties"
+    weight: 100
+    parent: "developing_app_guide"
+toc: true
+aliases:
+  - /riak-docs/riak/2.2.6/dev/advanced/replication-properties
+  - /riak-docs/riak/kv/2.2.6/dev/advanced/replication-properties
+---
+
+[usage bucket types]: {{}}riak/kv/2.2.6/developing/usage/bucket-types
+[concept eventual consistency]: {{}}riak/kv/2.2.6/learn/concepts/eventual-consistency
+[use ref strong consistency]: {{}}riak/kv/2.2.6/using/reference/strong-consistency
+[concept clusters]: {{}}riak/kv/2.2.6/learn/concepts/clusters
+
+Riak was built to act as a multi-node [cluster][concept clusters].  It
+distributes data across multiple physical servers, which enables it to
+provide strong availability guarantees and fault tolerance.
+
+The [CAP theorem](http://en.wikipedia.org/wiki/CAP_theorem), which
+undergirds many of the design decisions behind Riak's architecture,
+defines distributed systems in terms of three desired properties:
+consistency, availability, and partition (i.e. failure) tolerance. Riak
+can be used either as an AP, i.e. available/partition-tolerant, system
+or as a CP, i.e. consistent/partition-tolerant, system. The former
+relies on an [eventual consistency][concept eventual consistency] model, while the latter relies on
+a special [strong consistency][use ref strong consistency] subsystem.
+
+Although the [CAP theorem](http://en.wikipedia.org/wiki/CAP_theorem)
+dictates that there is a necessary trade-off between data consistency
+and availability, if you are using Riak in an eventually consistent
+manner, you can fine-tune that trade-off. The ability to make these
+kinds of fundamental choices has immense value for your applications and
+is one of the features that differentiates Riak from other databases.
+
+At the bottom of the page, you'll find a [screencast]({{}}riak/kv/2.2.6/developing/app-guide/replication-properties#screencast) that briefly explains how to adjust your
+replication levels to match your application and business needs.
+
+> **Note on strong consistency**
+>
+> An option introduced in Riak version 2.0 is to use Riak as a [strongly consistent]({{}}riak/kv/2.2.6/using/reference/strong-consistency/) system for data in specified buckets. Using Riak in this way is fundamentally different from adjusting replication properties and fine-tuning the availability/consistency trade-off, as it sacrifices
+_all_ availability guarantees when necessary. Therefore, you
+should consult the [Using Strong Consistency]({{}}riak/kv/2.2.6/developing/app-guide/strong-consistency) documentation, as this option will not be covered
+in this tutorial.
+
+## How Replication Properties Work
+
+When using Riak, there are two ways of choosing replication properties:
+1. On a per-request basis
+2. In a more programmatic fashion, [using bucket types][usage bucket types]
+
+### Per-request Replication Properties
+
+The simplest way to apply replication properties to objects stored in
+Riak is to specify those properties
+
+### Replication Properties Through Bucket Types
+
+Let's say, for example, that you want to apply an `n_val` of 5, an `r`
+of 3, and a `w` of 3 to all of the data in some of the [buckets]({{}}riak/kv/2.2.6/learn/concepts/buckets) that
+you're using. In order to set those replication properties, you should
+create a bucket type that sets those properties. Below is an example:
+
+```bash
+riak-admin bucket-type create custom_props '{"props":{"n_val":5,"r":3,"w":3}}'
+riak-admin bucket-type activate custom_props
+```
+
+Now, any time you store an object in a bucket with the type
+`custom_props` those properties will apply to it.
+
+## Available Parameters
+
+The table below lists the most frequently used replication parameters
+that are available in Riak. Symbolic values like `quorum` are discussed
+[below]({{}}riak/kv/2.2.6/developing/app-guide/replication-properties#symbolic-consistency-names). Each
+parameter will be explained in more detail in later sections:
+
+Parameter | Common name | Default value | Description
+:---------|:------------|:--------------|:-----------
+`n_val` | N | `3` | Replication factor, i.e. the number of nodes in the cluster on which an object is to be stored
+`r` | R | `quorum` | The number of servers that must respond to a read request
+`w` | W | `quorum` | Number of servers that must respond to a write request
+`pr` | PR | `0` | The number of primary vnodes that must respond to a read request
+`pw` | PW | `0` | The number of primary vnodes that must respond to a write request
+`dw` | DW | `quorum` | The number of servers that must report that a write has been successfully written to disk
+`rw` | RW | `quorum` | If R and W are undefined, this parameter will substitute for both R and W during object deletes. It is extremely unlikely that you will need to adjust this parameter.
+`notfound_ok` | | `true` | This parameter determines how Riak responds if a read fails on a node. Setting to `true` (the default) is the equivalent to setting R to 1: if the first node to respond doesn't have a copy of the object, Riak will immediately return a `not found` error. If set to `false`, Riak will continue to look for the object on the number of nodes specified by N (aka `n_val`).
+`basic_quorum` | | `false` | If `notfound_ok` is set to `false`, Riak will be more thorough in looking for an object on multiple nodes. Setting `basic_quorum` to `true` in this case will instruct Riak to wait for only a `quorum` of responses to return a `notfound` error instead of N responses.
+
+## A Primer on N, R, and W
+
+The most important thing to note about Riak's replication controls is
+that they can be at the bucket level. You can use [bucket types]({{}}riak/kv/2.2.6/developing/usage/bucket-types)
+to set up bucket `A` to use a particular set of replication properties
+and bucket `B` to use entirely different properties.
+
+At the bucket level, you can choose how many copies of data you want to
+store in your cluster (N, or `n_val`), how many copies you wish to read
+from at one time (R, or `r`), and how many copies must be written to be
+considered a success (W, or `w`).
+
+In addition to the bucket level, you can also specify replication
+properties on the client side for any given read or write. The examples
+immediately below will deal with bucket-level replication settings, but
+check out the [section below]({{}}riak/kv/2.2.6/developing/app-guide/replication-properties#client-level-replication-settings)
+for more information on setting properties on a per-operation basis.
+
+The most general trade-off to be aware of when setting these values is
+the trade-off between **data accuracy** and **client responsiveness**.
+Choosing higher values for N, R, and W will mean higher accuracy because
+more nodes are checked for the correct value on read and data is written
+to more nodes upon write; but higher values will also entail degraded
+responsiveness, especially if one or more nodes is failing, because Riak
+has to wait for responses from more nodes.
+
+## N Value and Replication
+
+All data stored in Riak will be replicated to the number of nodes in the
+cluster specified by a bucket's N value (`n_val`). The default `n_val`
+in Riak is 3, which means that data stored in a bucket with the default
+N will be replicated to three different nodes, thus storing three
+**replicas** of the object.
+
+In order for this to be effective, you need at least three nodes in your
+cluster. The merits of this system, however, can be demonstrated using
+your local environment.
+
+Let's create a bucket type that sets the `n_val` for any bucket with
+that type to 2. To do so, you must create and activate a bucket type
+that sets this property:
+
+```bash
+riak-admin bucket-type create n_val_equals_2 '{"props":{"n_val":2}}'
+riak-admin bucket-type activate n_val_equals_2
+```
+
+Now, all buckets that bear the type `n_val_equals_2` will have `n_val`
+set to 2. Here's an example write:
+
+```curl
+curl -XPUT http://localhost:8098/types/n_val_equals_2/buckets/test_bucket/keys/test_key \
+  -H "Content-Type: text/plain" \
+  -d "the n_val on this write is 2"
+```
+
+Now, whenever we write to a bucket of this type, Riak will write a
+replica of the object to two different nodes.
+
+{{% note title="A Word on Setting the N Value" %}}
+`n_val` must be greater than 0 and less than or equal to the number of actual
+nodes in your cluster to get all the benefits of replication. We advise
+against modifying the `n_val` of a bucket after its initial creation as this
+may result in failed reads because the new value may not be replicated to all
+the appropriate partitions.
+{{% /note %}}
+
+## R Value and Read Failure Tolerance
+
+Read requests to Riak are sent to all N nodes that are known to be
+currently responsible for the data. The R value (`r`) enables you to
+specify how many of those nodes have to return a result on a given read
+for the read to be considered successful. This allows Riak to provide
+read availability even when nodes are down or laggy.
+
+You can set R anywhere from 1 to N; lower values mean faster response
+time but a higher likelihood of Riak not finding the object you're
+looking for, while higher values mean that Riak is more likely to find
+the object but takes longer to look.
+
+As an example, let's create and activate a bucket type with `r` set to
+`1`. All reads performed on data in buckets with this type require a
+result from only one node.
+
+```bash
+riak-admin bucket-type create r_equals_1 '{"props":{"r":1}}'
+riak-admin bucket-type activate r_equals_1
+```
+
+Here's an example read request using the `r_equals_1` bucket type:
+
+```ruby
+bucket = client.bucket_type('r_equals_1').bucket('animal_facts')
+obj = bucket.get('chimpanzee')
+```
+
+```java
+Location chimpanzeeFact =
+  new Location(new Namespace("r_equals_1", "animal_facts"), "chimpanzee");
+FetchValue fetch = new FetchValue.Builder(chimpanzeeFact).build();
+FetchValue.Response response = client.execute(fetch);
+RiakObject obj = response.getValue(RiakObject.class);
+System.out.println(obj.getValue().toString());
+```
+
+```php
+$response = (new \Basho\Riak\Command\Builder\FetchObject($riak))
+  ->buildLocation('chimpanzee', 'animal_facts', 'r_equals_1')
+  ->build()
+  ->execute();
+
+echo $response->getObject()->getData();
+```
+
+```python
+bucket = client.bucket_type('r_equals_1').bucket('animal_facts')
+bucket.get('chimpanzee')
+```
+
+```erlang
+{ok, Obj} = riakc_pb_socket:get(Pid,
+                                {<<"r_equals_1">>, <<"animal_facts">>},
+                                <<"chimpanzee">>).
+```
+
+```curl
+curl http://localhost:8098/types/r_equals_1/buckets/animal_facts/keys/chimpanzee
+```
+
+As explained above, reads to buckets with the `r_equals_1` type will
+typically be completed more quickly, but if the first node to respond
+to a read request has yet to receive a replica of the object, Riak will
+return a `not found` response (which may happen even if the object lives
+on one or more other nodes). Setting `r` to a higher value will mitigate
+this risk.
+
+## W Value and Write Fault Tolerance
+
+As with read requests, writes to Riak are sent to all N nodes that are
+know to be currently responsible for the data. The W value (`w`) enables
+you to specify how many nodes must complete a write to be considered
+successful---a direct analogy to R. This allows Riak to provide write
+availability even when nodes are down or laggy.
+
+As with R, you can set W to any value between 1 and N. The same
+performance vs. fault tolerance trade-offs that apply to R apply to W.
+
+As an example, let's create and activate a bucket type with `w` set to
+`3`:
+
+```bash
+riak-admin bucket-type create w_equals_3 '{"props":{"w":3}}'
+riak-admin activate w_equals_3
+```
+
+Now, we can attempt a write to a bucket bearing the type `w_equals_3`:
+
+```ruby
+bucket = client.bucket_type('w_equals_3').bucket('animal_facts')
+obj = Riak::RObject.new(bucket, 'giraffe')
+obj.raw_data = 'The species name of the giraffe is Giraffa camelopardalis'
+obj.content_type = 'text/plain'
+obj.store
+```
+
+```java
+Location storyKey =
+  new Location(new Namespace("w_equals_3", "animal_facts"), "giraffe");
+RiakObject obj = new RiakObject()
+        .setContentType("text/plain")
+        .setValue(BinaryValue.create("The species name of the giraffe is Giraffa camelopardalis"));
+StoreValue store = new StoreValue.Builder(obj)
+        .withLocation("giraffe")
+        .build();
+client.execute(store);
+```
+
+```php
+(new \Basho\Riak\Command\Builder\StoreObject($riak))
+  ->buildLocation('giraffe', 'animal_facts', 'w_equals_3')
+  ->build()
+  ->execute();
+```
+
+```python
+bucket = client.bucket_type('w_equals_3').bucket('animal_facts')
+obj = RiakObject(client, bucket, 'giraffe')
+obj.content_type = 'text/plain'
+obj.data = 'The species name of the giraffe is Giraffa camelopardalis'
+obj.store()
+```
+
+```erlang
+Obj = riakc_object:new({<<"w_equals_3">>, <<"animal_facts">>},
+                       <<"giraffe">>,
+                       <<"The species name of the giraffe is Giraffa camelopardalis">>,
+                       <<"text/plain">>),
+riakc_pb_socket:put(Pid, Obj).
+```
+
+```curl
+curl -XPUT \
+  -H "Content-type: text/plain" \
+  -d "The species name of the giraffe is Giraffa camelopardalis" \
+  http://localhost:8098/types/w_equals_3/buckets/animal_facts/keys/giraffe
+```
+
+Writing our `story.txt` will return a success response from Riak only if
+3 nodes respond that the write was successful. Setting `w` to 1, for
+example, would mean that Riak would return a response more quickly, but
+with a higher risk that the write will fail because the first node it
+seeks to write the object to is unavailable.
+
+## Primary Reads and Writes with PR and PW
+
+In Riak's replication model, there are N [vnodes]({{}}riak/kv/2.2.6/learn/glossary/#vnode),
+called _primary vnodes_, that hold primary responsibility for any given
+key. Riak will attempt reads and writes to primary vnodes first, but in
+case of failure, those operations will go to failover nodes in order to
+comply with the R and W values that you have set. This failover option
+is called _sloppy quorum_.
+
+In addition to R and W, you can also set integer values for the *primary
+read* (PR) and _primary write_ (PW) parameters that specify how many
+primary nodes must respond to a request in order to report success to
+the client. The default for both values is zero.
+
+Setting PR and/or PW to non-zero values produces a mode of operation
+called _strict quorum_. This mode has the advantage that the client is
+more likely to receive the most up-to-date values, but at the cost of a
+higher probability that reads or writes will fail because primary vnodes
+are unavailable.
+
+{{% note title="Note on PW" %}}
+If PW is set to a non-zero value, there is a higher risk (usually very small)
+that failure will be reported to the client upon write. But this does not
+necessarily mean that the write has failed completely. If there are reachable
+primary vnodes, those vnodes will still write the new data to Riak. When the
+failed vnode returns to service, it will receive the new copy of the data via
+either read repair or active anti-entropy.
+{{% /note %}}
+
+## Durable Writes with DW
+
+The W and PW parameters specify how many vnodes must _respond_ to a
+write in order for it to be deemed successful. What they do not specify
+is whether data has actually been written to disk in the storage backend.
+The DW parameters enables you to specify a number of vnodes between 1
+and N that must write the data to disk before the request is deemed
+successful. The default value is `quorum` (more on symbolic names below).
+
+How quickly and robustly data is written to disk depends on the
+configuration of your backend or backends. For more details, see the
+documentation on [Bitcask]({{}}riak/kv/2.2.6/setup/planning/backend/bitcask), [LevelDB]({{}}riak/kv/2.2.6/setup/planning/backend/leveldb), and [multiple backends]({{}}riak/kv/2.2.6/setup/planning/backend/multi).
+
+## Delete Quorum with RW
+
+{{% note title="Deprecation notice" %}}
+It is no longer necessary to specify an RW value when making delete requests.
+We explain its meaning here, however, because RW still shows up as a property
+of Riak buckets (as `rw`) for the sake of backwards compatibility. Feel free
+to skip this explanation unless you are curious about the meaning of RW.
+{{% /note %}}
+
+Deleting an object requires successfully reading an object and then
+writing a tombstone to the object's key that specifies that an object
+once resided there. In the course of their operation, all deletes must
+comply with any R, W, PR, and PW values that apply along the way.
+
+If R and W are undefined, however, the RW (`rw`) value will substitute
+for both R and W during object deletes. In recent versions of Riak, it
+is nearly impossible to make reads or writes that do not somehow specify 
+oth R and W, and so you will never need to worry about RW.
+
+## The Implications of `notfound_ok`
+
+The `notfound_ok` parameter is a bucket property that determines how
+Riak responds if a read fails on a node. If `notfound_ok` is set to
+`true` (the default value) and the first vnode to respond doesn't have a
+copy of the object, Riak will assume that the missing value is
+authoritative and immediately return a `not found` result to the client.
+This will generally lead to faster response times.
+
+On the other hand, setting `notfound_ok` to `false` means that the
+responding vnode will wait for something other than a `not found` error
+before reporting a value to the client. If an object doesn't exist under
+a key, the coordinating vnode will wait for N vnodes to respond with
+`not found` before it reports `not found` to the client. This setting
+makes Riak search more thoroughly for objects but at the cost of slower
+response times, a problem can be mitigated by setting `basic_quorum` to
+`true`, which is discussed in the next section.
+
+## Early Failure Return with `basic_quorum`
+
+Setting `notfound_ok` to `false` on a request (or as a bucket property)
+is likely to introduce additional latency. If you read a non-existent
+key, Riak will check all 3 responsible vnodes for the value before
+returning `not found` instead of checking just one.
+
+This latency problem can be mitigated by setting `basic_quorum` to
+`true`, which will instruct Riak to query a quorum of nodes instead of N
+nodes. A quorum of nodes is calculated as floor(N/2) + 1, meaning that 5
+nodes will produce a quorum of 3, 6 nodes a quorum of 4, 7 nodes a
+quorum of 4, 8 nodes a quorum of 5, etc.
+
+The default for `basic_quorum` is `false`, so you will need to
+explicitly set it to `true` on reads or in a bucket's properties. While
+the scope of this setting is fairly narrow, it can reduce latency in
+read-heavy use cases.
+
+## Symbolic Consistency Names
+
+Riak provides a number of "symbolic" consistency options for R, W, PR,
+RW, and DW that are often easier to use and understand than specifying
+integer values. The following symbolic names are available:
+
+* `all` --- All replicas must reply. This is the same as setting R, W, PR, RW, or DW equal to N.
+* `one` --- This is the same as setting 1 as the value for R, W, PR, RW, or DW.
+* `quorum` --- A majority of the replicas must respond, that is, half plus one. For the default N value of 3, this calculates to 2, an N value of 5 calculates to 3, and so on.
+* `default` --- Uses whatever the per-bucket consistency property is for R, W, PR, RW, or DW, which may be any of the above symbolic values or an integer.
+
+Not submitting a value for R, W, PR, RW, or DW is the same as using
+`default`.
+
+## Client-level Replication Settings
+
+Adjusting replication properties at the bucket level by [using bucket types][usage bucket types]
+is how you set default properties for _all_ of a bucket's reads and
+writes. But you can also set replication properties for specific reads
+and writes without setting those properties at the bucket level, instead
+specifying them on a per-operation basis.
+
+Let's say that you want to set `r` to 2 and `notfound_ok` to `true` for
+just one read. We'll fetch [John Stockton](http://en.wikipedia.org/wiki/John_Stockton)'s
+statistics from the `nba_stats` bucket.
+
+```ruby
+bucket = client.bucket('nba_stats')
+obj = bucket.get('john_stockton', r: 2, notfound_ok: true)
+```
+
+```java
+Location johnStocktonStats =
+  new Namespace(new Namespace("nba_stats"), "john_stockton");
+FetchValue fetch = new FetchValue.Builder(johnStocktonStats)
+        .withOption(FetchOption.R, new Quorum(2))
+        .withOption(FetchOption.NOTFOUND_OK, true)
+        .build();
+client.execute(fetch);
+```
+
+```php
+(new \Basho\Riak\Command\Builder\FetchObject($riak))
+  ->buildLocation('john_stockton', 'nba_stats')
+  ->withParameter('r', 2)
+  ->withParameter('notfound_ok', true)
+  ->build()
+  ->execute();
+```
+
+```python
+bucket = client.bucket('nba_stats')
+obj = bucket.get('john_stockton', r=2, notfound_ok=True)
+```
+
+```erlang
+{ok, Obj} = riakc_pb_socket:get(Pid,
+                                <<"nba_stats">>,
+                                <<"john_stockton">>,
+                                [{r, 2}, {notfound_ok, true}]).
+```
+
+```curl
+curl http://localhost:8098/buckets/nba_stats/keys/john_stockton?r=2¬found_ok=true
+```
+
+Now, let's say that you want to attempt a write with `w` set to 3 and
+`dw` set to 2. As in the previous example, we'll be using the `default`
+bucket type, which enables us to not specify a bucket type upon write.
+Here's what that would look like:
+
+```ruby
+bucket = client.bucket('nba_stats')
+obj = Riak::RObject.new(bucket, 'michael_jordan')
+obj.content_type = 'application/json'
+obj.data = '{"stats":{ ... large stats object ... }}'
+obj.store(w: 3, dw: 2)
+```
+
+```java
+Location michaelJordanKey =
+  new Location(new Namespace("nba_stats"), "michael_jordan");
+RiakObject obj = new RiakObject()
+        .setContentType("application/json")
+        .setValue(BinaryValue.create("{'stats':{ ... large stats object ... }}"));
+StoreValue store = new StoreValue.Builder(obj)
+        .withLocation(michaelJordanKey)
+        .withOption(StoreOption.W, new Quorum(3))
+        .withOption(StoreOption.DW, new Quorum(2))
+        .build();
+client.execute(store);
+```
+
+```php
+(new \Basho\Riak\Command\Builder\StoreObject($riak))
+  ->buildJsonObject('{'stats':{ ... large stats object ... }}')
+  ->buildLocation('john_stockton', 'nba_stats')
+  ->withParameter('w', 3)
+  ->withParameter('dw', 2)
+  ->build()
+  ->execute();
+```
+
+```erlang
+Obj = riakc_obj:new(<<"nba_stats">>,
+                    <<"michael_jordan">>,
+                    <<"{'stats':{ ... large stats object ... }}">>,
+                    <<"application/json">>),
+riakc_pb_socket:put(Pid, Obj).
+```
+
+```curl
+curl -XPUT \
+  -H "Content-Type: application/json" \
+  -d '{"stats":{ ... large stats object ... }}' \
+  http://localhost:8098/buckets/nba_stats/keys/michael_jordan?w=3&dw=2
+```
+
+All of Basho's [official Riak clients]({{}}riak/kv/2.2.6/developing/client-libraries) enable you to
+set replication properties this way. For more detailed information,
+refer to the section on [development usage with Riak KV]({{}}riak/kv/2.2.6/developing/usage)
+or to client-specific documentation:
+
+* [Ruby](https://github.com/basho/riak-ruby-client/blob/master/README.md)
+* [Java](http://basho.github.io/riak-java-client/2.0.0/)
+* [Python](http://basho.github.io/riak-python-client/)
+* [Erlang](http://basho.github.io/riak-erlang-client/)
+
+## Illustrative Scenarios
+
+In case the above explanations were a bit too abstract for your tastes,
+the following table lays out a number of possible scenarios for reads
+and writes in Riak and how Riak is likely to respond. Some of these
+scenarios involve issues surrounding conflict resolution, vector clocks,
+and siblings, so we recommend reading the [Vector Clocks]({{}}riak/kv/2.2.6/learn/concepts/causal-context#vector-clocks) documentation for more information.
+
+#### Read Scenarios
+
+These scenarios assume that a read request is sent to all 3 primary
+vnodes responsible for an object.
+
+Scenario | What happens in Riak
+:--------|:--------------------
+All 3 vnodes agree on the value | Once the first 2 vnodes return the value, that value is returned to the client
+2 of 3 vnodes agree on the value, and those 2 are the first to reach the coordinating node | The value is returned to the client. Read repair will deal with the conflict per the later scenarios, which means that a future read may return a different value or siblings
+2 conflicting values reach the coordinating node and vector clocks allow for resolution | The vector clocks are used to resolve the conflict and return a single value, which is propagated via read repair to the relevant vnodes
+2 conflicting values reach the coordinating node, vector clocks indicate a fork in the object history, and `allow_mult` is set to `false` | The object with the most recent timestamp is returned and propagated via read repair to the relevant vnodes
+2 siblings or conflicting values reach the coordinating node, vector clocks indicate a fork in the object history, and `allow_mult` is set to `true` | All keys are returned as siblings, optionally with associated values (depending on how the request is made)
+
+#### Write Scenarios
+
+These scenarios assume that a write request is sent to all 3 primary
+vnodes responsible for an object.
+
+Scenario | What happens in Riak
+:--------|:--------------------
+A vector clock is included with the write request, and is newer than the vclock attached to the existing object | The new value is written and success is indicated as soon as 2 vnodes acknowledge the write
+A vector clock is included with the write request but conflicts with the vclock attached to the existing object, with `allow_mult` set to `true` | The new value is created as a sibling for future reads
+A vector clock is included with the write request but conflicts with (or is older than) the vclock attached to the existing object, with `allow_mult` set to `false` | Riak will decide which object "wins" on the basis of timestamps; no sibling will be created
+A vector clock is not included with the write request and an object already exists, with `allow_mult` set to `true` | The new value is created as a sibling for future reads
+A vector clock is not included with the write request and an object already exists, with `allow_mult` set to `false` | The new value overwrites the existing value
+
+## Screencast
+
+Here is a brief screencast that shows just how the N, R, and W values
+function in our running 3-node Riak cluster:
+
+
+
+Tuning CAP Controls in Riak from
+Basho Technologies on Vimeo.
diff --git a/content/riak/kv/2.2.6/developing/app-guide/strong-consistency.md b/content/riak/kv/2.2.6/developing/app-guide/strong-consistency.md
new file mode 100644
index 0000000000..d193c4c3d5
--- /dev/null
+++ b/content/riak/kv/2.2.6/developing/app-guide/strong-consistency.md
@@ -0,0 +1,257 @@
+---
+title: "Strong Consistency"
+description: ""
+project: "riak_kv"
+project_version: "2.2.6"
+menu:
+  riak_kv-2.2.6:
+    name: "Strong Consistency"
+    identifier: "app_guide_strong_consistency"
+    weight: 101
+    parent: "developing_app_guide"
+toc: true
+aliases:
+  - /riak-docs/riak/2.2.6/dev/advanced/strong-consistency
+  - /riak-docs/riak/kv/2.2.6/dev/advanced/strong-consistency
+---
+
+[use ref strong consistency]: {{}}riak/kv/2.2.6/using/reference/strong-consistency
+[concept eventual consistency]: {{}}riak/kv/2.2.6/learn/concepts/eventual-consistency
+[use ref strong consistency#trade-offs]: {{}}riak/kv/2.2.6/using/reference/strong-consistency/#trade-offs
+[glossary vnode]: {{}}riak/kv/2.2.6/learn/glossary/#vnode
+[config strong consistency#enable]: {{}}riak/kv/2.2.6/configuring/strong-consistency/#enabling-strong-consistency
+[usage bucket types]: {{}}riak/kv/2.2.6/developing/usage/bucket-types
+[cluster ops bucket types]: {{}}riak/kv/2.2.6/using/cluster-operations/bucket-types
+[apps replication properties]: {{}}riak/kv/2.2.6/developing/app-guide/replication-properties
+[config strong consistency]: {{}}riak/kv/2.2.6/configuring/strong-consistency
+[config strong consistency#fault]: {{}}riak/kv/2.2.6/configuring/strong-consistency/#fault-tolerance
+[concept causal context]: {{}}riak/kv/2.2.6/learn/concepts/causal-context
+[concept causal context#vector]: {{}}riak/kv/2.2.6/learn/concepts/causal-context/#vector-clocks
+[concept version vector]: {{}}riak/kv/2.2.6/learn/concepts/causal-context/#dotted-version-vectors
+[usage conflict resolution]: {{}}riak/kv/2.2.6/developing/usage/conflict-resolution
+[usage update objects]: {{}}riak/kv/2.2.6/developing/usage/updating-objects
+[use ref strong consistency#vs]: {{}}riak/kv/2.2.6/using/reference/strong-consistency/#strong-vs.-eventual-consistency
+[dev client libraries]: {{}}riak/kv/2.2.6/developing/client-libraries
+[getting started]: {{}}riak/kv/2.2.6/developing/getting-started
+[config strong consistency#details]: {{}}riak/kv/2.2.6/configuring/strong-consistency/#implementation-details
+
+> **Please Note:**
+>
+> Riak KV's strong consistency is an experimental feature and may be removed from the product in the future. Strong consistency is not commercially supported or production-ready. Strong consistency is incompatible with Multi-Datacenter Replication, Riak Search, Bitcask Expiration, LevelDB Secondary Indexes, Riak Data Types and Commit Hooks. We do not recommend its usage in any production environment.
+
+In versions 2.0 and later, Riak allows you to create buckets that
+provide [strong consistency][use ref strong consistency] guarantees for the data stored within
+them, enabling you to use Riak as a CP system (consistent plus partition
+tolerant) for all of the data in that bucket. You can store just some of
+your data in strongly consistent buckets or all of your data, depending
+on your use case. Strong consistency was added to complement Riak's
+standard [eventually consistent][concept eventual consistency], high
+availability mode.
+
+## Tradeoffs
+
+When data is stored in a bucket with strong consistency guarantees, a
+value is guaranteed readable by any client _immediately_ after a
+successful write has occurred to a given key. In this sense, single-key
+strongly consistent operations are atomic, and operations on a given key
+are [linearizable](http://en.wikipedia.org/wiki/Linearizability). This
+behavior comes at the expense of availability because a [quorum][use ref strong consistency#trade-offs] of primary [vnodes][glossary vnode] responsible for the key must be online and reachable or the request will
+fail.
+
+This trade-off is unavoidable for strongly consistent data, but the
+[choice is now yours](http://en.wikipedia.org/wiki/CAP_theorem) to make.
+
+## Enabling Strong Consistency
+
+Complete instructions on enabling strong consistency can be found in
+our documentation on [configuring strong consistency][config strong consistency#enable].
+
+## Creating Consistent Bucket Types
+
+[Strong Consistency][use ref strong consistency] requirements in Riak are applied on a bucket-by-bucket basis, meaning that you can use some buckets in an eventually consistent fashion and others in a strongly consistent
+fashion, depending on your use case.
+
+To apply strong consistency to a bucket, you must create a [bucket type][usage bucket types] that sets the `consistent` bucket property to
+`true`, activate that type, and then apply that type to specific
+bucket/key pairs.
+
+To give an example, we'll create a bucket type called
+`strongly_consistent` with the `consistent` bucket property set to
+`true`:
+
+```bash
+riak-admin bucket-type create strongly_consistent \
+    '{"props":{"consistent":true}}'
+```
+
+> **Note on bucket type names**
+>
+> You can name [bucket types][usage bucket types] whatever you wish, with
+the exception of `default`, which is a reserved term (a full listing of
+the properties associated with the `default` bucket type can be found in
+the documentation on [bucket properties and operations][cluster ops bucket types]).
+
+Once the `strongly_consistent` bucket type has been created, we can
+check the status of the type to ensure that it has propagated through
+all nodes and is thus ready to be activated:
+
+```bash
+riak-admin bucket-type status strongly_consistent
+```
+
+If the console outputs `strongly_consistent has been created and may be
+activated` and the properties listing shows that `consistent` has been
+set to `true`, then you may proceed with activation:
+
+```bash
+riak-admin bucket-type activate strongly_consistent
+```
+
+When activation is successful, the console will return the following:
+
+```bash
+strongly_consistent has been activated
+```
+
+Now, any bucket that bears the type `strongly_consistent`---or whatever
+you wish to name it---will provide strong consistency guarantees.
+
+Elsewhere in the Riak docs, you can find more information on [using bucket types][usage bucket types], on the concept of [strong consistency][use ref strong consistency], and on strong
+consistency [for operators][config strong consistency].
+
+## Replication Properties
+
+Strongly consistent operations in Riak function much differently from
+their [eventually consistent][concept eventual consistency] counterparts.
+Whereas eventually consistent operations enable you to set values for a
+variety of [replication properties][apps replication properties] either on each request or at the
+bucket level, [using bucket types][usage bucket types], these settings are quietly ignored
+for strongly consistent operations. These settings include `r`, `pr`,
+`w`, `rw`, and others. Two replication properties that _can_ be set,
+however, are `n_val` and `return_body`.
+
+The `n_val` property is extremely important for two reasons:
+
+1. It dictates how fault tolerant a strongly consistent bucket is. More
+   information can be found in [our recommendations for operators][config strong consistency#fault].
+2. Once the `n_val` property is set for a given bucket type, it cannot
+   be changed. If you wish to change the `n_val` for one or more
+   strongly consistent buckets [using bucket types][usage bucket types], you will need to
+   create a new bucket type with the desired `n_val`.
+
+We also recommend setting the `n_val` on strongly consistent buckets to
+at least 5. More on why we make this recommendation can be found in
+[Fault Tolerance][config strong consistency#fault].
+
+## Causal Context
+
+Riak uses [causal context][concept causal context] to determine the causal history of objects.
+In versions of Riak KV prior to 2.0, [vector clocks][concept causal context#vector] were used to provide objects with causal context
+metadata. In Riak versions 2.0 and later there is an option to use
+[dotted version vectors][concept version vector], which function much like vector clocks from
+the standpoint of clients, but with important advantages over vector
+clocks.
+
+While we strongly recommend attaching context to objects for all
+updates---whether traditional vector clocks or the newer dotted version
+vectors---they are purely [optional][usage conflict resolution] for all
+eventually consistent operations in Riak. This is not the case for
+strongly consistent operations. **When modifying strongly consistent
+objects in Riak, you _must_ attach a causal context**.
+
+If you attempt to modify a strongly consistent object without attaching
+a context to the request, the request will always fail. And while it is
+possible to make writes to non-existing keys without attaching context,
+we recommend doing this only if you are certain that the key does not
+yet exist.
+
+Instructions on using causal context can be found in our documentation
+on [object updates][usage update objects].
+
+## Strongly Consistent Writes
+
+Writing to strongly consistent keys involves some of the same best
+practices that we advise when writing to eventually consistent keys. We
+recommend bearing the following in mind:
+
+1. If you _know_ that a key does not yet exist, you can write to that
+   key without supplying a context with the object. If you are unsure, then you should default to supplying a context object.
+2. If an object already exists under a key, strong consistency demands
+   that you supply a [causal context](#causal-context). If you do not supply one, the update
+   will necessarily fail.
+3. Because strongly consistent writes must occasionally
+   [sacrifice availability][use ref strong consistency#vs] for the sake of
+   consistency, **strongly consistent updates can fail even under normal
+   conditions**, particularly in the event of concurrent updates.
+
+## Error Messages
+
+For the most part, performing reads, writes, and deletes on data in
+strongly consistent buckets works much like it does in
+non-strongly-consistent-buckets. One important exception to this is how
+writes are performed. Strongly consistent buckets cannot allow siblings
+by definition, and so all writes to existing keys must include a context
+with the object.
+
+If you attempt a write to a non-empty key without including causal
+context, you will receive the following error:
+
+```ruby
+Riak::Conflict: The object is in conflict (has siblings) and cannot be treated singly or saved:
+```
+
+```java
+java.lang.IllegalArgumentException: VClock cannot be null.
+```
+
+```php
+$response->isSuccess();  // false
+$response->getStatusCode(); // 412
+```
+
+```python
+riak.RiakError: 'failed'
+```
+
+```erlang
+{error,<<"failed">>}
+```
+
+```curl
+412 Precondition Failed

Precondition Failed

Precondition Failed


mochiweb+webmachine web server
+``` + +> **Getting Started with Riak KV clients** +> +> If you are connecting to Riak using one of Basho's official +[client libraries][dev client libraries], you can find more information about getting started with your client in our [Developing with Riak KV: Getting Started][getting started] section. + +## Known Issue with Client Libraries + +All of Basho's official [client libraries][dev client libraries] currently convert errors returned by Riak into generic exceptions, with a message derived from the error message returned by Riak. In many cases this presents no +problems, since many error conditions are normal when using Riak. + +When working with strong consistency, however, operations like +[conditional puts][config strong consistency#details] commonly +produce errors that are difficult for clients to interpret. For example, +it is expected behavior for conditional puts to fail in the case of +concurrent updates to an object. At present, the official Riak clients +will convert this failure into an exception that is no different from +other error conditions, i.e. they will not indicate any +strong-consistency-specific errors. + +The best solution to this problem at the moment is to catch these +exceptions on the application side and parse server-side error messages +to see if the error involved a conditional failure. If so, you should +set up your application to retry any updates, perhaps a specified number +of times or perhaps indefinitely, depending on the use case. + +If you do set up a retry logic of this sort, however, it is necessary +to retry the entire read/modify/put cycle, meaning that you will need +to fetch the object, modify it, and then write. If you perform a simple +put over and over again, without reading the object, the update will +continue to fail. + +A future version of Riak will address these issues by modifying the +server API to more accurately report errors specific to strongly +consistent operations. diff --git a/content/riak/kv/2.2.6/developing/app-guide/write-once.md b/content/riak/kv/2.2.6/developing/app-guide/write-once.md new file mode 100644 index 0000000000..69df6542b9 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/app-guide/write-once.md @@ -0,0 +1,155 @@ +--- +title: "Write Once" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Write Once" + identifier: "app_guide_write_once" + weight: 102 + parent: "developing_app_guide" +toc: true +version_history: + in: "2.1.0+" +aliases: + - /riak-docs/riak/2.2.6/dev/advanced/write-once + - /riak-docs/riak/kv/2.2.6/dev/advanced/write-once +--- + +[glossary vnode]: {{}}riak/kv/2.2.6/learn/glossary/#vnode +[bucket type]: {{}}riak/kv/2.2.6/developing/usage/bucket-types +[Riak data types]: {{}}riak/kv/2.2.6/developing/data-types +[strong consistency]: {{}}riak/kv/2.2.6/developing/app-guide/strong-consistency + +Write-once buckets are buckets whose entries are intended to be written exactly once and never updated or overwritten. Buckets of this type circumvent the normal "coordinated PUT" path, which would otherwise result in a read on the coordinating vnode before the write. Avoiding coordinated PUTs results in higher throughput and lower PUT latency, though at the cost of different semantics in the degenerate case of sibling resolution. + +{{% note %}} +Write-once buckets do not support Riak commit hooks. Because Riak objects are +inserted into the realtime queue using a postcommit hook, realtime replication +is unavailable for write-once buckets. Fullsync replication will, however, +replicate the data. +{{% /note %}} + +## Configuration + +When the new `write_once` [bucket type][bucket type] parameter is set to +`true`, buckets of type will treat all key/value entries as semantically "write +once;" once written, entries should not be modified or overwritten by the user. + +The `write_once` property is a boolean property applied to a bucket type and +may only be set at bucket creation time. Once a bucket type has been set with +this property and activated, the `write_once` property may not be modified. + +The `write_once` property is incompatible with [Riak data types][Riak data types] +and [strong consistency][strong consistency], This means that if you attempt +to create a bucket type with the `write_once` property set to `true`, any +attempt to set the `datatype` parameter or to set the `consistent` parameter +to `true` will fail. + +The `write_once` property may not be set on the default bucket type, and may +not be set on individual buckets. If you set the `lww` or `allow_mult` +parameters on a write-once bucket type, those settings will be ignored, as +sibling values are disallowed by default. + +The following example shows how to configure a bucket type with the +`write_once` property: + +```bash +riak-admin bucket-type create my-bucket-type '{"props": {"write_once": true}}' +# my-bucket-type created + +riak-admin bucket-type activate my-bucket-type +# my-bucket-type has been activated + +riak-admin bucket-type status my-bucket-type +# my-bucket-type is active +... +write_once: true +... +``` + +## Quorum + +The write path used by write-once buckets supports the `w`, `pw`, and `dw` +configuration values. However, if `dw` is specified, then the value of `w` is +taken to be the maximum of the `w` and `dw` values. For example, for an `n_val` +of 3, if `dw` is set to `all`, then `w` will be `3`. + +This write additionally supports the `sloppy_quorum` property. If set to +`false`, only primary nodes will be selected for calculation of write quorum +nodes. + +## Runtime + +The write-once path circumvents the normal coordinated PUT code path, and +instead sends write requests directly to all [vnodes][glossary vnode] (or +vnode proxies) in the effective preference list for the write operation. + +In place of the `put_fsm` used in the normal path, we introduce a collection of +new intermediate worker processes (implementing `gen_server` behavior). The +role of these intermediate processes is to dispatch put requests to vnode or +vnode proxies in the preflist and to aggregate replies. Unlike the `put_fsm`, +the write-once workers are long-lived for the lifecycle of the `riak_kv` +application. They are therefore stateful and store request state in a state- +local dictionary. + +The relationship between the `riak_client`, write-once workers, and vnode +proxies is illustrated in the following diagram: + +
+![Write Once]({{}}images/write_once.png) +
+ +## Client Impacts + +Since the write-once code path is optimized for writes of data that will not +be updated and therefore may potentially issue asynchronous writes, some +client features might not work as expected. For example, PUT requests asking +for the object to be returned will behave like requests that do not +request the object to be returned when they are performed against write-once +buckets. + + +## Siblings + +As mentioned, entries in write-once buckets are intended to be written only +once---users who are not abusing the semantics of the bucket type should not be +updating or over-writing entries in buckets of this type. However, it is +possible for users to misuse the API, accidentally or otherwise, which might +result in incomparable entries for the same key. + +In the case of siblings, write-once buckets will resolve the conflict by +choosing the "least" entry, where sibling ordering is based on a deterministic +SHA-1 hash of the objects. While this algorithm is repeatable and deterministic +at the database level, it will have the appearance to the user of "random write +wins." + +{{% note %}} +As mentioned in [Configuration](#configuration), write-once buckets and Riak +Data Types are incompatible because of this. +{{% /note %}} + + +## Handoff + +The write-once path supports handoff scenarios, such that if a handoff occurs +during PUTs in a write-once bucket, the values that have been written will be +handed off to the newly added Riak node. + +## Asynchronous Writes + +For backends that support asynchronous writes, the write-once path will +dispatch a write request to the backend and handle the response +asynchronously. This behavior allows the vnode to free itself for other work +instead of waiting on the write response from the backend. + +At the time of writing, the only backend that supports asynchronous writes is +LevelDB. Riak will automatically fall back to synchronous writes with all other +backends. + +{{% note title="Note on the `multi` backend" %}} +The [Multi]({{}}riak/kv/2.2.6/setup/planning/backend/multi) backend does not +support asynchronous writes. Therefore, if LevelDB is used with the Multi +backend, it will be used in synchronous mode. +{{% /note %}} diff --git a/content/riak/kv/2.2.6/developing/client-libraries.md b/content/riak/kv/2.2.6/developing/client-libraries.md new file mode 100644 index 0000000000..4d73f2de0c --- /dev/null +++ b/content/riak/kv/2.2.6/developing/client-libraries.md @@ -0,0 +1,304 @@ +--- +title: "Client Libraries" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Client Libraries" + identifier: "developing_client_libraries" + weight: 106 + parent: "developing" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/using/libraries + - /riak-docs/riak/kv/2.2.6/dev/using/libraries +--- + +## Basho-Supported Libraries + +Basho officially supports a number of open-source client libraries for a +variety of programming languages and environments. + +Language | Source | Documentation | Download +:--------|:-------|:--------------|:-------- +Java | [riak-java-client](https://github.com/basho/riak-java-client) | [javadoc](http://basho.github.com/riak-java-client), [wiki](https://github.com/basho/riak-java-client/wiki) | [Maven Central](http://search.maven.org/?#search%7Cgav%7C1%7Cg%3A%22com.basho.riak%22%20AND%20a%3A%22riak-client%22) | +Ruby | [riak-ruby-client](https://github.com/basho/riak-ruby-client) | [GitHub Pages](http://basho.github.io/riak-ruby-client/) | [RubyGems](https://rubygems.org/gems/riak-client) +Python | [riak-python-client](https://github.com/basho/riak-python-client) | [sphinx](http://basho.github.com/riak-python-client) | [PyPI](http://pypi.python.org/pypi?:action=display&name=riak#downloads) +C# | [riak-dotnet-client](https://github.com/basho/riak-dotnet-client) | [api docs](http://basho.github.io/riak-dotnet-client-api/), [wiki](https://github.com/basho/riak-dotnet-client/wiki) | [NuGet package](http://www.nuget.org/List/Packages/RiakClient), [GitHub Releases](https://github.com/basho/riak-dotnet-client/releases) +Node.js | [riak-nodejs-client](https://github.com/basho/riak-nodejs-client) | [api docs](http://basho.github.com/riak-nodejs-client/), [wiki](https://github.com/basho/riak-nodejs-client/wiki) | [NPM](https://www.npmjs.com/package/basho-riak-client), [GitHub Releases](https://github.com/basho/riak-nodejs-client/releases) +PHP | [riak-php-client](https://github.com/basho/riak-php-client) | [apigen](http://basho.github.io/riak-php-client) +Erlang | [riak-erlang-client (riakc)](https://github.com/basho/riak-erlang-client) | [edoc](http://basho.github.com/riak-erlang-client/) | [GitHub](https://github.com/basho/riak-erlang-client) +Go | [riak-go-client](https://github.com/basho/riak-go-client) | [GoDoc](https://godoc.org/github.com/basho/riak-go-client) | [GitHub](https://github.com/basho/riak-go-client) + +**Note**: All official clients use the integrated issue tracker on +GitHub for bug reporting. + +In addition to the official clients, Basho provides some unofficial +client libraries, listed below. There are also many client libraries and +related [community projects]({{}}community/projects/). + + +## Community Libraries + +The Riak Community is developing at a break-neck pace, and the number of +community-contributed libraries and drivers is growing right along side +it. Here is a list of projects that may suit your programming needs or +curiosities. If you know of something that needs to be added or are +developing something that you wish to see added to this list, please +fork the [Riak Docs repo on GitHub](https://github.com/basho/basho_docs) +and send us a pull request. + +{{% note title="Note on community-produced libraries" %}} +All of these projects and libraries are at various stages of completeness and +may not suit your application's needs based on their level of maturity and +activity. +{{% /note %}} + +### Client Libraries and Frameworks + +#### C/C++ + +* [riak-cpp](https://github.com/ajtack/riak-cpp) --- A C++ Riak client + library for use with C++11 compilers +* [Riak C Driver](https://github.com/fenek/riak-c-driver) --- A library + to communicate with Riak using cURL and Protocol Buffers +* [Riack](https://github.com/trifork/riack) --- A simple C client + library +* [Riack++](https://github.com/TriKaspar/riack_cpp) --- A C++ wrapper + around riack + +#### Clojure + +* [knockbox](https://github.com/reiddraper/knockbox) --- An eventual + consistency toolbox for Clojure +* [Welle](http://clojureriak.info) --- An expressive Clojure client with + batteries included +* [clj-riak](http://github.com/mmcgrana/clj-riak) --- Clojure bindings + to the Riak Protocol Buffers API +* [sumo](https://github.com/reiddraper/sumo) --- A Protocol + Buffer-specific client for Riak with KV, 2i, and MapReduce support +* [kria](https://github.com/bluemont/kria) --- Riak 2.0 Asynchronous + (NIO.2) Clojure client. Callback driven, low level, Protocol Buffer + API, Java 7. + +#### ColdFusion + +* [Riak-Cache-Extension](https://github.com/getrailo/Riak-Cache-Extension) + --- A Riak-backed cache extension for Railo/ColdFusion + +#### Common Lisp + +* [cl-riak (1)](https://github.com/whee/cl-riak) +* [cl-riak (2)](https://github.com/eriknomitch/cl-riak) + +#### Dart + +* [riak-dart](https://github.com/agilord/riak_dart_client) --- HTTP + client for Riak written in Dart + +#### Django (Python) + +* [django-riak-sessions](https://github.com/flashingpumpkin/django-riak-sessions) + --- Riak-based Session Backend for Django +* [Django Riak Engine](https://github.com/oubiwann/django-riak-engine) + --- A Riak backend for Django + +#### Erlang + +* [Uriak Pool](https://github.com/unisontech/uriak_pool) --- Erlang + connection pool library from the team at + [Unison](http://www.unison.com) +* [Riak PBC Pool](https://github.com/snoopaloop/Riak-PBC-Pool) --- Riak + Protocol Buffer Client pool application +* [Pooly](https://github.com/aberman/pooly) --- Riak Process Pool +* [riakpool](https://github.com/dweldon/riakpool) --- Application for + maintaining a dynamic pool of Protocol Buffer client connections to a + Riak database +* [pooler](https://github.com/seth/pooler) --- An OTP Process Pool + Application +* [krc](https://github.com/klarna/krc) --- A simple wrapper around the + official Riak client for Erlang +* [riakc_pool](https://github.com/brb/riakc_pool) --- A really simple + Riak client process pool based on poolboy + +#### Go + +* [riaken](https://github.com/riaken) --- A fast and extendable Riak + Protocol Buffer Client +* [goriakpbc](https://github.com/tpjg/goriakpbc) --- A Golang Riak + client inspired by the Ruby riak-client from Basho and riakpbc from mrb +* [riakpbc](https://github.com/mrb/riakpbc) --- A Riak Protocol Buffer + client in Go +* [goriak](https://github.com/zegl/goriak) --- Go language driver for Riak KV + +#### Grails + +* [Grails ORM for Riak](http://www.grails.org/plugin/riak) + +#### Griffon + +* [Riak Plugin for + Griffon](http://docs.codehaus.org/display/GRIFFON/Riak+Plugin) + +#### Groovy + +* [spring-riak](https://github.com/jbrisbin/spring-riak) --- Riak + support from Groovy and/or Java + +#### Haskell + +* [Riak Haskell Client](https://github.com/markhibberd/riak-haskell-client) + --- A fast Haskell client library from the team at MailRank. + +#### Java + +* [Riak-Java-PB-Client](http://github.com/krestenkrab/riak-java-pb-client) + --- Java Client Library for Riak based on the Protocol Buffers API +* [Asynchronous Riak Java Client](https://github.com/jbrisbin/riak-async-java-client) + --- Asynchronous, NIO-based Protocol Buffers client for Riak +* [Riak Module for the Play + Framework](http://www.playframework.org/modules/riak-head/home) + +#### Lisp-flavored Erlang + +* [Gutenberg](https://github.com/dysinger/gutenberg/) --- Riak MapReduce + examples written in LFE + +#### Node.js + +* [zukai](https://github.com/natural/zukai) --- Riak ODM for Node.js + from Troy Melhase +* [riak-pb](https://github.com/CrowdProcess/riak-pb) --- Riak Protocol + Buffers client for Node.js from the team at + [CrowdProcess](http://crowdprocess.com) +* [node_riak](https://github.com/mranney/node_riak) --- Voxer's + production Node.js client for Riak. +* [riakpbc](https://github.com/nlf/riakpbc) --- A simple Riak Protocol + Buffer client library for Node.js +* [nodiak](https://npmjs.org/package/nodiak) --- Supports bulk + get/save/delete, sibling auto-resolution, MapReduce chaining, Search, + and 2i's +* [resourceful-riak](https://github.com/admazely/resourceful-riak) --- A + Riak engine to the + [resourceful](https://github.com/flatiron/resourceful/) model + framework from [flatiron](https://github.com/flatiron/) +* [Connect-Riak](https://github.com/frank06/connect-riak) --- Riak + session store for Connect backed by [Riak-js](http://riakjs.org/) +* [Riak-js](http://riakjs.com) --- Node.js client for Riak with support + for HTTP and Protocol Buffers +* [Riakjs-model](https://github.com/dandean/riakjs-model) --- a model + abstraction around riak-js +* [Node-Riak](http://github.com/orlandov/node-riak) --- A wrapper around + Node's HTTP facilities for communicating with Riak +* [riak-dc](https://github.com/janearc/riak-dc) --- A very thin, very small + http-based interface to Riak using promises intended to be used for small + tools like command-line applications; aims to have the "most-synchronous- + like" interface. +* [Nori](https://github.com/sgonyea/nori) --- Experimental Riak HTTP + library for Node.js modeled after Ripple +* [OrionNodeRiak](http://github.com/mauritslamers/OrionNodeRiak) --- + Node-based server and database-frontend for Sproutcore +* [Chinood](https://npmjs.org/package/chinood) --- Object data mapper + for Riak built on Nodiak +* [SimpleRiak](https://npmjs.org/package/simpleriak) --- A very simple + Riak HTTP client + +#### OCaml + +* [Riak OCaml Client](http://metadave.github.com/riak-ocaml-client/) --- + Riak OCaml client +* [OCaml Riakc](https://github.com/orbitz/ocaml-riakc) --- A Protocol + Buffers client for Riak + +#### Perl + +* [Net::Riak](http://search.cpan.org/~franckc/Net-Riak/) --- A Perl + interface to Riak +* [AnyEvent-Riak adapter](http://github.com/franckcuny/anyevent-riak) + --- Non-blocking Riak adapter using anyevent +* [riak-tiny](https://github.com/tempire/riak-tiny) --- Perl interface + to Riak without Moose +* [Riak::Light](https://metacpan.org/module/Riak::Light) --- Fast and + lightweight Perl client for Riak (PBC only) + +#### PHP + +* [riak-client](https://github.com/php-riak/riak-client) --- A Riak + 2.0-compliant PHP client with support for Protocol Buffers by [Fabio + Silva](https://github.com/FabioBatSilva) +* [Ripple-PHP](https://github.com/KevBurnsJr/ripple-php) --- A port of + Ripple to PHP +* [riiak](https://bitbucket.org/intel352/riiak) --- A Riak PHP client + library for the [Yii Framework](http://www.yiiframework.com/) +* [riak-php](https://github.com/marksteele/riak-php) --- A Riak PHP + client with support for Protocol Buffers +* [RiakBundle](https://github.com/remialvado/RiakBundle) --- + [Symfony](http://symfony.com) Bundle designed to ease interaction + with Riak +* [php_riak](https://github.com/TriKaspar/php_riak) --- A PHP extension + written in C, Both Riak client and PHP session module + +#### Python + +* [Aioriak](https://github.com/rambler-digital-solutions/aioriak) + --- Asyncio PBC Riak 2.0+ client library. (Based on official Basho + python client) +* [Riakasaurus](https://github.com/calston/riakasaurus) --- A Riak + client library for Twisted (based on txriak) +* [RiakKit](http://shuhaowu.com/riakkit) --- A small Python ORM that + sits on top of riak-python-client, similar to mongokit and couchdbkit +* [riakalchemy](https://github.com/Linux2Go/riakalchemy) --- Object + mapper for Riak written in Python +* [riak_crdt](https://github.com/ericmoritz/riak_crdt) --- A CRDT + (Conflict-Free Replicated Data Type) loader for Riak using the [CRDT + API](https://github.com/ericmoritz/crdt) +* [txriak](https://launchpad.net/txriak) --- A Twisted module for + communicating with Riak via the HTTP interface +* [txriakidx](https://github.com/williamsjj/txriakidx) --- Riak client + for Twisted Python that implements transparent indexes + +#### Racket + +* [riak.rkt](https://github.com/shofetim/riak.rkt) --- Racket API to + Riak +* [Racket Riak](https://github.com/dkvasnicka/racket-riak) --- Racket + 1.3.x API to Riak + +#### Ruby + +* [Risky](https://github.com/aphyr/risky) --- A lightweight Ruby ORM for + Riak +* [riak_sessions](http://github.com/igorgue/riak_sessions) --- + Riak-backed session storage for Rack +* [Riaktor](http://github.com/benmyles/riaktor) --- Ruby client and + object mapper for Riak +* [dm-riak-adapter](http://github.com/mikeric/dm-riak-adapter) --- + DataMapper adapter for Riak +* [Riak PB Client](https://github.com/sgonyea/riak-pbclient) --- Riak + Protocol Buffer Client in Ruby +* [Devise-Ripple](http://github.com/frank06/devise-ripple) --- An ORM + strategy to use Devise with Riak +* [ripple-anaf](http://github.com/bkaney/ripple-anaf) --- Accepts nested + attributes support for Ripple +* [Pabst](https://github.com/sgonyea/pabst) --- Cross-platform Ruby + extension for Protocol Buffers written in both Objective-C and + Objective-C++ + +#### Scala + +* [Riakka](http://github.com/timperrett/riakka) --- Scala library for + talking to Riak +* [Ryu](http://github.com/softprops/ryu) --- A Tornado Whirlwind Kick + Scala client for the Riak raw HTTP interface +* [Raiku](https://github.com/gideondk/Raiku) --- An Akka IO- and + Sentinel-driven Riak Scala client + +#### Smalltalk + +* [Phriak](http://www.squeaksource.com/Phriak/) --- A Riak client for + Pharo Smalltalk based on Runar Jordahl's EpigentRiakInterface +* [EpigentRiakInterface](http://www.squeaksource.com/EpigentRiakInterface/) + --- A Pharo Smalltalk interface to Riak. There is also a blog post + with some additional info about the client + [here](http://blog.epigent.com/2011/03/riak-interface-for-pharo-smalltalk.html). diff --git a/content/riak/kv/2.2.6/developing/data-modeling.md b/content/riak/kv/2.2.6/developing/data-modeling.md new file mode 100644 index 0000000000..3fc390cec2 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/data-modeling.md @@ -0,0 +1,10 @@ +--- +layout: redirect +target: "riak/kv/2.2.6/learn/use-cases/" +--- + +This page exists solely to redirect from the generated URL to the above `target` + +We prefer to store these redirects as .html files in static/, but -- to maintain +the git history of this (possibly malformed?) file -- we're going to start off +by using this generated redirect. diff --git a/content/riak/kv/2.2.6/developing/data-types.md b/content/riak/kv/2.2.6/developing/data-types.md new file mode 100644 index 0000000000..9d16156c55 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/data-types.md @@ -0,0 +1,275 @@ +--- +title_supertext: "Developing with Riak KV" +title: "Data Types" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Data Types" + identifier: "developing_data_types" + weight: 102 + parent: "developing" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/using/data-types + - /riak-docs/riak/kv/2.2.6/dev/using/data-types + - /riak-docs/riak/2.2.6/dev/data-modeling/data-types + - /riak-docs/riak/kv/2.2.6/dev/data-modeling/data-types +--- + +[wiki crdt]: https://en.wikipedia.org/wiki/Conflict-free_replicated_data_type#Others +[concept crdt]: ../../learn/concepts/crdts +[ops bucket type]: ../../using/cluster-operations/bucket-types + +Riak KV has Riak-specific data types based on [convergent replicated data types (CRDTs)][wiki crdt]. While Riak KV was built as a data-agnostic key/value store, Riak data types enable you to use Riak KV as a data-aware system and perform transactions on 6 CRDT-inspired data types: + +- [Flags](./maps#flags) +- [Registers](./maps#registers) +- [Counters](./counters) +- [Sets](./sets) +- [GSets](./gsets) +- [Maps](./maps) + +Riak KV also has 1 context-free data type, that has similar usage but does not require contexts. + +- [HyperLogLogs](./hyperloglogs) (abbreviated `hll` in many places) + + +Counters, sets, gsets, maps, and hyperloglogs can be used as bucket-level data types or types that you interact with directly. Flags and registers must be [embedded in maps](./maps). + +For more information on how CRDTs work in Riak KV see [Concepts: Data Types][concept crdt]. + +## Getting Started with Riak Data Types + +The following section explains how to set up a bucket that uses Riak data types. To get started using Riak data types: + +1. [Create a bucket with the `datatype` parameter set](#creating-a-bucket-with-a-data-type). +2. [Confirm the bucket was properly configured](#confirm-bucket-configuration). +3. [Activate the bucket type](#activate-the-bucket-type). + +### Creating a Bucket with a Riak Data Type + +First create a [bucket type][ops bucket type] that sets the `datatype` bucket parameter to either `counter`, `map`, `set`, or `hll`. + +The following would create a separate bucket type for each of the four +bucket-level data types: + +```bash +riak-admin bucket-type create maps '{"props":{"datatype":"map"}}' +riak-admin bucket-type create sets '{"props":{"datatype":"set"}}' +riak-admin bucket-type create counters '{"props":{"datatype":"counter"}}' +riak-admin bucket-type create hlls '{"props":{"datatype":"hll"}}' +riak-admin bucket-type create gsets '{"props":{"datatype":"gset"}}' +``` + +> **Note** +> +> The names `maps`, `sets`, `counters`, `hlls` and `gsets` are not reserved +terms. You are free to name bucket types whatever you like, with +the exception of `default`. + +### Confirm Bucket configuration + +Once you've created a bucket with a Riak data type, you can check +to make sure that the bucket property configuration associated with that +type is correct. This can be done through the `riak-admin` interface: + +```bash +riak-admin bucket-type status maps +``` + +This will return a list of bucket properties and their associated values +in the form of `property: value`. If our `maps` bucket type has been set +properly, we should see the following pair in our console output: + +``` +datatype: map +``` + +### Activate Bucket type + +If a bucket type has been properly constructed, it needs to be activated +to be usable in Riak. This can also be done using the `bucket-type` +command interface: + +```bash +riak-admin bucket-type activate maps +``` + +To check whether activation has been successful, simply use the same +`bucket-type status` command shown above. + +See the [Usage Examples](#usage-examples) section for further information on using Riak data types in the context of an application. + +## Required Bucket Properties + +In order for Riak data types to work the bucket should have the following bucket properties: + +- `allow_mult = true` +- `last_write_wins = false` + +These settings are set by default and should not be changed. + +## Data Types and Context + +Data type context is similar to [causal context](../../learn/concepts/causal-context): it tells Riak KV which version of the data type a client is attempting to modify. Context is required by Riak KV when making decisions about convergence. + +If no context is given when attempting a remove or remove-like operation, the operation may fail (removing a field that is not present) or succeed and remove more than intended (removing updates unseen by the client). + +> **Note** +> +> The counter data type does not use context; Riak KV will return an empty value when the context is requested from a counter. + +In the example below we'll fetch the context [from a user data map created for Ahmed](./maps#create-a-map): + +```java +// Using the "ahmedMap" Location from above: + +FetchMap fetch = new FetchMap.Builder(ahmedMap).build(); +FetchMap.Response response = client.execute(fetch); +Context ctx = response.getContext(); +System.out.prinntln(ctx.getValue().toString()) + +// An indecipherable string of Unicode characters should then appear +``` + +```ruby +bucket = client.bucket('users') +ahmed_map = Riak::Crdt::Map.new(bucket, 'ahmed_info', 'maps') +ahmed_map.instance_variable_get(:@context) + +# => "\x83l\x00\x00\x00\x01h\x02m\x00\x00\x00\b#\t\xFE\xF9S\x95\xBD3a\x01j" +``` + +```php +$map = (new \Basho\Riak\Command\Builder\FetchMap($riak)) + ->atLocation($location) + ->build() + ->execute() + ->getMap(); + +echo $map->getContext(); // g2wAAAACaAJtAAAACLQFHUkv4m2IYQdoAm0AAAAIxVKxCy5pjMdhCWo= +``` + +```python +bucket = client.bucket_type('maps').bucket('users') +ahmed_map = Map(bucket, 'ahmed_info') +ahmed_map.context + +# g2wAAAABaAJtAAAACCMJ/vlTlb0zYQFq +``` + +```csharp +// https://github.com/basho/riak-dotnet-client/blob/develop/src/RiakClientExamples/Dev/Using/DataTypes.cs + +// Note: using a previous UpdateMap or FetchMap result +Console.WriteLine(format: "Context: {0}", args: Convert.ToBase64String(result.Context)); + +// Output: +// Context: g2wAAAACaAJtAAAACLQFHUkv4m2IYQdoAm0AAAAIxVKxCy5pjMdhCWo= +``` + +```javascript +var options = { + bucketType: 'maps', + bucket: 'customers', + key: 'ahmed_info' +}; + +client.fetchMap(options, function (err, rslt) { + if (err) { + throw new Error(err); + } + + logger.info("context: '%s'", rslt.context.toString('base64')); +}); + +// Output: +// context: 'g2wAAAACaAJtAAAACLQFHUmjDf4EYTBoAm0AAAAIxVKxC6F1L2dhSWo=' +``` + +```erlang +%% You cannot fetch a data type's context directly using the Erlang +%% client. This is actually quite all right, as the client automatically +%% manages contexts when making updates. +``` + +> **Context with the Ruby, Python, and Erlang clients** +> +> In the Ruby, Python, and Erlang clients, you will not need to manually +handle context when making data type updates. The clients will do it all +for you. The one exception amongst the official clients is the Java +client. We'll explain how to use data type contexts with the Java client +directly below. + +### Context with the Java and PHP Clients + +With the Java and PHP clients, you'll need to manually fetch and return data type contexts for the following operations: + +* Disabling a flag within a map +* Removing an item from a set (whether the set is on its own or within a + map) +* Removing a field from a map + +Without context, these operations simply will not succeed due to the +convergence logic driving Riak data types. The example below shows you +how to fetch a data type's context and then pass it back to Riak. More +specifically, we'll remove the `paid_account` flag from the map: + +```java +// This example uses our "ahmedMap" location from above: + +FetchMap fetch = new FetchMap.Builder(ahmedMap) + .build(); +FetchMap.Response response = client.execute(fetch); +Context ctx = response.getContext(); +MapUpdate removePaidAccountField = new MapUpdate() + .removeFlag("paid_account"); +UpdateMap update = new UpdateMap.Builder(ahmedMap, removePaidAccountField) + .withContext(ctx) + .build(); +client.execute(update); +``` + + +```php +$map = (new \Basho\Riak\Command\Builder\FetchMap($riak)) + ->atLocation($location) + ->build() + ->execute() + ->getMap(); + +$updateSet = (new \Basho\Riak\Command\Builder\UpdateSet($riak)) + ->remove('opera'); + +(new \Basho\Riak\Command\Builder\UpdateMap($riak)) + ->updateSet('interests', $updateSet) + ->atLocation($location) + ->withContext($map->getContext()) + ->build() + ->execute(); +``` + +## Usage Examples + +- [Flags](./maps#flags) +- [Registers](./maps#registers) +- [Counters](./counters) +- [Sets](./sets) +- [Maps](./maps) +- [GSets](./gsets) +- [Hyperloglogs](./hyperloglogs) + +The pages listed above detail using Riak data types at the application level using Basho's [officially supported Riak KV clients](../client-libraries). For more on getting started with client libraries check out the [Developing with Riak KV: Getting Started](../getting-started) section. + +All the examples use the bucket type names from above (`counters`, `sets`, and `maps`). You're free to substitute your own bucket type names if you wish. + +## Data Types and Search + +Riak data types can be searched like any other object, but with the +added benefit that your data type is indexed as a different type by Solr, +the search platform behind Riak Search. + +In our Search documentation we offer a [full tutorial](../usage/searching-data-types) as well as a variety of [examples](../usage/search/#data-types-and-search-examples), including code +samples from each of our official client libraries. diff --git a/content/riak/kv/2.2.6/developing/data-types/counters.md b/content/riak/kv/2.2.6/developing/data-types/counters.md new file mode 100644 index 0000000000..6f70a9eae6 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/data-types/counters.md @@ -0,0 +1,631 @@ +--- +title_supertext: "Developing with Riak KV" +title: "Data Types: Counters" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Counters" + identifier: "data_types_counters" + weight: 100 + parent: "developing_data_types" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/using/data-types/counters + - /riak-docs/riak/kv/2.2.6/dev/using/data-types/counters + - /riak-docs/riak/2.2.6/dev/data-modeling/data-types/counters + - /riak-docs/riak/kv/2.2.6/dev/data-modeling/data-types/counters +--- + +Counters are a bucket-level Riak data type that can be used by themselves, associated with a bucket/key pair, or used [within a map](../maps#counters-within-maps). A counter's value can only be a positive integer, negative integer, or zero. + +The examples in this section will show you how to use counters on their own. + +## Set Up a Bucket Type + +> If you've already created and activated a bucket type with the `datatype` parameter set to `counter`, skip to the [next section](#client-setup). + +Start by creating a bucket type with the `datatype` parameter set to `counter`: + +```bash +riak-admin bucket-type create counters '{"props":{"datatype":"counter"}}' +``` + +> **Note** +> +> The `counters` bucket type name provided above is an example and is not required to be `counters`. You are free to name bucket types whatever you like, with the exception of `default`. + +After creating a bucket with a Riak data type, confirm the bucket property configuration associated with that type is correct: + +```bash +riak-admin bucket-type status counters +``` + +This returns a list of bucket properties and their values +in the form of `property: value`. + +If our `counters` bucket type has been set properly we should see the following pair in our console output: + +```bash +datatype: counter +``` + +Once we have confirmed the bucket type is properly configured, we can activate the bucket type to be used in Riak KV: + +```bash +riak-admin bucket-type activate counters +``` + +We can check if activation has been successful by using the same `bucket-type status` command shown above: + +```bash +riak-admin bucket-type status counters +``` + +After creating and activating our new `counters` bucket type, we can setup our client to start using the bucket type as detailed in the next section. + +## Client Setup + +First, we need to direct our client to the bucket type/bucket/key +location that contains our counter. + +For this example we'll use the `counters` bucket type created and activated above and a bucket called `counters`: + +```java +// In the Java client, a bucket/bucket type combination is specified +// using a Namespace object. To specify bucket, bucket type, and key, +// use a Location object that incorporates the Namespace object, as is +// done below. +Namespace countersBucket = new Namespace("counters", "counters"); +Location location = new Location(countersBucket, ""); +``` + +```ruby +bucket = client.bucket_type('counters').bucket('counters') +``` + +```php +$bucket = new \Basho\Riak\Bucket('counters', 'counters'); +``` + +```python +bucket = client.bucket_type('counters').bucket('counters') +``` + +```csharp +// https://github.com/basho/riak-dotnet-client/blob/develop/src/RiakClientExamples/Dev/Using/DataTypes.cs + +// You can either use the appropriate Options class or the Builder + +// Options: +var options = new FetchCounterOptions("counters", "counters", ""); + +// Builder: +FetchCounter cmd = new FetchCounter.Builder() + .WithBucketType("counters") + .WithBucket("counters") + .WithKey("") + .Build(); +``` + +```javascript +// The following can be passed as options to FetchCounter +var options = { + bucketType: 'counters', + bucket: 'counters', + key: '' +}; +``` + +```erlang +%% Buckets are simply named binaries in the Erlang client. See the +%% examples below for more information +``` + +```curl +curl http://localhost:8098/types/counters/buckets/counters/datatypes/ + +# Note that this differs from the URL structure for non-Data-Type +# requests, which end in /keys/ +``` + +## Create a Counter + +To create a counter, you need to specify a bucket/key pair to hold that +counter. Here is the general syntax for doing so: + +```java +// Here, we'll use the Namespace object that we created above and +// incorporate it into a Location object that includes the key (as yet +// unspecified) for our counter + +// Using the countersBucket Namespace object from above: +Location counter = new Location(countersBucket, ""); + +// Or we can specify the Location all at once: +Location counter = new Location(new Namespace("counters", "counters"), ""); +``` + +```ruby +counter = Riak::Crdt::Counter.new(bucket, key, bucket_type) + +# Or you can specify a bucket and bucket type all at once and pass that +# into the constructor +bucket = client.bucket_type(bucket_type).bucket(bucket) +counter = Riak::Crdt::Counter.new(bucket, key) +``` + +```php +# using the $bucket var created earlier +$location = new \Basho\Riak\Location('key', $bucket); +``` + +```python +# The client detects the bucket type's data type and automatically +# returns the right datatype for you, in this case a counter +counter = bucket.new(key) + +# This way is also acceptable: +from riak.datatypes import Counter + +counter = Counter(bucket, key) +``` + +```csharp +// https://github.com/basho/riak-dotnet-client/blob/develop/src/RiakClientExamples/Dev/Using/DataTypes.cs + +var fetchCounterOptions = new FetchCounterOptions("counters", "counters", ""); +FetchCounter cmd = new FetchCounter(fetchCounterOptions); +RiakResult rslt = client.Execute(cmd); +CounterResponse response = cmd.Response; +``` + +```javascript +// The following can be passed as options to the *Counter methods on the +// Node.js Client object +var options = { + bucketType: 'counters', + bucket: 'counters', + key: '' +}; +``` + +```erlang +%% Counters are not encapsulated with the bucket/key in the Erlang +%% client. See the examples below for more information. +``` + +```curl +# This will create a counter with an initial value of 0 + +curl -XPOST http://localhost:8098/types/counters/buckets//datatypes/ \ + -H "Content-Type: application/json" \ + -d '{"increment": 0}' +``` + +Let's say that we want to create a counter called `traffic_tickets` in +our `counters` bucket to keep track of our legal misbehavior. We can +create this counter and ensure that the `counters` bucket will use our +`counters` bucket type like this: + +```java +// Using the countersBucket Namespace object from above: + +Location trafficTickets = new Location(countersBucket, "traffic_tickets"); +``` + +```ruby +bucket = client.bucket_type('counters').bucket('counters') +counter = Riak::Crdt::Counter.new(bucket, 'traffic_tickets') + +# Alternatively, the Ruby client enables you to set a bucket type as +# being globally associated with a Riak data type. The following would +# set all counter buckets to use the counters bucket type: + +Riak::Crdt::DEFAULT_BUCKET_TYPES[:counter] = 'counters' + +# This would enable us to create our counter without specifying a bucket type +bucket = client.bucket('counters') +counter = Riak::Crdt::Counter.new(bucket, 'traffic_tickets') +``` + +```php +# using the $bucket var created earlier +$location = new \Basho\Riak\Location('traffic_tickets', $bucket); +``` + +```python +bucket = client.bucket_type('counters').bucket('traffic_tickets') +counter = bucket.new('traffic_tickets') +``` + +```csharp +// https://github.com/basho/riak-dotnet-client/blob/develop/src/RiakClientExamples/Dev/Using/DataTypes.cs + +var fetchCounterOptions = new FetchCounterOptions("counters", "counters", "traffic_tickts"); +FetchCounter cmd = new FetchCounter(fetchCounterOptions); +RiakResult rslt = client.Execute(cmd); +CounterResult = cmd.Result; +``` + +```javascript +// Using the options from above: + +var options = { + bucketType: 'counters', + bucket: 'counters', + key: 'traffic_tickets' +}; +``` + +```erlang +Counter = riakc_counter:new(). + +%% Counters in the Erlang client are opaque data structures that collect +%% operations as you mutate them. We will associate the data structure +%% with a bucket type, bucket, and key later on. +``` + +```curl +curl -XPOST http://localhost:8098/types/counters/buckets/counters/datatypes/traffic_tickets \ + -H "Content-Type: application/json" \ + -d '{"increment": 0}' +``` + +## Increment a Counter + +Now that our client knows which bucket/key pairing to use for our +counter, `traffic_tickets` will start out at 0 by default. If we happen +to get a ticket that afternoon, we can increment the counter: + +```java +// Using the "trafficTickets" Location from above: + +CounterUpdate cu = new CounterUpdate(1); +UpdateCounter update = new UpdateCounter.Builder(trafficTickets, cu) + .build(); +client.execute(update); +``` + +```ruby +counter.increment + +# This will increment the counter both on the application side and in +Riak +``` + +```php +(new \Basho\Riak\Command\Builder\IncrementCounter($riak)) + ->withIncrement(1) + ->atLocation($location) + ->build() + ->execute(); +``` + +```python +counter.increment() + +# Updates are staged locally and have to be explicitly sent to Riak +# using the store() method. +counter.store() +``` + +```csharp +// https://github.com/basho/riak-dotnet-client/blob/develop/src/RiakClientExamples/Dev/Using/DataTypes.cs + +UpdateCounter updateCmd = new UpdateCounter.Builder(increment: 1) + .WithBucketType("counters") + .WithBucket("counters") + .WithKey("traffic_tickets") + .WithReturnBody(true) + .Build(); + +RiakResult rslt = client.Execute(updateCmd); +CounterResponse response = updateCmd.Response; +// response.Value will be 1 +``` + +```javascript +// Using the options from above: + +var options = { + bucketType: 'counters', + bucket: 'counters', + key: 'traffic_tickets', + increment: 1 +}; +client.updateCounter(options, + function (err, rslt) { + if (err) { + throw new Error(err); + } + }); +``` + +```erlang +Counter1 = riakc_counter:increment(Counter). +``` + +```curl +curl -XPOST http://localhost:8098/types/counters/buckets/counters/datatypes/traffic_tickets \ + -H "Content-Type: application/json" \ + -d '{"increment": 1}' +``` + +## Increment a Counter by More Than 1 + +The default value of an increment operation is 1, but you can increment +by more than 1 (but always by an integer). + +Continuing with our `traffic_tickets` example, let's say we receive 5 tickets in a single day: + +```java +// Using the "trafficTickets" Location from above: +CounterUpdate cu = new CounterUpdate(5); +UpdateCounter update = new UpdateCounter.Builder(trafficTickets, cu) + .build(); +client.execute(update); +``` + +```ruby +counter.increment(5) +``` + +```php +(new \Basho\Riak\Command\Builder\IncrementCounter($riak)) + ->withIncrement(5) + ->atLocation($location) + ->build() + ->execute(); +``` + +```python +counter.increment(5) +``` + +```csharp +// https://github.com/basho/riak-dotnet-client/blob/develop/src/RiakClientExamples/Dev/Using/DataTypes.cs + +var builder = new UpdateCounter.Builder(5) + .WithBucketType("counters") + .WithBucket("counters") + .WithKey("traffic_tickets") + .WithReturnBody(true); + +UpdateCounter updateCmd = builder.Build(); + +rslt = client.Execute(updateCmd); +CounterResponse response = updateCmd.Response; +// response.Value is 5 more than before + +// To decrement: +// Modify the builder's increment, then construct a new command +builder.WithIncrement(-5); +updateCmd = builder.Build(); + +rslt = client.Execute(updateCmd); +CheckResult(rslt); + +response = updateCmd.Response; +// response.Value is 5 less than before +``` + +```javascript +var options = { + bucketType: 'counters', + bucket: 'counters', + key: 'traffic_tickets', + increment: 5 +}; +client.updateCounter(options, + function (err, rslt) { + if (err) { + throw new Error(err); + } + }); +``` + +```erlang +Counter2 = riakc_counter:increment(5, Counter1). +``` + +```curl +curl -XPOST http://localhost:8098/types/counters/buckets/counters/datatypes/traffic_tickets \ + -H "Content-Type: application/json" \ + -d '{"increment": 5}' +``` + +## Retrieve Counter Value + +We can retrieve the value of the counter and view how many tickets have accumulated: + +```java +// Using the "trafficTickets" Location from above: +FetchCounter fetch = new FetchCounter.Builder(trafficTickets) + .build(); +FetchCounter.Response response = client.execute(fetch); +RiakCounter counter = response.getDatatype(); +Long ticketsCount = counter.view(); +``` + +```ruby +counter.value +# Output will always be an integer +``` + +```php +$trafficTickets = (new \Basho\Riak\Command\Builder\FetchCounter($riak)) + ->atLocation($location) + ->build() + ->execute() + ->getCounter(); + +$trafficTickets->getData(); # returns an integer +``` + +```python +counter.dirty_value + +# The value fetched from Riak is always immutable, whereas the "dirty +# value" takes into account local modifications that have not been +# sent to the server. For example, whereas the call above would return +# 6, the call below will return 0' since we started with an empty +# counter: + +counter.value + +# To fetch the value stored on the server, use the call below. Note +# that this will clear any changes to the counter that have not yet been +# sent to Riak +counter.reload() +``` + +```csharp +// https://github.com/basho/riak-dotnet-client/blob/develop/src/RiakClientExamples/Dev/Using/DataTypes.cs + +var fetchCounterOptions = new FetchCounterOptions("counters", "counters", "traffic_tickts"); +FetchCounter cmd = new FetchCounter(fetchCounterOptions); +RiakResult rslt = client.Execute(cmd); +CounterResponse response = cmd.Response; +// response.Value has the counter value +``` + +```javascript +var options = { + bucketType: 'counters', + bucket: 'counters', + key: 'traffic_tickets' +}; +client.fetchCounter(options, + function (err, rslt) { + if (err) { + throw new Error(err); + } + + if (rslt.notFound) { + logger.error("bt: %s, b: %s, k: %s, counter: NOT FOUND", + options.bucketType, options.bucket, options.key); + } else { + logger.info("bt: %s, b: %s, k: %s, counter: %d", + options.bucketType, options.bucket, options.key, + rslt.counterValue); + } + } +); +``` + +```erlang +riakc_counter:dirty_value(Counter2). + +%% The value fetched from Riak is always immutable, whereas the "dirty +%% value" takes into account local modifications that have not been +%% sent to the server. For example, whereas the call above would return +%% '6', the call below will return '0' since we started with an empty +%% counter: + +riakc_counter:value(Counter2). + +%% To fetch the value stored on the server, use the call below: + +{ok, CounterX} = riakc_pb_socket:fetch_type(Pid, + {<<"counters">>, <<"counters">>}, + <<"traffic_tickets">>). +``` + +```curl +curl http://localhost:8098/types/counters/buckets/counters/datatypes/traffic_tickets + +# Response: +{"type":"counter", "value": } +``` + +## Decrement a Counter + +Counters enable you to decrement values in addition to incrementing them as seen above. + +For example, let's say we hire an expert lawyer who gets one of the traffic tickets stricken from our record: + +```java +// Using the "trafficTickets" Location from above: +CounterUpdate cu = new CounterUpdate(-1); +UpdateCounter update = new UpdateCounter.Builder(trafficTickets, cu) + .build(); +client.execute(update); +``` + +```ruby +counter.decrement + +# Just like incrementing, you can also decrement by more than one, e.g.: +counter.decrement(3) +``` + +```php +(new \Basho\Riak\Command\Builder\IncrementCounter($riak)) + ->withIncrement(-3) + ->atLocation($location) + ->build() + ->execute(); +``` + +```python +counter.decrement() + +# Just like incrementing, you can also decrement by more than one, e.g.: +counter.decrement(3) +``` + +```csharp +// https://github.com/basho/riak-dotnet-client/blob/develop/src/RiakClientExamples/Dev/Using/DataTypes.cs + +var updateCmd = new UpdateCounter.Builder(-3) + .WithBucketType("counters") + .WithBucket("counters") + .WithKey("traffic_tickets") + .Build(); + +rslt = client.Execute(updateCmd); +response = updateCmd.Response; +// response.Value is three less than before +``` + +```javascript +var options = { + bucketType: 'counters', + bucket: 'counter', + key: 'traffic_tickets', + increment: -1 +}; + +// As with incrementing, you can also decrement by more than one, e.g.: +var options = { + bucketType: 'counters', + bucket: 'counter', + key: 'traffic_tickets', + increment: -3 +}; +``` + +```erlang +Counter3 = riakc_counter:decrement(Counter2). + +%% As with incrementing, you can also decrement by more than one: + +Counter4 = riakc_counter:decrement(3, Counter3). + +%% At some point, we'll want to send our local updates to the server +%% so they get recorded and are visible to others. Extract the update +%% using the to_op/1 function, then pass it to +%% riakc_pb_socket:update_type/4,5. + +riakc_pb_socket:update_type(Pid, {<<"counters">>,<<"counters">>}, + <<"traffic_tickets">>, + riakc_counter:to_op(Counter4)). +``` + +```curl +curl -XPOST http://localhost:8098/types/counters/buckets/counters/datatypes/traffic_tickets \ + -H "Content-Type: application/json" \ + -d '{"decrement": 3}' +``` diff --git a/content/riak/kv/2.2.6/developing/data-types/gsets.md b/content/riak/kv/2.2.6/developing/data-types/gsets.md new file mode 100644 index 0000000000..a4df1e6edf --- /dev/null +++ b/content/riak/kv/2.2.6/developing/data-types/gsets.md @@ -0,0 +1,632 @@ +--- +title_supertext: "Developing with Riak KV" +title: "Data Types:GSets" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Sets" + identifier: "data_types_gsets" + weight: 101 + parent: "developing_data_types" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/using/data-types/gsets + - /riak-docs/riak/kv/2.2.6/dev/using/data-types/gsets + - /riak-docs/riak/2.2.6/dev/data-modeling/data-types/gsets + - /riak-docs/riak/kv/2.2.6/dev/data-modeling/data-types/gsets +--- + +GSets are a bucket-level Riak data type that can be used by themselves or associated with a bucket/key pair. They do not yet have the ability to be used [within a map like regular sets](../maps#sets-within-maps). + +GSets are collections of unique binary values (such as strings). All of the values in a gset are unique and are automatically sorted alphabetically irresepective of the order they were added. + +For example, if you attempt to add the element `shovel` to a set that already contains `shovel`, the operation will be ignored by Riak KV. + +Unlike sets, elements can only be added and no element modification or deletion is possible. + +> **Known Issue** +> +> Unlike other data types, gsets require other data to be present in the cluster before they can be created. If you are unable to create a gset on a new cluster, please try [creating a set](../sets#set-up-a-bucket-type) first and then retrying with your gset. Please see [issue #950](https://github.com/basho/riak_core/issues/950) for details. + +## Set Up a Bucket Type + +> If you've already created and activated a bucket type with `gset` as the `datatype` parameter, skip to the [next section](#client-setup). + +Start by creating a bucket type with the `datatype` parameter `gset`: + +```bash +riak-admin bucket-type create gsets '{"props":{"datatype":"gset"}}' +``` + +> **Note** +> +> The `gsets` bucket type name provided above is an example and is not required to be `gsets`. You are free to name bucket types whatever you like, with the exception of `default`. + +After creating a bucket with a Riak data type, confirm the bucket property configuration associated with that type is correct: + +```bash +riak-admin bucket-type status gsets +``` + +This returns a list of bucket properties and their values +in the form of `property: value`. + +If our `gsets` bucket type has been set properly we should see the following pair in our console output: + +``` +datatype: gset +``` + +Once we have confirmed the bucket type is properly configured, we can activate the bucket type to be used in Riak KV: + +```bash +riak-admin bucket-type activate gsets +``` + +We can check if activation has been successful by using the same `bucket-type status` command shown above: + +```bash +riak-admin bucket-type status gsets +``` + +After creating and activating our new `gsets` bucket type, we can setup our client to start using the bucket type as detailed in the next section. + +## Client Setup + +Using sets involves creating a bucket/key pair to house a gset and running gset-specific operations on that pair. + +Here is the general syntax for creating a bucket type/bucket/key +combination to handle a gset: + +```java +// In the Java client, a bucket/bucket type combination is specified +// using a Namespace object. To specify bucket, bucket type, and key, +// use a Location object that incorporates the Namespace object, as is +// done below. + +Location set = + new Location(new Namespace("", ""), ""); +``` + +```ruby +# Note: both the Riak Ruby Client and Ruby the language have a class +# called Set. Make sure that you refer to the Ruby version as ::Set and +# the Riak client version as Riak::Crdt::Set + +bucket = client.bucket_type('bucket_type_name').bucket('bucket_name') +set = Riak::Crdt::Set.new(bucket, key) +``` + +```php +$location = new \Basho\Riak\Location('key', new \Basho\Riak\Bucket('bucket_name', 'bucket_type')); +``` + +```python +# Note: The Python standard library `collections` module has an abstract +# base class called Set, which the Riak Client version subclasses as +# `riak.datatypes.Set`. These classes are not directly interchangeable. +# In addition to the base methods, `riak.datatypes.Set` also +# implements the `add` and `discard` methods from +# `collections.MutableSet`, but does not implement the rest of its +# API. Be careful when importing, or simply use the instances returned +# by `RiakBucket.get()` and `RiakBucket.new()` instead of directly +# importing the class. + +set = bucket.new(key) + +# or + +from riak.datatypes import Set +set = Set(bucket, key) +``` + +```csharp +// https://github.com/basho/riak-dotnet-client/blob/develop/src/RiakClientExamples/Dev/Using/DataTypes.cs + +// As with counters, with the Riak .NET Client you interact with gsets +// by building an Options object or using a Builder +var builder = new FetchGSet.Builder() + .WithBucketType("gsets") + .WithBucket("account-12345678") + .WithKey("2019-11-17"); + +// NB: builder.Options will only be set after Build() is called. +FetchGSet fetchGSetCommand = builder.Build(); + +FetchGSetOptions options = new FetchGSetOptions("gsets", "account-12345678", "2019-11-17"); + +// These two options objects are equal +Assert.AreEqual(options, builder.Options); +``` + +```javascript +// As with counters, with the Riak Node.js Client you interact with gsets on the +// basis of the gset's location in Riak, as specified by an options object. +// Below is an example: +var options = { + bucketType: 'gsets', + bucket: 'account-12345678', + key: '2019-11-17' +}; +``` + +```erlang +%% Like counters, sets are not encapsulated in a +%% bucket/key in the Erlang client. See below for more +%% information. +``` + +```curl +curl http://localhost:8098/types//buckets//datatypes/ + +# Note that this differs from the URL structure for non-data type requests, +# which end in /keys/ +``` + +## Create a GSet + +For the following example, we will use a set to store a list of transactions that occur for an account number on a specific date. +Let's create a Riak gset stored in the key `2019-11-17` in the bucket `account-12345678` using the `gsets` bucket type created previously: + +```java +// In the Java client, you specify the location of Data Types +// before you perform operations on them: + +Location citiesSet = + new Location(new Namespace("sets", "travel"), "cities"); +``` + +```ruby +travel = client.bucket_type('sets').bucket('travel') +cities_set = Riak::Crdt::Set.new(travel, 'cities') + +# Alternatively, the Ruby client enables you to set a bucket type as +# being globally associated with a Riak data type. The following would +# set all set buckets to use the sets bucket type: + +Riak::Crdt::DEFAULT_BUCKET_TYPES[:set] = 'sets' + +# This would enable us to create our set without specifying a bucket +# type: +travel = client.bucket('travel') +cities_set = Riak::Crdt::Set.new(travel, 'cities') +``` + +```php +$location = new \Basho\Riak\Location('2019-11-17', 'account-12345678', 'gsets'); +``` + +```python +travel = client.bucket_type('sets').bucket('travel') + +# The client detects the bucket type's data type and automatically +# returns the right data type for you, in this case a Riak set. +cities_set = travel.new('cities') + +# You can also create a reference to a set explicitly: +from riak.datatypes import Set + +cities_set = Set(travel, 'cities') +``` + +```csharp +// https://github.com/basho/riak-dotnet-client/blob/develop/src/RiakClientExamples/Dev/Using/DataTypes.cs + +// Now we'll create a Builder object for the gset with which we want to +// interact: +var builder = new FetchGSet.Builder() + .WithBucketType("gsets") + .WithBucket("account-12345678") + .WithKey("2019-11-17"); +``` + +```javascript +// Now we'll create a options object for the gset with which we want to +// interact: +var options = { + bucketType: 'gsets', + bucket: 'account-12345678', + key: '2019-11-17' +}; +``` + +```erlang +20191177Gset = riakc_gset:new(). + +%% GSets in the Erlang client are opaque data structures that +%% collect operations as you mutate them. We will associate the data +%% structure with a bucket type, bucket, and key later on. +``` + +```curl +# You cannot create an empty gset through the HTTP interface. GSets can +# only be created when an element is added to them, as in the examples +# below. +``` + +Upon creation, our set is empty. We can verify that it is empty at any +time: + +```java +// Using our "cities" Location from above: + +FetchSet fetch = new FetchSet.Builder(citiesSet) + .build(); +FetchSet.Response response = client.execute(fetch); +RiakSet set = response.getDatatype(); +boolean isEmpty = set.viewAsSet().isEmpty(); +``` + +```ruby +cities_set.empty? +``` + +```php +# use $location from earlier +$gset = (new \Basho\Riak\Command\Builder\FetchSet($riak)) + ->atLocation($location) + ->build() + ->execute() + ->getSet(); + +count($gset->getData()); +``` + +```python +len(cities_set) == 0 +``` + +```csharp +// https://github.com/basho/riak-dotnet-client/blob/develop/src/RiakClientExamples/Dev/Using/DataTypes.cs + +var builder = new FetchGSet.Builder() + .WithBucketType("gsets") + .WithBucket("account-12345678") + .WithKey("2019-11-17"); + +FetchGSet fetchGSetCommand = builder.Build(); +RiakResult rslt = client.Execute(fetchGSetCommand); +GSetResponse response = fetchGSetCommand.Response; +// response.Value will be null +``` + +```javascript +var options = { + bucketType: 'gsets', + bucket: 'account-12345678', + key: '2019-11-17' +}; +client.fetchSet(options, function (err, rslt) { + if (err) { + throw new Error(err); + } + + if (rslt.notFound) { + logger.info("gset '2019-11-17' is not found!"); + } +}); +``` + +```erlang +riakc_gset:size(20191117Gset) == 0. + +%% Query functions like size/1, is_element/2, and fold/3 operate over +%% the immutable value fetched from the server. In the case of a new +%% gset that was not fetched, this is an empty collection, so the size +%% is 0. +``` + +```curl +curl http://localhost:8098/types/gsets/buckets/account-12345678/datatypes/2019-11-17 + +# Response +{"type":"set","error":"notfound"} +``` + +## Add to a GSet + +But let's say that a pair of transactions occurred today. Let's add them to our `2019-11-17` set: + +```java +// Using our "cities" Location from above: + +SetUpdate su = new SetUpdate() + .add("Toronto") + .add("Montreal"); +UpdateSet update = new UpdateSet.Builder(citiesSet, su) + .build(); +client.execute(update); +``` + +```ruby +cities_set.add('Toronto') +cities_set.add('Montreal') +``` + +```php +# use $location from earlier +$response = (new \Basho\Riak\Command\Builder\UpdateSet($riak)) + ->add('transaction a') + ->add('transaction b') + ->atLocation($location) + ->withParameter('returnbody', 'true') + ->build() + ->execute(); +``` + +```python +cities_set.add('Toronto') +cities_set.add('Montreal') +``` + +```csharp +// https://github.com/basho/riak-dotnet-client/blob/develop/src/RiakClientExamples/Dev/Using/DataTypes.cs + +var adds = new[] { "transaction a", "transaction b" }; + +var builder = new UpdateGSet.Builder() + .WithBucketType("gsets") + .WithBucket("account-12345678") + .WithKey("2019-11-17") + .WithAdditions(adds); + +UpdateGSet cmd = builder.Build(); +RiakResult rslt = client.Execute(cmd); +GSetResponse response = cmd.Response; +Assert.Contains("transaction a", response.AsStrings.ToArray()); +Assert.Contains("transaction b", response.AsStrings.ToArray()); +``` + +```javascript +var options = { + bucketType: 'gsets', + bucket: 'account-1234578', + key: '2019-11-17' +}; +var cmd = new Riak.Commands.CRDT.UpdateGSet.Builder() + .withBucketType(options.bucketType) + .withBucket(options.bucket) + .withKey(options.key) + .withAdditions(['transaction a', 'transaction b']) + .withCallback( + function (err, rslt) { + if (err) { + throw new Error(err); + } + } + ) + .build(); +client.execute(cmd); +``` + +```erlang +20191117Gset1 = riakc_gset:add_element(<<"transaction a">>, 20191117Gset), +20191117Gset2 = riakc_gset:add_element(<<"transaction b">>, 20191117Gset1). +``` + +```curl +curl -XPOST http://localhost:8098/types/gsets/buckets/account-12345678/datatypes/2019-11-17 \ + -H "Content-Type: application/json" \ + -d '{"add_all":["transaction a", "transaction b"]}' +``` + +## Remove from a GSet + +Removal from a GSet is not possible. + +## Retrieve a GSet + +Now, we can check on which transactions are currently in our gset: + +```java +// Using our "cities" Location from above: + +FetchSet fetch = new FetchSet.Builder(citiesSet) + .build(); +FetchSet.Response response = client.execute(fetch); +Set binarySet = response.getDatatype().view(); +for (BinaryValue city : binarySet) { + System.out.println(city.toStringUtf8()); +} +``` + +```ruby +cities_set.members + +# +``` + +```php +# use $location from earlier +$gset = (new \Basho\Riak\Command\Builder\FetchSet($riak)) + ->atLocation($location) + ->build() + ->execute() + ->getSet(); + +var_dump($gset->getData()); +``` + +```python +cities_set.dirty_value + +# The value fetched from Riak is always immutable, whereas the "dirty +# value" takes into account local modifications that have not been +# sent to the server. For example, where the call above would return +# frozenset(['Toronto', 'Hamilton', 'Ottawa']), the call below would +# return frozenset([]). + +cities_set.value + +# To fetch the value stored on the server, use the call below. Note +# that this will clear any unsent additions or deletions. +cities_set.reload() +``` + +```csharp +// https://github.com/basho/riak-dotnet-client/blob/develop/src/RiakClientExamples/Dev/Using/DataTypes.cs + +foreach (var value in GSetResponse.AsStrings) +{ + Console.WriteLine("2019-11-17 Transactions: {0}", value); +} + +// Output: +// 2019-11-17 Transactions: transaction a +// 2019-11-17 Transactions: transaction b +``` + +```javascript +var options = { + bucketType: 'gsets', + bucket: 'account-12345678', + key: '2019-11-17' +}; +client.fetchSet(options, function(err, rslt) { + if (err) { + throw new Error(err); + } + + logger.info("2019-11-17 gset values: '%s'", + rslt.values.join(', ')); +}); + +// Output: +// info: 2019-11-17 gset values: 'transaction a, transaction b' +``` + +```erlang +riakc_gset:dirty_value(20191117Gset3). + +%% The value fetched from Riak is always immutable, whereas the "dirty +%% value" takes into account local modifications that have not been +%% sent to the server. For example, where the call above would return +%% [<<"Hamilton">>, <<"Ottawa">>, <<"Toronto">>], the call below would +%% return []. These are essentially ordsets: + +riakc_gset:value(20191117Gset3). + +%% To fetch the value stored on the server, use the call below: + +{ok, SetX} = riakc_pb_socket:fetch_type(Pid, + {<<"gsets">>,<<"account-12345678">>}, + <<"20191117">>). +``` + +```curl +curl http://localhost:8098/types/gsets/buckets/account-12345678/datatypes/2019-11-17 + +# Response +{"type":"set","value":["transaction a","transaction b"]} +``` + +## Find GSet Member + +Or we can see whether our gset includes a specific member: + +```java +// Using our "citiesSet" from above: + +System.out.println(citiesSet.contains(("Vancouver")); +System.out.println(citiesSet.contains("Ottawa")); +``` + +```ruby +cities_set.include? 'Vancouver' +# false + +cities_set.include? 'Ottawa' +# true +``` + +```php +in_array('transaction z', $gset->getData()); # false + +in_array('transaction a', $gset->getData()); # true +``` + +```python +'Vancouver' in cities_set +# False + +'Ottawa' in cities_set +# True +``` + +```csharp +// https://github.com/basho/riak-dotnet-client/blob/develop/src/RiakClientExamples/Dev/Using/DataTypes.cs + +using System.Linq; + +bool includesTransactionZ = response.AsStrings.Any(v => v == "transaction z"); +bool includesTransactionA = response.AsStrings.Any(v => v == "transaction a"); +``` + +```javascript +// Use standard javascript array method indexOf() + +var 2019-11-17_gset = result.values; +2019-11-17_gset.indexOf('transaction z'); // if present, index is >= 0 +2019-11-17_gset.indexOf('transaction a'); // if present, index is >= 0 +``` + +```erlang +%% At this point, GSet3 is the most "recent" set from the standpoint +%% of our application. + +riakc_gset:is_element(<<"transaction z">>, 20191117Gset3). +riakc_gset:is_element(<<"transaction a">>, 20191117Gset3). +``` + +```curl +# With the HTTP interface, this can be determined from the output of +# a fetch command like the one displayed in the example above +``` + +## Size of GSet + +We can also determine the size of the gset: + +```java +// Using our "citiesSet" from above: + +int numberOfCities = citiesSet.size(); +``` + +```ruby +cities_set.members.length +``` + +```php +count($gset->getData()); +``` + +```python +len(cities_set) +``` + +```csharp +// https://github.com/basho/riak-dotnet-client/blob/develop/src/RiakClientExamples/Dev/Using/DataTypes.cs + +using System.Linq; + +// Note: this enumerates the IEnumerable +gsetResponse.Values.Count(); +``` + +```javascript +// Use standard javascript array property length + +var 2019-11-17_gset_size = result.values.length; +``` + +```erlang +riakc_gset:size(20191117Gset3). +``` + +```curl +# With the HTTP interface, this can be determined from the output of +# a fetch command like the one displayed in the example above +``` diff --git a/content/riak/kv/2.2.6/developing/data-types/hyperloglogs.md b/content/riak/kv/2.2.6/developing/data-types/hyperloglogs.md new file mode 100644 index 0000000000..f447927ec0 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/data-types/hyperloglogs.md @@ -0,0 +1,639 @@ +--- +title_supertext: "Developing with Riak KV" +title: "Data Types: HyperLogLogs" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "HyperLogLogs" + identifier: "data_types_hyperloglogs" + weight: 100 + parent: "developing_data_types" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/using/data-types/hyperloglogs + - /riak-docs/riak/kv/2.2.6/dev/using/data-types/hyperloglogs + - /riak-docs/riak/2.2.6/dev/data-modeling/data-types/hyperloglogs + - /riak-docs/riak/kv/2.2.6/dev/data-modeling/data-types/hyperloglogs +--- + +The examples in this section will show you how to use hyperloglogs on their own. + +## Set Up a Bucket Type + +> If you've already created and activated a bucket type with the `datatype` parameter set to `hyperloglog`, skip to the [next section](#client-setup). + +Start by creating a bucket type with the `datatype` parameter set to `hyperloglog`: + +```bash +riak_admin bucket-type create hlls '{"props":{"datatype":"hll"}}' +``` + +> **Note** +> +> The `hlls` bucket type name provided above is an example and is not required to be `hlls`. You are free to name bucket types whatever you like, with the exception of `default`. + +After creating a bucket with a Riak data type, confirm the bucket property configuration associated with that type is correct: + +```bash +riak-admin bucket-type status hlls +``` + +This returns a list of bucket properties and their values +in the form of `property: value`. + +If our `hlls` bucket type has been set properly we should see the following pair in our console output: + +```bash +datatype: hll +``` + +Once we have confirmed the bucket type is properly configured, we can activate the bucket type to be used in Riak KV: + +```bash +riak-admin bucket-type activate hlls +``` + +We can check if activation has been successful by using the same `bucket-type status` command shown above: + +```bash +riak-admin bucket-type status hlls +``` + +After creating and activating our new `hlls` bucket type, we can setup our client to start using the bucket type as detailed in the next section. + +## Client Setup + +First, we need to direct our client to the bucket type/bucket/key +location that contains our counter. + +For this example we'll use the `hlls` bucket type created and activated above and a bucket called `hlls`: + +```erlang +%% Buckets are simply named binaries in the Erlang client. See the +%% examples below for more information +``` + +```java +// In the Java client, a bucket/bucket type combination is specified +// using a Namespace object. To specify bucket, bucket type, and key, +// use a Location object that incorporates the Namespace object, as is +// done below. + +Location hllLocation = + new Location(new Namespace("", ""), ""); +``` + +```python +bucket_type = client.bucket_type('hlls') +bucket = bucket_type.bucket('my_hlls') +hll = bucket.new(key) + +# or + +from riak.datatypes import Hll +hll = Hll(bucket, key) +``` + +```go +// Buckets and bucket types are simply strings in the Go client. + +// See the examples below for more information, or the full example at +// https://github.com/basho/riak-go-client/blob/master/examples/dev/using/data-types/hyperloglog.go + +// We will need the follow imports to run the examples: +import ( + "fmt" + "os" + "time" + + riak "github.com/basho/riak-go-client" + "errors" +) +``` + +```csharp +// In the C# client, buckets are just string parameters to operations. +// See the examples below for more information. +``` + +```javascript +// In the Node.js client, buckets are just string parameters to operations. +// See the examples below for more information. +``` + +```php +$command = (new Command\Builder\FetchHll($riak_client)) + ->buildLocation('', '', 'hlls') + ->build(); +``` + +```ruby +bucket = client.bucket_type('hlls').bucket('my_hlls') +``` + +```curl +curl http://localhost:8098/types//buckets//datatypes/ + +# Note that this differs from the URL structure for non-Data-Type +# requests, which end in /keys/ +``` + + +## Create a HyperLogLog data type + +To create a hyperloglog data structure, you need to specify a bucket/key pair to +hold that hyperloglog. Here is the general syntax for doing so: + +```erlang +HLL = riakc_hll:new(). + +%% Hyperloglogs in the Erlang client are opaque data structures that +%% collect operations as you mutate them. We will associate the data +%% structure with a bucket type, bucket, and key later on. +``` + +```java +// In the Java client, you specify the location of Data Types +// before you perform operations on them: + +Location hllLocation = + new Location(new Namespace("hlls", "hello"), "darkness"); + +// In the Java client, there is no intermediate "empty" hyperloglog data type. +// Hyperloglogs can be created when an element is added to them, as in the examples below. +``` + +```python +bucket_type = client.bucket_type('hlls') +bucket = bucket_type.bucket('my_hlls') +hll = bucket.new(key) + +# or + +from riak.datatypes import Hll +hll = Hll(bucket, key) +``` + +```go +// In the Go client, there is no intermediate "empty" hyperloglog data type. +// Hyperloglogs can be created when an element is added to them, as in the examples below. +``` + +```csharp +// In the C# client, there is no intermediate "empty" hyperloglog data type. +// Hyperloglogs can be created when an element is added to them, as in the examples below. +``` + +```javascript +// In the Node.js client, there is no intermediate "empty" hyperloglog data type. +// Hyperloglogs can be created when an element is added to them, as in the examples below. +``` + +```php +// Note that "hlls" is just an example HLL bucket type name used +// in these examples + +$command = (new Command\Builder\UpdateHll($riak_client)) + ->add('gosabres poked you.') + ->add('phprocks viewed your profile.') + ->add('phprocks started following you.') + ->buildBucket('', 'hlls') + ->build(); + +$response = $command->execute(); +``` + +```ruby +key = "darkness" +hll = Riak::Crdt::HyperLogLog.new(bucket, key) +``` + +```curl +# You cannot create an empty hyperloglog data structure through the HTTP +# interface. +# Hyperloglogs can only be created when an element is added to them, as in the +# examples below. +``` + +Upon creation, our hyperloglog data structure is empty: + +```erlang +HLL. + +%% which will return: +%% {hll,0,[]} +``` + +```java +FetchHll fetch = new FetchHll.Builder(hllLocation) + .build(); +RiakHll hll = client.execute(fetch); +boolean isEmpty = hll.getCardinality() == 0; +``` + +```python +is_empty = hll.value == 0 +``` + +```go +var resp *riak.FetchHllResponse + +builder := riak.NewFetchHllCommandBuilder() +cmd, err := builder.WithBucketType("hlls"). + WithBucket("hello"). + WithKey("darkness"). + Build() +if err != nil { + return err +} +if err = cluster.Execute(cmd); err != nil { + return err +} +if fc, ok := cmd.(*riak.FetchHllCommand); ok { + if fc.Response == nil { + return errors.New("expected non-nil Response") + } + resp = fc.Response +} + +fmt.Println("Hyperloglog cardinality: ", resp.Cardinality) +fmt.Println("Hyperloglog isNotFound: ", resp.IsNotFound) +return nil +``` + +```javascript +var options = { + bucketType: 'hlls', + bucket: 'hello', + key: 'darkness' +}; + +client.fetchHll(options, function (err, rslt) { + if (err) { + throw new Error(err); + } + + if (rslt.notFound) { + logger.info("Not Found"); + } +}); +// Prints "Not Found" to logger.info. +``` + +```csharp + var fetch = new FetchHll.Builder() + .WithBucketType("hlls") + .WithBucket("hello") + .WithKey("darkness") + .Build(); + +RiakResult rslt = client.Execute(fetch); +HllResponse response = fetch.Response; +if (response.NotFound) +{ + Console.WriteLine("Not Found"); +} +// Prints "Not Found" to the console. +``` + +```php +$command = (new Command\Builder\FetchHll($riak_client)) + ->buildLocation('darkness', 'hello', 'hlls') + ->build(); + +$response = $command->execute(); + +$response->getCode() == '404'; +``` + +```ruby +puts hll.cardinality +# Prints "0" +``` + +```curl +curl http://localhost:8098/types/hlls/buckets/hello/datatypes/darkness + +# Response +{"type":"hll","error":"notfound"} +``` + +## Add elements to a HyperLogLog data type + +```erlang +HLL1 = riakc_hll:add_element(<<"Jokes">>, HLL), +RepeatHLL1 = riakc_hll:add_element(<<"Jokes">>, HLL), +HLL2 = riakc_hll:add_elements([<<"Are">>, <<"Better">>, <<"Explained">>], HLL1), + +HLL2. + +%% which will return: +%% {hll,0,[<<"Are">>,<<"Better">>,<<"Explained">>, <<"Jokes">>]} +``` + +```java +HllUpdate hllUpdate = new HllUpdate() + .add("Jokes") + .add("Are") + .addAll(Arrays.asList("Better", "Explained", "Jokes")); + +hllUpdate.getElementAdds(); +// Returns the set of ["Jokes", "Are", "Better", "Explained"] +``` + +```python +bucket_type = client.bucket_type('hlls') +bucket = bucket_type.bucket('my_hlls') +myhll = datatypes.Hll(bucket, 'hll_one') +myhll.add('Jokes') +myhll.add('Are') +myhll.add('Better') +myhll.add('Explained') +myhll.add('Jokes') +myhll.store() +# myhll.value == 4 +``` + +```go +// We will add values in the next example +``` + +```csharp +// We will add values in the next example +``` + +```javascript +// We will add values in the next example +``` + +```php +$command = (new Command\Builder\UpdateHll($riak_client)) + ->add('Jokes') + ->add('Are') + ->add('Better') + ->add('Explained') + ->add('Jokes') + ->buildBucket('my_hlls', 'hlls') + ->build(); + +$response = $command->execute(); +``` + +```ruby +``` + +```curl +curl -XPOST http://localhost:8098/types/hlls/buckets/hello/datatypes/darkness \ + -H "Content-Type: application/json" \ + -d '{"add_all":["my", "old", "friend"]}' +``` + +However, when using a non-HTTP client, the approximate cardinality/value of our +data structure will be 0, locally, until its pushed to the server and then +[fetched](#retrieve-a-hyperloglog-datatype) from the server. + +```erlang +riakc_hll:value(HLL2) == 0. + +%% which will return: +%% true + +Port = 8087, +{ok, Pid} = riakc_pb_socket:start_link("127.0.0.1", Port), +Key = <<"Holy Diver">>, +BucketType = <<"hlls">>, +Bucket = {BucketType, <<"rainbow in the dark">>}, + +ok = riakc_pb_socket:update_type(Pid, Bucket, Key, riakc_hll:to_op(HLL2)). +ok = riakc_pb_socket:update_type(Pid, Bucket, Key, riakc_hll:to_op(RepeatHLL1)). +``` + +```java +// Using hllUpdate and hllLocation from above examples + +UpdateHll update = new UpdateHll.Builder(hllLocation, hllUpdate) + .build(); +client.execute(update); +``` + +```python +bucket_type = client.bucket_type('hlls') +bucket = bucket_type.bucket('my_hlls') +myhll = datatypes.Hll(bucket, 'hll_one') +myhll.add('Jokes') +myhll.add('Are') +myhll.add('Better') +myhll.add('Explained') +myhll.add('Jokes') +myhll.store() +# myhll.value == 4 +``` + +```go +adds := [][]byte{ + []byte("Jokes"), + []byte("Are"), + []byte("Better"), + []byte("Explained"), + []byte("Jokes"), +} + +builder := riak.NewUpdateHllCommandBuilder() +cmd, err := builder.WithBucketType("hlls"). + WithBucket("hello"). + WithKey("darkness"). + WithAdditions(adds...). + Build() +if err != nil { + return err +} + +return cluster.Execute(cmd) +``` + +```javascript +var options = { + bucketType: 'hlls', + bucket: 'hello', + key: 'darkness', + additions: ['Jokes', 'Are', 'Better', 'Explained', 'Jokes'], +}; + +client.updateHll(options, function (err, rslt) { + if (err) { + throw new Error(err); + } +}); +``` + +```csharp +var adds = new HashSet { "Jokes", "Are", "Better", "Explained", "Jokes" }; + +var update = new UpdateHll.Builder(adds) + .WithBucketType("hlls") + .WithBucket("hello") + .WithKey("darkness") + .WithReturnBody(true) + .Build(); + +RiakResult rslt = client.Execute(update); +``` + +```php +$command = (new Command\Builder\UpdateHll($riak_client)) + ->add('Jokes') + ->add('Are') + ->add('Better') + ->add('Explained') + ->add('Jokes') + ->buildLocation('darkness', 'hello', 'hlls') + ->build(); + +$response = $command->execute(); +``` + +```ruby +hll.add('Jokes') +hll.batch do |s| + s.add 'Are' + s.add 'Better' + s.add 'Explained' + s.add 'Jokes' +end +``` + +## Retrieve a HyperLogLog data type + +Now, we can check the approximate count-of (a.k.a. the cardinality of the elements +added to) our hyperloglog data structure: + +```erlang +{ok, HLL3} = riakc_pb_socket:fetch_type(Pid, Bucket, Key), +riakc_hll:value(HLL3) == 4. + +%% which would return: +%% true + +%% We added <<"Jokes">> twice, but, remember, the algorithm only counts the +%% unique elements we've added to the data structure. +``` + +```java +FetchHll hllFetchCmd = new FetchHll.Builder(location).build(); +RiakHll hll = client.execute(hllFetchCmd); +hll.getCardinality(); +// Which returns 4 + +// We added "Jokes" twice, but, remember, the algorithm only counts the +// unique elements we've added to the data structure. +``` + +```python +bucket_type = client.bucket_type('hlls') +bucket = bucket_type.bucket('my_hlls') +myhll = bucket.get('hll_one') +# myhll.value == 4 +``` + +```go +var resp *riak.FetchHllResponse + +builder := riak.NewFetchHllCommandBuilder() +cmd, err := builder.WithBucketType("hlls"). + WithBucket("hello"). + WithKey("darkness"). + Build() +if err != nil { + return err +} +if err = cluster.Execute(cmd); err != nil { + return err +} +if fc, ok := cmd.(*riak.FetchHllCommand); ok { + if fc.Response == nil { + return errors.New("expected non-nil Response") + } + resp = fc.Response +} + +// We added "Jokes" twice, but, remember, the algorithm only counts the +// unique elements we've added to the data structure. +fmt.Println("Hyperloglog cardinality: ", resp.Cardinality) +return nil +``` + +```javascript +var options = { + bucketType: 'hlls', + bucket: 'hello', + key: 'darkness' +}; + +client.fetchHll(options, function (err, rslt) { + if (err) { + throw new Error(err); + } + + if (rslt.notFound) { + logger.info("Not Found"); + } + logger.info("Hyperloglog cardinality is: " + rslt.cardinality); +}); +// Prints "Hyperloglog cardinality is: 4" +// We added "Jokes" twice, but, remember, the algorithm only counts the +// unique elements we've added to the data structure. +``` + +```csharp +var fetch = new FetchHll.Builder() + .WithBucketType("hlls") + .WithBucket("hello") + .WithKey("darkness") + .Build(); + +RiakResult rslt = client.Execute(fetch); +Assert.IsTrue(rslt.IsSuccess, rslt.ErrorMessage); + +HllResponse response = fetch.Response; +if (response.NotFound) +{ + Console.WriteLine("Not Found"); +} +else +{ + Console.WriteLine("Hyperloglog cardinality is: " + response.Cardinality); +} + +// Prints "Hyperloglog cardinality is: 4" +// We added "Jokes" twice, but, remember, the algorithm only counts the +// unique elements we've added to the data structure. +``` + +```php +$command = (new Command\Builder\FetchHll($riak_client)) + ->buildLocation('darkness', 'hello', 'hlls') + ->build(); + +$result = $command->execute(); + +// Note: as though we are in a PHP unit test +$this->assertTrue(is_int($response->getHll()->getData())); +$this->assertEquals(4, $response->getHll()->getData()); + +// We added "Jokes" twice, but, remember, the algorithm only counts the +// unique elements we've added to the data structure. +``` + +```ruby +puts hll.cardinality +# Prints "4" +``` + +```curl +curl http://localhost:8098/types/hlls/buckets/hello/datatypes/darkness + +# Response +{"type":"hll","value":"4"} +``` diff --git a/content/riak/kv/2.2.6/developing/data-types/maps.md b/content/riak/kv/2.2.6/developing/data-types/maps.md new file mode 100644 index 0000000000..c0cca4ffc6 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/data-types/maps.md @@ -0,0 +1,1881 @@ +--- +title_supertext: "Developing with Riak KV" +title: "Data Types: Maps" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Maps" + identifier: "data_types_maps" + weight: 102 + parent: "developing_data_types" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/using/data-types/maps + - /riak-docs/riak/kv/2.2.6/dev/using/data-types/maps + - /riak-docs/riak/2.2.6/dev/data-modeling/data-types/maps + - /riak-docs/riak/kv/2.2.6/dev/data-modeling/data-types/maps +--- + +Maps are the most versatile of the Riak data types because all other data types can be embedded within them, _including maps themselves_. This enables the creation of complex, custom data types from a few basic building blocks. + +Using counters, sets, and maps within maps are similar to working with those types at the bucket level. + +## Set Up a Bucket Type + +> If you've already created and activated a bucket type with the `datatype` parameter set to `map`, skip to the [next section](#client-setup). + +Start by creating a bucket type with the `datatype` parameter set to `map`: + +```bash +riak-admin bucket-type create maps '{"props":{"datatype":"map"}}' +``` + +> **Note** +> +> The `maps` bucket type name provided above is an example and is not required to be `maps`. You are free to name bucket types whatever you like, with the exception of `default`. + +After creating a bucket with a Riak data type, confirm the bucket property configuration associated with that type is correct: + +```bash +riak-admin bucket-type status maps +``` + +This returns a list of bucket properties and their values +in the form of `property: value`. + +If our `map` bucket type has been set properly we should see the following pair in our console output: + +```bash +datatype: map +``` + +Once we have confirmed the bucket type is properly configured, we can activate the bucket type to be used in Riak KV: + +```bash +riak-admin bucket-type activate maps +``` + +We can check if activation has been successful by using the same `bucket-type status` command shown above: + +```bash +riak-admin bucket-type status maps +``` + +After creating and activating our new `maps` bucket type, we can setup our client to start using the bucket type as detailed in the next section. + +## Client Setup + +First, we need to direct our client to the bucket type/bucket/key location that contains our map. + +The syntax for creating a map is analogous to the +syntax for creating other data types: + +```java +// In the Java client, a bucket/bucket type combination is specified +// using a Namespace object. To specify bucket, bucket type, and key, +// use a Location object that incorporates the Namespace object, as is +// done below. + +Location map = + new Location(new Namespace("", ""), ""); +``` + +```ruby +bucket = client.bucket_type('bucket_type_name').bucket('bucket_name') +map = Riak::Crdt::Map.new(bucket, key) +``` + +```php +$location = new \Basho\Riak\Location('key', 'bucket', 'bucket_type'); +``` + +```python +# The client detects the bucket type's datatype and automatically +# returns the right datatype for you, in this case a Map. +map = bucket.new(key) + +# This way is also acceptable: +from riak.datatypes import Map +map = Map(bucket, key) +``` + +```csharp +// https://github.com/basho/riak-dotnet-client/blob/develop/src/RiakClientExamples/Dev/Using/DataTypes.cs + +var builder = new UpdateMap.Builder() + .WithBucketType("") + .WithBucket("") + .WithKey(""); +``` + +```javascript +// Options to pass to the various map methods +var options = { + bucketType: '', + bucket: '', + key: '' +}; +``` + +```erlang +%% Maps in the Erlang client are opaque data structures that +%% collect operations as you mutate them. We will associate the data +%% structure with a bucket type, bucket, and key later on. +``` + +```curl +curl http://localhost:8098/types//buckets//datatypes/ + +# Note that this differs from the URL structure for non-data type requests, +# which end in /keys/ +``` + +## Create a Map + +For this example, say we want to use Riak KV to store information about our company's customers. We'll use the `maps` bucket type created and activated previously and a bucket called `customers`. Each customer's data will be contained in its own key in the `customers` bucket. + +We can create a map for the user Ahmed (`ahmed_info`) using the `maps` bucket type: + +```java +// In the Java client, you specify the location of data types +// before you perform operations on them: + +Location ahmedMap = + new Location(new Namespace("maps", "customers"), "ahmed_info"); +``` + +```ruby +customers = client.bucket_type('maps').bucket('customers') +map = Riak::Crdt::Map.new(customers, 'ahmed_info') + +# Alternatively, the Ruby client enables you to set a bucket type as being +# globally associated with a Riak data type. The following would set all +# map buckets to use the maps bucket type: + +Riak::Crdt::DEFAULT_BUCKET_TYPES[:map] = 'maps' + +# This would enable us to create our map without specifying a bucket type: + +customers = client.bucket('customers') +map = Riak::Crdt::Map.new(customers, 'ahmed_info') +``` + +```php +$location = new \Basho\Riak\Location('ahmed_info', 'customers', 'maps'); +``` + +```python +customers = client.bucket_type('map_bucket').bucket('customers') +map = customers.net('ahmed_info') +``` + +```csharp +// https://github.com/basho/riak-dotnet-client/blob/develop/src/RiakClientExamples/Dev/Using/DataTypes.cs + +var builder = new UpdateMap.Builder() + .WithBucketType("maps") + .WithBucket("customers") + .WithKey("ahmed_info"); +``` + +```javascript +var options = { + bucketType: 'maps', + bucket: 'customers', + key: 'ahmed_info' +}; +``` + +```erlang +Map = riakc_map:new(). + +%% Maps in the Erlang client are opaque data structures that +%% collect operations as you mutate them. We will associate the data +%% structure with a bucket type, bucket, and key later on. +``` + +```curl +# You cannot create an empty map through the HTTP interface. Maps can only +# be created when a field is added to them, as in the examples below. +``` + +## Registers + +Registers are essentially named binaries (like strings). Any binary +value can act as the value of a register. Like flags, registers cannot +be used on their own and must be embedded in Riak maps. + +### Registers Within Maps + +Continuing with our previous `customers` example, let's store some information in our map. + +The first piece of information we want to store in our map is Ahmed's name and +phone number, both of which are best stored as registers: + +```java +// Using our "ahmedMap" location from above: + +RegisterUpdate ru1 = new RegisterUpdate("Ahmed"); +RegisterUpdate ru2 = new RegisterUpdate("5551234567"); +MapUpdate mu = new MapUpdate() + .update("first_name", ru1) + .update("phone_number", ru2); +UpdateMap update = new UpdateMap.Builder(ahmedMap, mu) + .build(); +client.execute(update); +``` + +```ruby +# The Ruby client enables you to batch operations together if you're +# performing them on one data type. + +map.batch do |m| + m.registers['first_name'] = 'Ahmed' + m.registers['phone_number'] = '5551234567' +end + +# Integers need to be stored as strings and then converted back when +# the data is retrieved. The following would work as well: +map.registers['phone_number'] = 5551234567.to_s +``` + +```php +(new \Basho\Riak\Command\Builder\UpdateMap($riak)) + ->updateRegister('first_name', 'Ahmed') + ->updateRegister('phone_number', '5551234567') + ->atLocation($location) + ->build() + ->execute(); +``` + +```python +map.registers['first_name'].assign('Ahmed') +map.registers['phone_number'].assign('5551234567') + +# Integers need to be stored as strings and then converted back when the +# data is retrieved. The following would work as well: +map.registers['phone_number'].assign(str(5551234567)) + +map.store() +``` + +```csharp +// https://github.com/basho/riak-dotnet-client/blob/develop/src/RiakClientExamples/Dev/Using/DataTypes.cs + +var builder = new UpdateMap.Builder() + .WithBucketType("maps") + .WithBucket("customers") + .WithKey("ahmed_info"); + +var mapOperation = new UpdateMap.MapOperation(); + +// Ahmed's first name +mapOperation.SetRegister("first_name", "Ahmed"); + +// Ahmed's phone number +mapOperation.SetRegister("phone_number", "5551234567"); + +builder.WithMapOperation(mapOperation); + +UpdateMap cmd = builder.Build(); +RiakResult rslt = client.Execute(cmd); +MapResponse response = cmd.Response; +PrintMap(response.Value); +// Output as JSON: +// Map: {"Counters":{},"Sets":{},"Registers":{"first_name":"Ahmed","phone_number":"5551234567"},"Flags":{},"Maps":{}} +``` + +```javascript +var mapOp = new Riak.Commands.CRDT.UpdateMap.MapOperation(); +mapOp.setRegister('first_name', new Buffer('Ahmed')); +mapOp.setRegister('phone_number', new Buffer('5551234567')); + +var options = { + bucketType: 'maps', + bucket: 'customers', + key: 'ahmed_info', + op: mapOp +}; + +client.updateMap(options, function (err, rslt) { + if (err) { + throw new Error(err); + } +}); +``` + +```erlang +Map1 = riakc_map:update({<<"first_name">>, register}, + fun(R) -> riakc_register:set(<<"Ahmed">>, R) end, + Map), +Map2 = riakc_map:update({<<"phone_number">>, register}, + fun(R) -> riakc_register:set(<<"5551234567">>, R) end, + Map1). +``` + +```curl +# Updates can be performed all at once. The following will create two new +# registers in the map and also set the value of those registers to the +# desired values + +curl -XPOST http://localhost:8098/types/maps/buckets/customers/datatypes/ahmed_info \ + -H "Content-Type: application/json" \ + -d ' + { + "update": { + "first_name_register": "Ahmed", + "phone_number_register": "5551234567" + } + }' +``` + +If a register did not previously exist, Riak KV will create that register for you. + +## Flags + +Flags behave much like Boolean values, except that instead of `true` or +`false` flags have the values `enable` or `disable`. + +Flags cannot be used on their own, i.e. a flag cannot be stored in a bucket/key by itself. Instead, flags can only be stored within maps. + +To disable an existing flag, you have to read it or provide [a context](../#data-types-and-context). + +### Flags Within Maps + +Now let's say that we add an Enterprise plan to our pricing model. We'll +create an `enterprise_customer` flag to track whether Ahmed has signed +up for the new plan. He hasn't yet, so we'll set it to `false`: + +```java +// Using our "ahmedMap" location from above: + +MapUpdate mu = new MapUpdate() + .update("enterprise_customer", new FlagUpdate(false)); +UpdateMap update = new UpdateMap.Builder(ahmedMap, mu) + .build(); +client.execute(update); +``` + +```ruby +map.flags['enterprise_customer'] = false +``` + +```php +(new \Basho\Riak\Command\Builder\UpdateMap($riak)) + ->updateFlag('enterprise_customer', false) + ->atLocation($location) + ->build() + ->execute(); +``` + +```python +map.flags['enterprise_customer'].disable() +map.store() +``` + + +```csharp +// https://github.com/basho/riak-dotnet-client/blob/develop/src/RiakClientExamples/Dev/Using/DataTypes.cs + +// Using our builder from above: + +mapOperation = new UpdateMap.MapOperation(); +mapOperation.SetFlag("enterprise_customer", false); + +builder.WithMapOperation(mapOperation); +cmd = builder.Build(); +rslt = client.Execute(cmd); + +response = cmd.Response; + +// response.Value as JSON: +// Map: {"Counters":{},"Sets":{}, + "Registers":{"first_name":"Ahmed","phone_number":"5551234567"}, + "Flags":{"enterprise_customer":false},"Maps":{}} +``` + +```javascript +var mapOp = new Riak.Commands.CRDT.UpdateMap.MapOperation(); +mapOp.setFlag('enterprise_customer', false); + +var options = { + bucketType: 'maps', + bucket: 'customers', + key: 'ahmed_info', + op: mapOp +}; + +client.updateMap(options, function (err, rslt) { + if (err) { + throw new Error(err); + } +}); +``` + +```erlang +Map4 = riakc_map:update({<<"enterprise_customer">>, flag}, + fun(F) -> riakc_flag:disable(F) end, + Map3). +``` + +```curl +curl http://localhost:8098/types/maps/buckets/customers/datatypes/ahmed_info + +# Response +{"type":"map","value":{"first_name_register":"Ahmed","phone_number_register":"5551234567"},"context":"g2wAAAABaAJtAAAADCMJ/vn2jOEXAAAAAWEBag=="} + +curl -XPOST http://localhost:8098/types/maps/buckets/customers/datatypes/ahmed_info \ + -H "Content-Type: application/json" \ + -d ' + { + "update": { + "enterprise_customer_flag": "disable" + }, + "context" : "g2wAAAABaAJtAAAADCMJ/vn2jOEXAAAAAWEBag==" + }' +``` + +We can retrieve the value of that flag at any time: + +```java +// Using our "ahmedMap" location from above: + +FetchMap fetch = new FetchMap.Builder(ahmedMap).build(); +FetchMap.Response response = client.execute(fetch); +RiakMap map = response.getDatatype(); +System.out.println(map.getFlag("enterprise_customer").view()); +``` + +```ruby +map.flags['enterprise_customer'] + +# false +``` + +```php +$map = (new \Basho\Riak\Command\Builder\FetchMap($riak)) + ->atLocation($location) + ->build() + ->execute() + ->getMap(); + +echo $map->getFlag('enterprise_customer'); // false +``` + +```python +map.reload().flags['enterprise_customer'].value +``` + +```csharp +// https://github.com/basho/riak-dotnet-client/blob/develop/src/RiakClientExamples/Dev/Using/DataTypes.cs + +Map ahmedMap = response.Value; +ahmedMap.Flags["enterprise_customer"] +``` + +```javascript +var options = { + bucketType: 'maps', + bucket: 'customers', + key: 'ahmed_info' +}; + +client.fetchMap(options, function (err, rslt) { + if (err) { + throw new Error(err); + } + + console.log("fetched map: %s", JSON.stringify(rslt)); +}); +``` + +```erlang +%% The value fetched from Riak is always immutable, whereas the "dirty +%% value" takes into account local modifications that have not been +%% sent to the server. + +riakc_map:dirty_value(Map4). +``` + +```curl +curl http://localhost:8098/types/maps/buckets/customers/datatypes/ahmed_info +``` + +## Counters Within Maps + +We also want to know how many times Ahmed has visited our website. We'll +use a `page_visits` counter for that and run the following operation +when Ahmed visits our page for the first time: + +```java +// Using our "ahmedMap" location from above: + +MapUpdate mu = new MapUpdate() + .update("page_visits", cu); +UpdateMap update = new UpdateMap.Builder(ahmedMap, new CounterUpdate(1)) + .build(); +client.execute(update); +``` + +```ruby +map.counters['page_visits'].increment + +# This operation may return false even if successful +``` + +```php +$updateCounter = (new \Basho\Riak\Command\Builder\IncrementCounter($riak)) + ->withIncrement(1); + +(new \Basho\Riak\Command\Builder\UpdateMap($riak)) + ->updateCounter('page_visits', $updateCounter) + ->atLocation($location) + ->build() + ->execute(); +``` + +```python +map.counters['page_visits'].increment() +map.store() +``` + +```csharp +// https://github.com/basho/riak-dotnet-client/blob/develop/src/RiakClientExamples/Dev/Using/DataTypes.cs + +var mapOperation = new UpdateMap.MapOperation(); +mapOperation.IncrementCounter("page_visits", 1); + +builder.WithMapOperation(mapOperation); +UpdateMap cmd = builder.Build(); +RiakResult rslt = client.Execute(cmd); + +MapResponse response = cmd.Response; +// Map: {"Counters":{"page_visits":3}, + "Sets":{}, + "Registers":{"first_name":"Ahmed","phone_number":"5551234567"}, + "Flags":{"enterprise_customer":false}, + "Maps":{}} +``` + +```javascript +var mapOp = new Riak.Commands.CRDT.UpdateMap.MapOperation(); +mapOp.incrementCounter('page_visits', 1); + +var options = { + bucketType: 'maps', + bucket: 'customers', + key: 'ahmed_info', + op: mapOp +}; + +client.updateMap(options, function (err, rslt) { + if (err) { + throw new Error(err); + } +}); +``` + +```erlang +Map3 = riakc_map:update({<<"page_visits">>, counter}, + fun(C) -> riakc_counter:increment(1, C) end, + Map2). +``` + +```curl +# The following will create a new counter and increment it by 1 + +curl -XPOST http://localhost:8098/types/maps/buckets/customers/datatypes/ahmed_info \ + -H "Content-Type: application/json" \ + -d ' + { + "update": { + "page_visits_counter": 1 + } + }' +``` + +Even though the `page_visits` counter did not exist previously, the +above operation will create it (with a default starting point of 0) and +the increment operation will bump the counter up to 1. + +## Sets Within Maps + +We'd also like to know what Ahmed's interests are so that we can better +design a user experience for him. Through his purchasing decisions, we +find out that Ahmed likes robots, opera, and motorcycles. We'll store +that information in a set inside of our map: + +```java +// Using our "ahmedMap" location from above: + +SetUpdate su = new SetUpdate() + .add("robots") + .add("opera") + .add("motorcycles"); +MapUpdate mu = new MapUpdate() + .update("interests", su); +UpdateMap update = new UpdateMap.Builder(ahmedMap, mu) + .build(); +client.execute(update); +``` + +```ruby +map.batch do |m| + %{ robots opera motorcycles }.each do |interest| + m.sets['interests'].add(interest) + end +end +``` + +```php +$updateSet = (new \Basho\Riak\Command\Builder\UpdateSet($riak)) + ->add('robots') + ->add('opera') + ->add('motorcycles'); + +(new \Basho\Riak\Command\Builder\UpdateMap($riak)) + ->updateSet('interests', $updateSet) + ->atLocation($location) + ->build() + ->execute(); +``` + +```python +for interest in ['robots', 'opera', 'motorcycles']: + map.sets['interests'].add(interest) +map.store() +``` + +```csharp +// https://github.com/basho/riak-dotnet-client/blob/develop/src/RiakClientExamples/Dev/Using/DataTypes.cs + +var interestsAdds = new[] { "robots", "opera", "motorcycles" }; + +var mapOperation = new UpdateMap.MapOperation(); +mapOperation.AddToSet("interests", interestsAdds); + +builder.WithMapOperation(mapOperation); +UpdateMap cmd = builder.Build(); +RiakResult rslt = client.Execute(cmd); +MapResponse response = cmd.Response; + +// Map: {"Counters":{"page_visits":3}, + "Sets":{"interests":["motorcycles","opera","robots"]}, + "Registers":{"first_name":"Ahmed","phone_number":"5551234567"}, + "Flags":{"enterprise_customer":false}, + "Maps":{}} +``` + +```javascript +var mapOp = new Riak.Commands.CRDT.UpdateMap.MapOperation(); +mapOp.addToSet('interests', 'robots'); +mapOp.addToSet('interests', 'opera'); +mapOp.addToSet('interests', 'motorcycles'); + +var options = { + bucketType: 'maps', + bucket: 'customers', + key: 'ahmed_info', + op: mapOp +}; + +client.updateMap(options, function (err, rslt) { + if (err) { + throw new Error(err); + } +}); +``` + +```erlang +Map4 = riakc_map:update({<<"interests">>, set}, + fun(S) -> riakc_set:add_element(<<"robots">>, S) end, Map3), +Map5 = riakc_map:update({<<"interests">>, set}, + fun(S) -> riakc_set:add_element(<<"opera">>, S) end, + Map4), +Map6 = riakc_map:update({<<"interests">>, set}, + fun(S) -> riakc_set:add_element(<<"motorcycles">>, S) end, + Map4). +``` + +```curl +curl -XPOST http://localhost:8098/types/maps/buckets/customers/datatypes/ahmed_info \ + -H "Content-Type: application/json" \ + -d ' + { + "update": { + "interests_set": { + "add_all": [ + "robots", + "opera", + "motorcycles" + ] + } + } + }' +``` + +We can then verify that the `interests` set includes these three +interests: + +```java +// Using our "ahmedMap" location from above: + +FetchMap fetch = new FetchMap.Builder(ahmedMap) + .build(); +FetchMap.Response response = client.execute(fetch); +RiakMap map = response.getDatatype(); +RiakSet interestSet = map.getSet("interests"); +Set interests = interestSet.view(); +System.out.println(interests.contains(BinaryValue.create("robots"))); + +// Checking for "opera" and "motorcycles" works the same way +``` + +```ruby +map.batch do |m| + %w{ robots opera motorcycles }.each do |interest| + m.sets['interests'].include? interest + end +end + +# This will return three Boolean values +``` + +```php +$map = (new \Basho\Riak\Command\Builder\FetchMap($riak)) + ->atLocation($location) + ->build() + ->execute() + ->getMap(); + +$sets = $map->getSet('interests'); +var_dump($sets->getData()); +``` + +```python +reloaded_map = map.reload() +for interest in ['robots', 'opera', 'motorcycles']: + interest in reloaded_map.sets['interests'].value +``` + +```csharp +// https://github.com/basho/riak-dotnet-client/blob/develop/src/RiakClientExamples/Dev/Using/DataTypes.cs + +Map ahmedMap = response.Value; + +// All of the following return true: +ahmedMap.Sets.GetValue("interests").Contains("robots"); +ahmedMap.Sets.GetValue("interests").Contains("opera"); +ahmedMap.Sets.GetValue("interests").Contains("motorcycles"); +``` + +```javascript +var options = { + bucketType: 'maps', + bucket: 'customers', + key: 'ahmed_info' +}; + +client.fetchMap(options, function (err, rslt) { + if (err) { + throw new Error(err); + } + + assert(rslt.map.sets['interests'].indexOf('robots') !== -1); +}); +``` + +```erlang +riakc_map:dirty_value(Map6). +``` + +```curl +curl http://localhost:8098/types/maps/buckets/customers/datatypes/ahmed_info?include_context=false +``` + +We learn from a recent purchasing decision that Ahmed actually doesn't +seem to like opera. He's much more keen on indie pop. Let's change the +`interests` set to reflect that: + +```java +// Using our "ahmedMap" location from above: + +SetUpdate su = new SetUpdate() + .remove("opera") + .add("indie pop"); +MapUpdate mu = new MapUpdate() + .update("interests", su); +UpdateMap update = new UpdateMap.Builder(ahmedMap, mu) + .build(); +client.execute(update); +``` + +```ruby +map.batch do |m| + m.sets['interests'].remove('opera') + m.sets['interests'].add('indie pop') +end +``` + +```php +$updateSet = (new \Basho\Riak\Command\Builder\UpdateSet($riak)) + ->add('indie pop') + ->remove('opera'); + +(new \Basho\Riak\Command\Builder\UpdateMap($riak)) + ->updateSet('interests', $updateSet) + ->atLocation($location) + ->withContext($map->getContext()) + ->build() + ->execute(); +``` + +```python +map.sets['interests'].discard('opera') +map.sets['interests'].add('indie pop') +map.store() +``` + +```csharp +// https://github.com/basho/riak-dotnet-client/blob/develop/src/RiakClientExamples/Dev/Using/DataTypes.cs + +var mapOperation = new UpdateMap.MapOperation(); +mapOperation.AddToSet("interests", "indie pop"); +mapOperation.RemoveFromSet("interests", "opera"); + +builder + .WithMapOperation(mapOperation) + .WithContext(response.Context); + +UpdateMap cmd = builder.Build(); +RiakResult rslt = client.Execute(cmd); + +MapResponse response = cmd.Response; +Map ahmedMap = response.Value; + +// This is false +ahmedMap.Sets.GetValue("interests").Contains("opera"); + +// These are true +ahmedMap.Sets.GetValue("interests").Contains("indie pop"); +ahmedMap.Sets.GetValue("interests").Contains("robots"); +ahmedMap.Sets.GetValue("interests").Contains("motorcycles"); +``` + +```javascript +var options = { + bucketType: 'maps', + bucket: 'customers', + key: 'ahmed_info' +}; + +client.fetchMap(options, function (err, rslt) { + if (err) { + throw new Error(err); + } + + var mapOp = new Riak.Commands.CRDT.UpdateMap.MapOperation(); + mapOp.removeFromSet('interests', 'opera'); + mapOp.addToSet('interests', 'indie pop'); + + options.context = rslt.context; + options.op = mapOp; + + client.updateMap(options, function (err, rslt) { + if (err) { + throw new Error(err); + } + }); +}); +``` + +```erlang +Map7 = riakc_map:update({<<"interests">>, set}, + fun(S) -> riakc_set:del_element(<<"opera">>, S) end, Map6), +Map8 = riakc_map:update({<<"interests">>, set}, + fun(S) -> riakc_set:add_element(<<"indie pop">>, S) end, + Map7). +``` + +```curl +curl http://localhost:8098/types/maps/buckets/customers/datatypes/ahmed_info + +# Response +{"type":"map","value":{"enterprise_customer_flag":false,"first_name_register":"Ahmed","interests_set":["motorcycles","opera","robots"],"page_visits_counter":1,"phone_number_register":"5551234567"},"context":"g2wAAAABaAJtAAAADCMJ/vn2jOEXAAAAAWEEag=="} + +curl -XPOST http://localhost:8098/types/maps/buckets/customers/datatypes/ahmed_info \ + -H "Content-Type: application/json" \ + -d ' + { + "update": { + "interests_set": { + "remove": "opera", + "add": "indie pop" + } + }, + "context" : "g2wAAAABaAJtAAAADCMJ/vn2jOEXAAAAAWEEag==" + } + ' +``` + +## Maps Within Maps + +We've stored a wide of variety of information---of a wide variety of +types---within the `ahmed_info` map thus far, but we have yet to explore +recursively storing maps within maps (which can be nested as deeply as +you wish). + +Our company is doing well and we have lots of useful information about +Ahmed, but now we want to store information about Ahmed's contacts as +well. We'll start with storing some information about Ahmed's colleague +Annika inside of a map called `annika_info`. + +First, we'll store Annika's first name, last name, and phone number in +registers: + +```java +// Using our "ahmedMap" location from above: + +RegisterUpdate ru1 = new RegisterUpdate("Annika"); +RegisterUpdate ru2 = new RegisterUpdate("Weiss"); +RegisterUpdate ru3 = new RegisterUpdate("5559876543"); + +MapUpdate annikaUpdate = new MapUpdate() + .update("first_name", ru1) + .update("last_name", ru2) + .update("phone_number", ru3); +MapUpdate ahmedUpdate = new MapUpdate() + .update("annika_info", annikaUpdate); +UpdateMap update = new UpdateMap.Builder(ahmedMap, ahmedUpdate) + .build(); +client.execute(update); +``` + +```ruby +map.maps['annika_info'].batch do |m| + m.registers['first_name'] = 'Annika' + m.registers['last_name'] = 'Weiss' + m.registers['phone_number'] = 5559876543.to_s +end +``` + +```php +$annikaMap = (new \Basho\Riak\Command\Builder\UpdateMap($riak)) + ->updateRegister('first_name', 'Annika') + ->updateRegister('last_name', 'Weiss') + ->updateRegister('phone_number', '5559876543'); + +$response = (new \Basho\Riak\Command\Builder\UpdateMap($riak)) + ->updateMap('annika_info', $annikaMap) + ->atLocation($location) + ->withParameter('returnbody', 'true') + ->build() + ->execute(); +``` + +```python +map.maps['annika_info'].registers['first_name'].assign('Annika') +map.maps['annika_info'].registers['last_name'].assign('Weiss') +map.maps['annika_info'].registers['phone_number'].assign(str(5559876543)) +map.store() +``` + +```csharp +// https://github.com/basho/riak-dotnet-client/blob/develop/src/RiakClientExamples/Dev/Using/DataTypes.cs + +var mapOperation = new UpdateMap.MapOperation(); + +var annikaInfoOperation = mapOperation.Map("annika_info"); +annikaInfoOperation.SetRegister("first_name", "Annika"); +annikaInfoOperation.SetRegister("last_name", "Weiss"); +annikaInfoOperation.SetRegister("phone_number", "5559876543"); + +builder.WithMapOperation(mapOperation); +UpdateMap cmd = builder.Build(); +client.Execute(cmd); +``` + +```javascript +var options = { + bucketType: 'maps', + bucket: 'customers', + key: 'ahmed_info' +}; + +var mapOp = new Riak.Commands.CRDT.UpdateMap.MapOperation(); +mapOp.map('annika_info') + .setRegister('first_name', 'Annika') + .setRegister('last_name', 'Weiss') + .setRegister('phone_number', '5559876543'); + +options.op = mapOp; + +client.updateMap(options, function (err, rslt) { + if (err) { + throw new Error(err); + } +}); +``` + +```erlang +Map12 = riakc_map:update( + {<<"annika_info">>, map}, + fun(M) -> riakc_map:update( + {<<"first_name">>, register}, + fun(R) -> riakc_register:set(<<"Annika">>, R) end, M) end, + Map11), +Map13 = riakc_map:update( + {<<"annika_info">>, map}, + fun(M) -> riakc_map:update( + {<<"last_name">>, register}, + fun(R) -> riakc_register:set(<<"Weiss">>, R) end, M) end, + Map12), +Map14 = riakc_map:update( + {<<"annika_info">>, map}, + fun(M) -> riakc_map:update( + {<<"phone_number">>, register}, + fun(R) -> riakc_register:set(<<"5559876543">>, R) end, M) end, + Map13). +``` + +```curl +curl -XPOST http://localhost:8098/types/maps/buckets/customers/datatypes/ahmed_info \ + -H "Content-Type: application/json" \ + -d ' + { + "update": { + "annika_info_map": { + "update": { + "first_name_register": "Annika", + "last_name_register": "Weiss", + "phone_number_register": "5559876543" + } + } + } + } + ' +``` + +The value of a register in a map can be obtained without a special +method: + +```java +// Using our "ahmedMap" location from above: + +FetchMap fetch = new FetchMap.Builder(ahmedMap).build(); +FetchMap.Response response = client.execute(fetch); +String annikaFirstName = response.getDatatype() + .getMap("annika_info") + .getRegister("first_name") + .view() + .toString(); +``` + +```ruby +map.maps['annika_info'].registers['first_name'] + +# "Annika" +``` + +```php +# with param 'returnbody' = 'true', we can fetch the map from our last response +$map->getMap(); + +echo $map->getMap('annika_info')->getRegister('first_name'); // Annika +``` + +```python +map.reload().maps['annika_info'].registers['first_name'].value +``` + +```csharp +// https://github.com/basho/riak-dotnet-client/blob/develop/src/RiakClientExamples/Dev/Using/DataTypes.cs + +ahmedMap = response.Value; +ahmedMap.Maps["annika_info"].Registers.GetValue("first_name"); +``` + +```javascript +var options = { + bucketType: 'maps', + bucket: 'customers', + key: 'ahmed_info' +}; + +client.fetchMap(options, function (err, rslt) { + if (err) { + throw new Error(err); + } + + var annikaFirstName = + rslt.map.maps['annika_info'].registers['first_name'].toString('utf8'); +}); +``` + +```erlang +riakc_map:dirty_value(Map14). +``` + +```curl +# Specific values for fields inside of maps (or maps within maps, for that +# matter), cannot be obtained directly through the HTTP interface. +``` + +Registers can also be removed: + +```java +// This example uses our "ahmedMap" location from above. Operations that +// remove fields from maps require that you first fetch the opaque context +// attached to the map and then include the context in the update operation: + +FetchMap fetch = new FetchMap.Builder(ahmedMap) + .build(); +FetchMap.Response response = client.execute(fetch); +Context ctx = response.getContext(); +MapUpdate annikaUpdate = new MapUpdate() + .removeRegister("first_name"); +MapUpdate ahmedUpdate = new MapUpdate() + .update("annika_info", annikaUpdate); +UpdateMap update = new UpdateMap.Builder(ahmedMap, ahmedUpdate) + .withContext(ctx) + .build(); +client.execute(update); +``` + +```ruby +map.maps['annika_info'].registers.remove('first_name') +``` + +```php +$annikaMap = (new \Basho\Riak\Command\Builder\UpdateMap($riak)) + ->removeRegister('first_name'); + +(new \Basho\Riak\Command\Builder\UpdateMap($riak)) + ->updateMap('annika_info', $annikaMap) + ->atLocation($location) + ->withContext($map->getContext()) + ->build() + ->execute(); +``` + +```python +del map.maps['annika_info'].registers['first_name'] +map.store() +``` + +```csharp +// https://github.com/basho/riak-dotnet-client/blob/develop/src/RiakClientExamples/Dev/Using/DataTypes.cs + +var mapOperation = new UpdateMap.MapOperation(); +mapOperation.Map("annika_info").RemoveRegister("first_name"); + +// Note: using Context from last response +builder + .WithMapOperation(mapOperation) + .WithContext(response.Context); + +UpdateMap cmd = builder.Build(); +client.Execute(cmd); +``` + +```javascript +var options = { + bucketType: 'maps', + bucket: 'customers', + key: 'ahmed_info' +}; + +client.fetchMap(options, function (err, rslt) { + if (err) { + throw new Error(err); + } + + var mapOp = new Riak.Commands.CRDT.UpdateMap.MapOperation(); + mapOp.map('annika_info').removeRegister('first_name'); + + var options = { + bucketType: 'maps', + bucket: 'customers', + key: 'ahmed_info', + op: mapOp, + context: rslt.context, + }; + + client.updateMap(options, function (err, rslt) { + if (err) { + throw new Error(err); + } + }); +``` + +```erlang +Map15 = riakc_map:update({<<"annika_info">>, map}, + fun(M) -> riakc_map:erase({<<"phone_number">>, register}, M) end, + Map14). +``` + +```curl +curl http://localhost:8098/types/maps/buckets/customers/datatypes/ahmed_info + +# Response +{"type":"map","value":{"annika_info_map":{"first_name_register":"Annika","last_name_register":"Weiss","phone_number_register":"5559876543"},"enterprise_customer_flag":false,"first_name_register":"Ahmed","interests_set":["indie pop","motorcycles","robots"],"page_visits_counter":1,"phone_number_register":"5551234567"},"context":"g2wAAAABaAJtAAAADCMJ/vn2jOEXAAAAAWEGag=="} + +curl -XPOST http://localhost:8098/types/maps/buckets/customers/datatypes/ahmed_info \ + -H "Content-Type: application/json" \ + -d ' + { + "update": { + "annika_info_map": { + "remove": ["phone_number_register"] + } + }, + "context" : "g2wAAAABaAJtAAAADCMJ/vn2jOEXAAAAAWEGag==" + } + ' +``` + +Now, we'll store whether Annika is subscribed to a variety of plans +within the company as well: + +```java +// Using our "ahmedMap" location from above: + +FetchMap fetch = new FetchMap.Builder(ahmedMap).build(); +FetchMap.Response response = client.execute(fetch); +Context ctx = response.getContext(); +MapUpdate annikaUpdate = new MapUpdate() + .update("enterprise_plan", new FlagUpdate((false)) + .update("family_plan", new FlagUpdate(false)) + .update("free_plan", new FlagUpdate(true)); +MapUpdate ahmedUpdate = new MapUpdate() + .update("annika_info", annikaUpdate); +UpdateMap update = new UpdateMap.Builder(ahmedMap, ahmedUpdate) + .withContext(ctx) + .build(); +client.execute(update); +``` + +```ruby +map.maps['annika_info'].batch do |m| + m.flags['enterprise_plan'] = false + m.flags['family_plan'] = false + m.flags['free_plan'] = true +end +``` + +```php +$annikaMap = (new \Basho\Riak\Command\Builder\UpdateMap($riak)) + ->updateFlag('enterprise_plan', false) + ->updateFlag('family_plan', false) + ->updateFlag('free_plan', true); + +$response = (new \Basho\Riak\Command\Builder\UpdateMap($riak)) + ->updateMap('annika_info', $annikaMap) + ->atLocation($location) + ->withParameter('returnbody', 'true') + ->build() + ->execute(); +``` + +```python +map.maps['annika_info'].flags['enterprise_plan'].disable() +map.maps['annika_info'].flags['family_plan'].disable() +map.maps['annika_info'].flags['free_plan'].enable() +map.store() +``` + +```csharp +// https://github.com/basho/riak-dotnet-client/blob/develop/src/RiakClientExamples/Dev/Using/DataTypes.cs + +var mapOperation = new UpdateMap.MapOperation(); +mapOperation.Map("annika_info") + .SetFlag("enterprise_plan", false) + .SetFlag("family_plan", false) + .SetFlag("free_plan", true); + +builder.WithMapOperation(mapOperation); + +MapUpdate cmd = builder.Build(); +client.Execute(cmd); +``` + +```javascript +var options = { + bucketType: 'maps', + bucket: 'customers', + key: 'ahmed_info' +}; + +client.fetchMap(options, function (err, rslt) { + if (err) { + throw new Error(err); + } + + var mapOp = new Riak.Commands.CRDT.UpdateMap.MapOperation(); + var annika_map = mapOp.map('annika_info'); + annika_map.setFlag('enterprise_plan', false); + annika_map.setFlag('family_plan', false); + annika_map.setFlag('free_plan', true); + + var options = { + bucketType: 'maps', + bucket: 'customers', + key: 'ahmed_info', + op: mapOp, + context: rslt.context, + }; + + client.updateMap(options, function (err, rslt) { + if (err) { + throw new Error(err); + } + }); +}); +``` + +```erlang +Map16 = riakc_map:update( + {<<"annika_info">>, map}, + fun(M) -> riakc_map:update( + {<<"enterprise_plan">>, flag}, + fun(F) -> riakc_flag:disable(F) end, + M) end, + Map15), +Map17 = riakc_map:update( + {<<"annika_info">>, map}, + fun(M) -> riakc_map:update( + {<<"family_plan">>, flag}, + fun(F) -> riakc_flag:disable(F) end, + M) end, + Map16), +Map18 = riakc_map:update( + {<<"annika_info">>, map}, + fun(M) -> riakc_map:update( + {<<"free_plan">>, flag}, + fun(F) -> riakc_flag:enable(F) end, + M) end, + Map17). +``` + +```curl +curl http://localhost:8098/types/maps/buckets/customers/datatypes/ahmed_info + +# Response +{"type":"map","value":{"annika_info_map":{"first_name_register":"Annika","last_name_register":"Weiss"},"enterprise_customer_flag":false,"first_name_register":"Ahmed","interests_set":["indie pop","motorcycles","robots"],"page_visits_counter":1,"phone_number_register":"5551234567"},"context":"g2wAAAABaAJtAAAADCMJ/vn2jOEXAAAAAWEHag=="} + +curl -XPOST http://localhost:8098/types/maps/buckets/customers/datatypes/ahmed_info \ + -H "Content-Type: application/json" \ + -d ' + { + "update": { + "annika_info_map": { + "update": { + "enterprise_plan_flag": "disable", + "family_plan_flag": "disable", + "free_plan_flag": "enable" + } + } + }, + "context" : "g2wAAAABaAJtAAAADCMJ/vn2jOEXAAAAAWEHag==" + } + ' +``` + +The value of a flag can be retrieved at any time: + +```java +// Using our "ahmedMap" location from above: + +FetchMap fetch = new FetchMap.Builder(ahmedMap).build(); +FetchMap.Response response = client.execute(fetch); +boolean enterprisePlan = response.getDatatype() + .getMap("annika_info") + .getFlag("enterprise_plan") + .view(); +``` + +```ruby +map.maps['annika_info'].flags['enterprise_plan'] + +# false +``` + +```php +# with param 'returnbody' = 'true', we can fetch the map from our last response +$map->getMap(); + +echo $map->getMap('annika_info')->getFlag('enterprise_plan'); // false +``` + +```python +map.reload().maps['annika_info'].flags['enterprise_plan'].value +``` + +```csharp +// https://github.com/basho/riak-dotnet-client/blob/develop/src/RiakClientExamples/Dev/Using/DataTypes.cs + +ahmedMap = response.Value; +ahmedMap.Maps["annika_info"].Flags["enterprise_plan"]; +``` + +```javascript +var options = { + bucketType: 'maps', + bucket: 'customers', + key: 'ahmed_info' +}; + +client.fetchMap(options, function (err, rslt) { + if (err) { + throw new Error(err); + } + + var enterprisePlan = + rslt.map.maps.annika_info.flags.enterprise_plan; +}); +``` + +```erlang +riakc_map:dirty_value(Map18). +``` + +```curl +# Specific values for fields inside of maps (or maps within maps, for that +# matter), cannot be obtained directly through the HTTP interface. +``` + +It's also important to track the number of purchases that Annika has +made with our company. Annika just made her first widget purchase: + +```java +// Using our "ahmedMap" location from above: + +MapUpdate annikaUpdate = new MapUpdate() + .update("widget_purchases", new CounterUpdate(1)); +MapUpdate ahmedUpdate = new MapUpdate() + .update("annika_info", annikaUpdate); +UpdateMap update = new UpdateMap.Builder(ahmedMap, ahmedUpdate) + .build(); +client.execute(update); +``` + +```ruby +map.maps['annika_info'].counters['widget_purchases'].increment +``` + +```php +$updateCounter = (new \Basho\Riak\Command\Builder\IncrementCounter($riak)) + ->withIncrement(1); + +$annikaMap = (new \Basho\Riak\Command\Builder\UpdateMap($riak)) + ->updateCounter('widget_purchases', $updateCounter); + +(new \Basho\Riak\Command\Builder\UpdateMap($riak)) + ->updateMap('annika_info', $annikaMap) + ->atLocation($location) + ->build() + ->execute(); +``` + +```python +map.maps['annika_info'].counters['widget_purchases'].increment() +map.store() +``` + +```csharp +// https://github.com/basho/riak-dotnet-client/blob/develop/src/RiakClientExamples/Dev/Using/DataTypes.cs + +var mapOperation = new UpdateMap.MapOperation(); +mapOperation.Map("annika_info").IncrementCounter("widget_purchases", 1); + +builder.WithMapOperation(mapOperation); + +UpdateMap cmd = builder.Build(); +client.Execute(cmd); +``` + +```javascript +var mapOp = new Riak.Commands.CRDT.UpdateMap.MapOperation(); +mapOp.map('annika_info').incrementCounter('widget_purchases', 1); + +var options = { + bucketType: 'maps', + bucket: 'customers', + key: 'ahmed_info', + op: mapOp +}; + +client.updateMap(options, function (err, rslt) { + if (err) { + throw new Error(err); + } +}); +``` + +```erlang +Map19 = riakc_map:update( + {<<"annika_info">>, map}, + fun(M) -> riakc_map:update( + {<<"widget_purchases">>, counter}, + fun(C) -> riakc_counter:increment(1, C) end, + M) end, + Map18). +``` + +```curl +curl -XPOST http://localhost:8098/types/maps/buckets/customers/datatypes/ahmed_info \ + -H "Content-Type: application/json" \ + -d ' + { + "update": { + "annika_info_map": { + "update": { + "widget_purchases_counter": 1 + } + } + } + } + ' +``` + +Now let's store Annika's interests in a set: + +```java +// Using our "ahmedMap" location from above: + +SetUpdate su = new SetUpdate().add("tango dancing"); +MapUpdate annikaUpdate = new MapUpdate() + .update("interests", su); +MapUpdate ahmedUpdate = new MapUpdate() + .update("annika_info", annikaUpdate); +UpdateMap update = new UpdateMap.Builder(ahmedMap, ahmedUpdate) + .build(); +client.execute(update); +``` + +```ruby +map.maps['annika_info'].sets['interests'].add('tango dancing') +``` + +```php +$updateSet = (new \Basho\Riak\Command\Builder\UpdateSet($riak)) + ->add('tango dancing'); + +$annikaMap = (new \Basho\Riak\Command\Builder\UpdateMap($riak)) + ->updateSet('interests', $updateSet); + +$response = (new \Basho\Riak\Command\Builder\UpdateMap($riak)) + ->updateMap('annika_info', $annikaMap) + ->atLocation($location) + ->withParameter('returnbody', 'true') + ->build() + ->execute(); +``` + +```python +map.maps['annika_info'].sets['interests'].add('tango dancing') +map.store() +``` + +```csharp +// https://github.com/basho/riak-dotnet-client/blob/develop/src/RiakClientExamples/Dev/Using/DataTypes.cs + +var mapOperation = new UpdateMap.MapOperation(); +mapOperation.Map("annika_info").AddToSet("interests", "tango dancing"); + +builder.WithMapOperation(mapOperation); +client.Execute(builder.Build()); +``` + +```javascript +var mapOp = new Riak.Commands.CRDT.UpdateMap.MapOperation(); +var annika_map = mapOp.map('annika_info'); +annika_map.addToSet('interests', 'tango dancing'); + +var options = { + bucketType: 'maps', + bucket: 'customers', + key: 'ahmed_info', + op: mapOp +}; + +client.updateMap(options, function (err, rslt) { + if (err) { + throw new Error(err); + } +}); +``` + +```erlang +Map20 = riakc_map:update( + {<<"annika_info">>, map}, + fun(M) -> riakc_map:update( + {<<"interests">>, set}, + fun(S) -> riakc_set:add_element(<<"tango dancing">>, S) end, + M) end, + Map19). +``` + +```curl +curl -XPOST http://localhost:8098/types/maps/buckets/customers/datatypes/ahmed_info \ + -H "Content-Type: application/json" \ + -d ' + { + "update": { + "annika_info_map": { + "update": { + "interests_set": { + "add": "tango dancing" + } + } + } + } + } + ' +``` + +We can remove that interest in just the way that we would expect: + +```java +// Using our "ahmedMap" location from above: + +SetUpdate su = new SetUpdate().remove("tango dancing"); +MapUpdate annikaUpdate = new MapUpdate() + .update("interests", su); +MapUpdate ahmedUpdate = new MapUpdate() + .update("annika_info", annikaUpdate); +UpdateMap update = new UpdateMap.Builder(ahmedMap, ahmedUpdate) + .withUpdate(ahmedUpdate) + .build(); +client.execute(update); +``` + +```ruby +map.maps['annika_info'].sets['interests'].remove('tango dancing') +``` + +```php +$updateSet = (new \Basho\Riak\Command\Builder\UpdateSet($riak)) + ->remove('tango dancing'); + +$annikaMap = (new \Basho\Riak\Command\Builder\UpdateMap($riak)) + ->updateSet('interests', $updateSet); + +(new \Basho\Riak\Command\Builder\UpdateMap($riak)) + ->updateMap('annika_info', $annikaMap) + ->atLocation($location) + ->withContext($response->getMap()->getContext()) + ->build() + ->execute(); +``` + +```python +map.maps['annika_info'].sets['interests'].discard('tango dancing') +map.store() +``` + +```csharp +// https://github.com/basho/riak-dotnet-client/blob/develop/src/RiakClientExamples/Dev/Using/DataTypes.cs + +var mapOperation = new UpdateMap.MapOperation(); +mapOperation.Map("annika_info").RemoveFromSet("interests", "tango dancing"); + +// Note: using Context from previous response +builder + .WithMapOperation(mapOperation) + .WithContext(response.Context); +client.Execute(builder.Build()); +``` + +```javascript +var options = { + bucketType: 'maps', + bucket: 'customers', + key: 'ahmed_info' +}; + +client.fetchMap(options, function (err, rslt) { + var mapOp = new Riak.Commands.CRDT.UpdateMap.MapOperation(); + var annika_map = mapOp.map('annika_info'); + annika_map.removeFromSet('interests', 'tango dancing'); + + options = { + bucketType: 'maps', + bucket: 'customers', + key: 'ahmed_info', + op: mapOp, + context: rslt.context + }; + + client.updateMap(options, function (err, rslt) { + if (err) { + throw new Error(err); + } + }); +}); +``` + +```erlang +Map21 = riakc_map:update( + {<<"annika_info">>, map}, + fun(M) -> riakc_map:update( + {<<"interests">>, set}, + fun(S) -> riakc_set:del_element(<<"tango dancing">>, S) end, + M) end, + Map20). +``` + +```curl +curl http://localhost:8098/types/maps/buckets/customers/datatypes/ahmed_info + +# Response +{"type":"map","value":{"annika_info_map":{"enterprise_plan_flag":false,"family_plan_flag":false,"first_name_register":"Annika","free_plan_flag":true,"interests_set":["tango dancing"],"last_name_register":"Weiss","widget_purchases_counter":1},"enterprise_customer_flag":false,"first_name_register":"Ahmed","interests_set":["indie pop","motorcycles","robots"],"page_visits_counter":1,"phone_number_register":"5551234567"},"context":"g2wAAAABaAJtAAAADCMJ/vn2jOEXAAAAAWEKag=="} + +curl -XPOST http://localhost:8098/types/maps/buckets/customers/datatypes/ahmed_info \ + -H "Content-Type: application/json" \ + -d ' + { + "update": { + "annika_info_map": { + "update": { + "interests_set": { + "remove": "tango dancing" + } + } + } + }, + "context" : "g2wAAAABaAJtAAAADCMJ/vn2jOEXAAAAAWEKag==" + } + ' +``` + +If we wanted to add store information about one of Annika's specific +purchases, we could do so within a map: + +```java +// Using our "ahmedMap" location from above: + +MapUpdate purchaseUpdate = new MapUpdate() + .update("first_purchase", new FlagUpdate(true) + .update("amount", new RegisterUpdate("1271")) + .update("items", new SetUpdate().add("large widget")); +MapUpdate annikaUpdate = new MapUpdate() + .update("purchase", purchaseUpdate); +MapUpdate ahmedUpdate = new MapUpdate() + .update("annika_info", annikaUpdate); +UpdateMap update = new UpdateMap.Builder(ahmedMap, ahmedUpdate) + .withUpdate(ahmedUpdate) + .build(); +client.execute(update); +``` + +```ruby +map.maps['annika_info'].maps['purchase'].batch do |m| + m.flags['first_purchase'] = true + m.register['amount'] = 1271.to_s + m.sets['items'].add('large widget') +end +``` + +```php +$updateSet = (new \Basho\Riak\Command\Builder\UpdateSet($riak)) + ->add('large widget'); + +$purchaseMap = (new \Basho\Riak\Command\Builder\UpdateMap($riak)) + ->updateFlag('first_purchase', true) + ->updateRegister('amount', '1271') + ->updateSet('items', $updateSet); + +$annikaMap = (new \Basho\Riak\Command\Builder\UpdateMap($riak)) + ->updateMap('purchase', $purchaseMap); + +$response = (new \Basho\Riak\Command\Builder\UpdateMap($riak)) + ->updateMap('annika_info', $annikaMap) + ->atLocation($location) + ->withParameter('returnbody', 'true') + ->build() + ->execute(); +``` + +```python +map.maps['annika_info'].maps['purchase'].flags['first_purchase'].enable() +map.maps['annika_info'].maps['purchase'].register['amount'].assign(str(1271)) +map.maps['annika_info'].maps['purchase'].sets['items'].add('large widget') +# and so on +map.store() +``` + +```csharp +// https://github.com/basho/riak-dotnet-client/blob/develop/src/RiakClientExamples/Dev/Using/DataTypes.cs + +var mapOperation = new UpdateMap.MapOperation(); +mapOperation.Map("annika_info").Map("purchase") + .SetFlag("first_purchase", true) + .SetRegister("amount", "1271") + .AddToSet("items", "large widget"); + +builder.WithMapOperation(mapOperation); +client.Execute(builder.Build()); +``` + +```javascript +var mapOp = new Riak.Commands.CRDT.UpdateMap.MapOperation(); +var annika_map = mapOp.map('annika_info'); +var annika_purchase_map = annika_map.map('purchase'); +annika_purchase_map.setFlag('first_purchase', true); +annika_purchase_map.setRegister('amount', '1271'); +annika_purchase_map.addToSet('items', 'large widget'); + +var options = { + bucketType: 'maps', + bucket: 'customers', + key: 'ahmed_info', + op: mapOp +}; + +client.updateMap(options, function (err, rslt) { + if (err) { + throw new Error(err); + } +}); +``` + +```erlang +Map22 = riakc_map:update( + {<<"annika_info">>, map}, + fun(M) -> riakc_map:update( + {<<"purchase">>, map}, + fun(M) -> riakc_map:update( + {<<"first_purchase">>, flag}, + fun(R) -> riakc_flag:enable(R) end, + M) end, + M) end, + Map21 +). +``` + +```curl +curl -XPOST http://localhost:8098/types/maps/buckets/customers/datatypes/ahmed_info \ + -H "Content-Type: application/json" \ + -d ' + { + "update": { + "annika_info_map": { + "update": { + "purchase_map": { + "update": { + "first_purchase_flag": "enable", + "amount_register": "1271", + "items_set": { + "add": "large widget" + } + } + } + } + } + } + } + ' +``` diff --git a/content/riak/kv/2.2.6/developing/data-types/sets.md b/content/riak/kv/2.2.6/developing/data-types/sets.md new file mode 100644 index 0000000000..57f51b1c04 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/data-types/sets.md @@ -0,0 +1,764 @@ +--- +title_supertext: "Developing with Riak KV" +title: "Data Types: Sets" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Sets" + identifier: "data_types_sets" + weight: 101 + parent: "developing_data_types" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/using/data-types/sets + - /riak-docs/riak/kv/2.2.6/dev/using/data-types/sets + - /riak-docs/riak/2.2.6/dev/data-modeling/data-types/sets + - /riak-docs/riak/kv/2.2.6/dev/data-modeling/data-types/sets +--- + +Sets are a bucket-level Riak data type that can be used by themselves, associated with a bucket/key pair, or used [within a map](../maps#sets-within-maps). + +Sets are collections of unique binary values (such as strings). All of +the values in a set are unique. + +For example, if you attempt to add the element `shovel` to a set that already contains `shovel`, the operation will be ignored by Riak KV. + +## Set Up a Bucket Type + +> If you've already created and activated a bucket type with `set` as the `datatype` parameter, skip to the [next section](#client-setup). + +Start by creating a bucket type with the `datatype` parameter `set`: + +```bash +riak-admin bucket-type create sets '{"props":{"datatype":"set"}}' +``` + +> **Note** +> +> The `sets` bucket type name provided above is an example and is not required to be `sets`. You are free to name bucket types whatever you like, with the exception of `default`. + +After creating a bucket with a Riak data type, confirm the bucket property configuration associated with that type is correct: + +```bash +riak-admin bucket-type status sets +``` + +This returns a list of bucket properties and their values +in the form of `property: value`. + +If our `sets` bucket type has been set properly we should see the following pair in our console output: + +``` +datatype: set +``` + +Once we have confirmed the bucket type is properly configured, we can activate the bucket type to be used in Riak KV: + +```bash +riak-admin bucket-type activate sets +``` + +We can check if activation has been successful by using the same `bucket-type status` command shown above: + +```bash +riak-admin bucket-type status sets +``` + +After creating and activating our new `sets` bucket type, we can setup our client to start using the bucket type as detailed in the next section. + +## Client Setup + +Using sets involves creating a bucket/key pair to house a set and running set-specific operations on that pair. + +Here is the general syntax for creating a bucket type/bucket/key +combination to handle a set: + +```java +// In the Java client, a bucket/bucket type combination is specified +// using a Namespace object. To specify bucket, bucket type, and key, +// use a Location object that incorporates the Namespace object, as is +// done below. + +Location set = + new Location(new Namespace("", ""), ""); +``` + +```ruby +# Note: both the Riak Ruby Client and Ruby the language have a class +# called Set. Make sure that you refer to the Ruby version as ::Set and +# the Riak client version as Riak::Crdt::Set + +bucket = client.bucket_type('bucket_type_name').bucket('bucket_name') +set = Riak::Crdt::Set.new(bucket, key) +``` + +```php +$location = new \Basho\Riak\Location('key', new \Basho\Riak\Bucket('bucket_name', 'bucket_type')); +``` + +```python +# Note: The Python standard library `collections` module has an abstract +# base class called Set, which the Riak Client version subclasses as +# `riak.datatypes.Set`. These classes are not directly interchangeable. +# In addition to the base methods, `riak.datatypes.Set` also +# implements the `add` and `discard` methods from +# `collections.MutableSet`, but does not implement the rest of its +# API. Be careful when importing, or simply use the instances returned +# by `RiakBucket.get()` and `RiakBucket.new()` instead of directly +# importing the class. + +set = bucket.new(key) + +# or + +from riak.datatypes import Set +set = Set(bucket, key) +``` + +```csharp +// https://github.com/basho/riak-dotnet-client/blob/develop/src/RiakClientExamples/Dev/Using/DataTypes.cs + +// As with counters, with the Riak .NET Client you interact with sets +// by building an Options object or using a Builder +var builder = new FetchSet.Builder() + .WithBucketType("sets") + .WithBucket("travel") + .WithKey("cities"); + +// NB: builder.Options will only be set after Build() is called. +FetchSet fetchSetCommand = builder.Build(); + +FetchSetOptions options = new FetchSetOptions("sets", "travel", "cities"); + +// These two options objects are equal +Assert.AreEqual(options, builder.Options); +``` + +```javascript +// As with counters, with the Riak Node.js Client you interact with sets on the +// basis of the set's location in Riak, as specified by an options object. +// Below is an example: +var options = { + bucketType: 'sets', + bucket: 'travel', + key: 'cities' +}; +``` + +```erlang +%% Like counters, sets are not encapsulated in a +%% bucket/key in the Erlang client. See below for more +%% information. +``` + +```curl +curl http://localhost:8098/types//buckets//datatypes/ + +# Note that this differs from the URL structure for non-data type requests, +# which end in /keys/ +``` + +## Create a Set + +For the following example, we will use a set to store a list of cities that we +want to visit. Let's create a Riak set stored in the key `cities` in the bucket `travel` using the `sets` bucket type created previously: + +```java +// In the Java client, you specify the location of Data Types +// before you perform operations on them: + +Location citiesSet = + new Location(new Namespace("sets", "travel"), "cities"); +``` + +```ruby +travel = client.bucket_type('sets').bucket('travel') +cities_set = Riak::Crdt::Set.new(travel, 'cities') + +# Alternatively, the Ruby client enables you to set a bucket type as +# being globally associated with a Riak data type. The following would +# set all set buckets to use the sets bucket type: + +Riak::Crdt::DEFAULT_BUCKET_TYPES[:set] = 'sets' + +# This would enable us to create our set without specifying a bucket +# type: +travel = client.bucket('travel') +cities_set = Riak::Crdt::Set.new(travel, 'cities') +``` + +```php +$location = new \Basho\Riak\Location('cities', 'travel', 'sets'); +``` + +```python +travel = client.bucket_type('sets').bucket('travel') + +# The client detects the bucket type's data type and automatically +# returns the right data type for you, in this case a Riak set. +cities_set = travel.new('cities') + +# You can also create a reference to a set explicitly: +from riak.datatypes import Set + +cities_set = Set(travel, 'cities') +``` + +```csharp +// https://github.com/basho/riak-dotnet-client/blob/develop/src/RiakClientExamples/Dev/Using/DataTypes.cs + +// Now we'll create a Builder object for the set with which we want to +// interact: +var builder = new FetchSet.Builder() + .WithBucketType("sets") + .WithBucket("travel") + .WithKey("cities"); +``` + +```javascript +// Now we'll create a options object for the set with which we want to +// interact: +var options = { + bucketType: 'sets', + bucket: 'travel', + key: 'cities' +}; +``` + +```erlang +CitiesSet = riakc_set:new(). + +%% Sets in the Erlang client are opaque data structures that +%% collect operations as you mutate them. We will associate the data +%% structure with a bucket type, bucket, and key later on. +``` + +```curl +# You cannot create an empty set through the HTTP interface. Sets can +# only be created when an element is added to them, as in the examples +# below. +``` + +Upon creation, our set is empty. We can verify that it is empty at any +time: + +```java +// Using our "cities" Location from above: + +FetchSet fetch = new FetchSet.Builder(citiesSet) + .build(); +FetchSet.Response response = client.execute(fetch); +RiakSet set = response.getDatatype(); +boolean isEmpty = set.viewAsSet().isEmpty(); +``` + +```ruby +cities_set.empty? +``` + +```php +# use $location from earlier +$set = (new \Basho\Riak\Command\Builder\FetchSet($riak)) + ->atLocation($location) + ->build() + ->execute() + ->getSet(); + +count($set->getData()); +``` + +```python +len(cities_set) == 0 +``` + +```csharp +// https://github.com/basho/riak-dotnet-client/blob/develop/src/RiakClientExamples/Dev/Using/DataTypes.cs + +var builder = new FetchSet.Builder() + .WithBucketType("sets") + .WithBucket("travel") + .WithKey("cities"); + +FetchSet fetchSetCommand = builder.Build(); +RiakResult rslt = client.Execute(fetchSetCommand); +SetResponse response = fetchSetCommand.Response; +// response.Value will be null +``` + +```javascript +var options = { + bucketType: 'sets', + bucket: 'travel', + key: 'cities' +}; +client.fetchSet(options, function (err, rslt) { + if (err) { + throw new Error(err); + } + + if (rslt.notFound) { + logger.info("set 'cities' is not found!"); + } +}); +``` + +```erlang +riakc_set:size(CitiesSet) == 0. + +%% Query functions like size/1, is_element/2, and fold/3 operate over +%% the immutable value fetched from the server. In the case of a new +%% set that was not fetched, this is an empty collection, so the size +%% is 0. +``` + +```curl +curl http://localhost:8098/types/sets/buckets/travel/datatypes/cities + +# Response +{"type":"set","error":"notfound"} +``` + +## Add to a Set + +But let's say that we read a travel brochure saying that Toronto and +Montreal are nice places to go. Let's add them to our `cities` set: + +```java +// Using our "cities" Location from above: + +SetUpdate su = new SetUpdate() + .add("Toronto") + .add("Montreal"); +UpdateSet update = new UpdateSet.Builder(citiesSet, su) + .build(); +client.execute(update); +``` + +```ruby +cities_set.add('Toronto') +cities_set.add('Montreal') +``` + +```php +# use $location from earlier +$response = (new \Basho\Riak\Command\Builder\UpdateSet($riak)) + ->add('Toronto') + ->add('Montreal') + ->atLocation($location) + ->withParameter('returnbody', 'true') + ->build() + ->execute(); +``` + +```python +cities_set.add('Toronto') +cities_set.add('Montreal') +``` + +```csharp +// https://github.com/basho/riak-dotnet-client/blob/develop/src/RiakClientExamples/Dev/Using/DataTypes.cs + +var adds = new[] { "Toronto", "Montreal" }; + +var builder = new UpdateSet.Builder() + .WithBucketType("sets") + .WithBucket("travel") + .WithKey("cities") + .WithAdditions(adds); + +UpdateSet cmd = builder.Build(); +RiakResult rslt = client.Execute(cmd); +SetResponse response = cmd.Response; + +Assert.Contains("Toronto", response.AsStrings.ToArray()); +Assert.Contains("Montreal", response.AsStrings.ToArray()); +``` + +```javascript +var options = { + bucketType: 'sets', + bucket: 'travel', + key: 'cities' +}; +var cmd = new Riak.Commands.CRDT.UpdateSet.Builder() + .withBucketType(options.bucketType) + .withBucket(options.bucket) + .withKey(options.key) + .withAdditions(['Toronto', 'Montreal']) + .withCallback( + function (err, rslt) { + if (err) { + throw new Error(err); + } + } + ) + .build(); +client.execute(cmd); +``` + +```erlang +CitiesSet1 = riakc_set:add_element(<<"Toronto">>, CitiesSet), +CitiesSet2 = riakc_set:add_element(<<"Montreal">>, CitiesSet1). +``` + +```curl +curl -XPOST http://localhost:8098/types/sets/buckets/travel/datatypes/cities \ + -H "Content-Type: application/json" \ + -d '{"add_all":["Toronto", "Montreal"]}' +``` + +## Remove from a Set + +Later on, we hear that Hamilton and Ottawa are nice cities to visit in +Canada, but if we visit them, we won't have time to visit Montreal, so +we need to remove it from the list. + +Note that removing an element from a set is trickier than adding elements. In +order to remove an item (or multiple items), we need to first fetch the +set, which provides our client access to the set's [causal context](../../../learn/concepts/causal-context). + +Once we've fetched the set, we can remove the element `Montreal` and +store the set: + +```java +// Using our "citiesSet" Location from above + +// First, we get a response +FetchSet fetch = new FetchSet.Builder(citiesSet).build(); +FetchSet.Response response = client.execute(fetch); + +// Then we can fetch the set's causal context +Context ctx = response.getContext(); + +// Now we build a SetUpdate operation +SetUpdate su = new SetUpdate() + .remove("Montreal") + .add("Hamilton") + .add("Ottawa"); + +// Finally, we update the set, specifying the context +UpdateSet update = new UpdateSet.Builder(citiesSet, su) + .withContext(ctx) + .build(); +client.execute(update); + +// More information on using causal context with the Java client can be +// found at the bottom of this document +``` + +```ruby +cities_set.remove('Montreal') +cities_set.add('Hamilton') +cities_set.add('Ottawa') +``` + +```php +# use $location & $response from earlier +(new \Basho\Riak\Command\Builder\UpdateSet($riak)) + ->add('Hamilton') + ->add('Ottawa') + ->remove('Montreal') + ->atLocation($location) + ->withContext($response->getSet()->getContext()) + ->build() + ->execute(); +``` + +```python +cities_set.discard('Montreal') +cities_set.add('Hamilton') +cities_set.add('Ottawa') +cities_set.store() +``` + +```csharp +// https://github.com/basho/riak-dotnet-client/blob/develop/src/RiakClientExamples/Dev/Using/DataTypes.cs + +var removes = new[] { "Montreal" }; +var adds = new[] { "Hamilton", "Ottawa" }; + +// Note: +// using the builder from above +// using the Context member from the above response +builder + .WithAdditions(adds) + .WithRemovals(removes) + .WithContext(response.Context); + +UpdateSet cmd = builder.Build(); +RiakResult rslt = client.Execute(cmd); +SetResponse response = cmd.Response; + +// using System.Linq +var responseStrings = response.AsStrings.ToArray(); + +Assert.Contains("Toronto", responseStrings); +Assert.Contains("Hamilton", responseStrings); +Assert.Contains("Ottawa", responseStrings); +``` + +```javascript +var options = { + bucketType: 'sets', + bucket: 'travel', + key: 'cities' +}; +client.fetchSet(options, function (err, rslt) { + if (err) { + throw new Error(err); + } + + // NB: clone package https://www.npmjs.com/package/clone + var update_opts = clone(options); + update_opts.context = rslt.context; + update_opts.additions = ['Hamilton', 'Ottawa']; + update_opts.removals = ['Montreal', 'Ottawa']; + + client.updateSet(update_opts, function (err, rslt) { + if (err) { + throw new Error(err); + } + }); +}); +``` + +```erlang +CitiesSet3 = riakc_set:del_element(<<"Montreal">>, CitiesSet2), +CitiesSet4 = riakc_set:add_element(<<"Hamilton">>, CitiesSet3), +CitiesSet5 = riakc_set:add_element(<<"Ottawa">>, CitiesSet4). +``` + +```curl +curl http://localhost:8098/types/sets/buckets/travel/datatypes/cities + +# Response +{"type":"set","value":["Montreal","Toronto"],"context":"g2wAAAABaAJtAAAADCMJ/vn7tg36AAAAAWECag=="} + +curl -XPOST http://localhost:8098/types/sets/buckets/travel/datatypes/cities \ + -H "Content-Type: application/json" \ + -d '{"remove": "Montreal","add_all":["Hamilton", "Ottawa"],"context":"g2wAAAABaAJtAAAADCMJ/vn7tg36AAAAAWECag=="}' +``` + +## Retrieve a Set + +Now, we can check on which cities are currently in our set: + +```java +// Using our "cities" Location from above: + +FetchSet fetch = new FetchSet.Builder(citiesSet) + .build(); +FetchSet.Response response = client.execute(fetch); +Set binarySet = response.getDatatype().view(); +for (BinaryValue city : binarySet) { + System.out.println(city.toStringUtf8()); +} +``` + +```ruby +cities_set.members + +# +``` + +```php +# use $location from earlier +$set = (new \Basho\Riak\Command\Builder\FetchSet($riak)) + ->atLocation($location) + ->build() + ->execute() + ->getSet(); + +var_dump($set->getData()); +``` + +```python +cities_set.dirty_value + +# The value fetched from Riak is always immutable, whereas the "dirty +# value" takes into account local modifications that have not been +# sent to the server. For example, where the call above would return +# frozenset(['Toronto', 'Hamilton', 'Ottawa']), the call below would +# return frozenset([]). + +cities_set.value + +# To fetch the value stored on the server, use the call below. Note +# that this will clear any unsent additions or deletions. +cities_set.reload() +``` + +```csharp +// https://github.com/basho/riak-dotnet-client/blob/develop/src/RiakClientExamples/Dev/Using/DataTypes.cs + +foreach (var value in setResponse.AsStrings) +{ + Console.WriteLine("Cities Set Value: {0}", value); +} + +// Output: +// Cities Set Value: Hamilton +// Cities Set Value: Ottawa +// Cities Set Value: Toronto +``` + +```javascript +var options = { + bucketType: 'sets', + bucket: 'travel', + key: 'cities' +}; +client.fetchSet(options, function(err, rslt) { + if (err) { + throw new Error(err); + } + + logger.info("cities set values: '%s'", + rslt.values.join(', ')); +}); + +// Output: +// info: cities set values: 'Hamilton, Ottawa, Toronto' +``` + +```erlang +riakc_set:dirty_value(CitiesSet5). + +%% The value fetched from Riak is always immutable, whereas the "dirty +%% value" takes into account local modifications that have not been +%% sent to the server. For example, where the call above would return +%% [<<"Hamilton">>, <<"Ottawa">>, <<"Toronto">>], the call below would +%% return []. These are essentially ordsets: + +riakc_set:value(CitiesSet5). + +%% To fetch the value stored on the server, use the call below: + +{ok, SetX} = riakc_pb_socket:fetch_type(Pid, + {<<"sets">>,<<"travel">>}, + <<"cities">>). +``` + +```curl +curl http://localhost:8098/types/sets/buckets/travel/datatypes/cities + +# Response +{"type":"set","value":["Hamilton","Ottawa","Toronto"],"context":"g2wAAAABaAJtAAAADCMJ/vn7tg36AAAAAWEEag=="} + +# You can also fetch the value of the set without the context included: +curl http://localhost:8098/types/sets/buckets/travel/datatypes/cities?include_context=false + +# Response +{"type":"set","value":["Hamilton", "Ottawa", "Toronto"]} +``` + +## Find Set Member + +Or we can see whether our set includes a specific member: + +```java +// Using our "citiesSet" from above: + +System.out.println(citiesSet.contains(("Vancouver")); +System.out.println(citiesSet.contains("Ottawa")); +``` + +```ruby +cities_set.include? 'Vancouver' +# false + +cities_set.include? 'Ottawa' +# true +``` + +```php +in_array('Vancouver', $set->getData()); # false + +in_array('Ottawa', $set->getData()); # true +``` + +```python +'Vancouver' in cities_set +# False + +'Ottawa' in cities_set +# True +``` + +```csharp +// https://github.com/basho/riak-dotnet-client/blob/develop/src/RiakClientExamples/Dev/Using/DataTypes.cs + +using System.Linq; + +bool includesVancouver = response.AsStrings.Any(v => v == "Vancouver"); +bool includesOttawa = response.AsStrings.Any(v => v == "Ottawa"); +``` + +```javascript +// Use standard javascript array method indexOf() + +var cities_set = result.values; +cities_set.indexOf('Vancouver'); // if present, index is >= 0 +cities_set.indexOf('Ottawa'); // if present, index is >= 0 +``` + +```erlang +%% At this point, Set5 is the most "recent" set from the standpoint +%% of our application. + +riakc_set:is_element(<<"Vancouver">>, CitiesSet5). +riakc_set:is_element(<<"Ottawa">>, CitiesSet5). +``` + +```curl +# With the HTTP interface, this can be determined from the output of +# a fetch command like the one displayed in the example above +``` + +## Size of Set + +We can also determine the size of the set: + +```java +// Using our "citiesSet" from above: + +int numberOfCities = citiesSet.size(); +``` + +```ruby +cities_set.members.length +``` + +```php +count($set->getData()); +``` + +```python +len(cities_set) +``` + +```csharp +// https://github.com/basho/riak-dotnet-client/blob/develop/src/RiakClientExamples/Dev/Using/DataTypes.cs + +using System.Linq; + +// Note: this enumerates the IEnumerable +setResponse.Values.Count(); +``` + +```javascript +// Use standard javascript array property length + +var cities_set_size = result.values.length; +``` + +```erlang +riakc_set:size(CitiesSet5). +``` + +```curl +# With the HTTP interface, this can be determined from the output of +# a fetch command like the one displayed in the example above +``` diff --git a/content/riak/kv/2.2.6/developing/faq.md b/content/riak/kv/2.2.6/developing/faq.md new file mode 100644 index 0000000000..8fd7c284c1 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/faq.md @@ -0,0 +1,654 @@ +--- +title_supertext: "Developing with Riak KV" +title: "Frequently Asked Questions" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Developing FAQ" + identifier: "developing_faq" + weight: 108 + parent: "developing" +toc: true +aliases: + - /riak-docs/riak/2.2.6/community/faqs/developing + - /riak-docs/riak/kv/2.2.6/community/faqs/developing +--- + +[[Basho Bench]: {{}}riak/kv/2.2.6/using/performance/benchmarking +[Bitcask]: {{}}riak/kv/2.2.6/setup/planning/backend/bitcask +[Bucket Properties]: {{}}riak/kv/2.2.6/developing/usage +[built-in functions list]: https://github.com/basho/riak_kv/blob/master/priv/mapred_builtins.js +[commit hooks]: {{}}riak/kv/2.2.6/developing/usage/commit-hooks +[Configuration Files]: {{}}riak/kv/2.2.6/configuring/reference +[contrib.basho.com]: https://github.com/basho/riak_function_contrib +[Erlang Riak Client]: {{}}riak/kv/2.2.6/developing/client-libraries +[MapReduce]: {{}}riak/kv/2.2.6/developing/usage/mapreduce +[Memory]: {{}}riak/kv/2.2.6/setup/planning/backend/memory +[Riak CS]: {{}}riak/cs/2.1.1 +[System Planning]: {{}}riak/kv/2.2.6/setup/planning/start/#network-configuration-load-balancing +[vector clocks]: {{}}riak/kv/2.2.6/learn/concepts/causal-context#vector-clocks + + +## General + + +**Q: How can I automatically expire a key from Riak? I want to regularly purge items from Riak that are older than a certain timestamp, but MapReduce times out on large numbers of items. Can I expire data automatically?** + +**A:** + If you're using [Bitcask], the default storage backend, and you want items to expire at a consistent interval (assuming that they are not updated), set the `expiry_secs` option in `app.config`. Items that persist past this threshold will not be returned on get/fetch operations and will eventually be removed from disk by Bitcask's merging process. For example: + + ```erlang + {bitcask, [ + {data_root, "data/bitcask"}, + {expiry_secs, 86400} %% Expire after a day + ]}, + ``` + + There is no limit on how large or small the `expiry_secs` setting can be as long as it is greater than 0. + + You can also set auto-expire using the [Memory] storage backend, but it will be limited by RAM. + + +--- + +**Q: Is there better performance for a few objects in many buckets, or many objects in a few buckets?** + + +**A:** + Generally speaking, it does not matter if you have many buckets with a small number of objects or a small number of buckets with a large number of objects. Buckets that use the cluster's default bucket properties (which can be set in your `app.config` file) are essentially free. + + If the buckets require different bucket properties, however, those custom properties incur some cost because changes in bucket properties must be gossiped around the cluster. If you create many, many buckets with custom properties, the cost can indeed have an impact. + + +--- + +**Q: Can I list buckets or keys in production?** + + +**A:** + It is *not* recommended that you list the buckets in production because it is a costly operation irrespective of the bucket's size. + + Buckets are not like directories on a file system or tables in a database; rather, they are logical properties applied to objects, i.e. there is no *actual* separation of objects by bucket. + + A filter must be applied to all of the objects in the system in order to find those residing in a particular bucket. Buckets are intended for configuration purposes (e.g. replication properties) rather than for general queries. + + To keep track of groups of objects there are several options with various trade-offs: secondary indexes, search, or a list using links. + + +--- + +**Q: Why do secondary indexes (2i) return inconsistent results after using `force-remove` to drop a node from the cluster?** + + +**A:** + The Riak key/value store distributes values across all of the partitions in the ring. In order to minimize synchronization issues with secondary indexes, Riak stores index information in the same partition as the data values. + + When a node fails or is taken out of the cluster without using riak-admin leave, all of the data held by that node is lost to the cluster. This leaves N - 1 consistent replicas of the data. If `riak-admin force-remove` is used to remove the downed node, the remaining clusters will claim the partitions the failed node previously held. The data in the newly claimed vnodes will be made consistent one key at a time through the read-repair mechanism as each key is accessed, or through Active Anti-entropy (AAE) if enabled. + + As a simplistic example, consider this hypothetical cluster: + + * 5 nodes (nodes A-E) + * ring size = 16 + * `n_val` = 3. + + For this example, I am using simple small integers instead of the actual 160-bit partition index values for the sake of simplicity. The partitions are assigned to the nodes as follows: + +``` +A: 0-5-10-15 +B: 1-6-11 +C: 2-7-12 +D: 3-8-12 +E: 4-9-14 +``` + When a value is stored in Riak, the `{bucket, key}` is hashed to determine its first primary partition, and the value is stored in that partition and the next `n_val` - 1 partitions in the ring. + A preflist consists of the vnode which owns the key, and the next `n_val` vnodes in the ring, in order. In this scenario there are 16 preflists: + + + + + + +
0-1-21-2-32-3-43-4-5
4-5-65-6-76-7-87-8-9
8-9-109-10-1110-11-1211-12-13
12-13-1413-14-1514-15-015-0-1
+ + Index information for each partition is co-located with the value data. In order to get a full result set for a secondary index query, Riak will need to consult a "covering set" of vnodes that includes at least one member of each preflist. This will require a minimum of 1/`n_val` of the vnodes, rounded up, in this case 6. There are 56 possible covering sets consisting of 6 vnodes: + + + + + + + + + + + + + + + + +
0-1-4-7-10-130-2-4-7-10-130-2-5-7-10-130-2-5-8-10-13
0-2-5-8-11-130-2-5-8-11-140-3-4-7-10-130-3-5-7-10-13
0-3-5-8-10-130-3-5-8-11-130-3-5-8-11-140-3-6-7-10-13
0-3-6-8-10-130-3-6-8-11-130-3-6-8-11-140-3-6-9-10-13
0-3-6-9-11-130-3-6-9-11-140-3-6-9-12-130-3-6-9-12-14
0-3-6-9-12-151-2-5-8-11-141-3-5-8-11-141-3-6-8-11-14
1-3-6-9-11-141-3-6-9-12-141-3-6-9-12-151-4-5-8-11-14
1-4-6-8-11-141-4-6-9-11-141-4-6-9-12-141-4-6-9-12-15
1-4-7-8-11-141-4-7-9-11-141-4-7-9-12-141-4-7-9-12-15
1-4-7-10-11-141-4-7-10-12-141-4-7-10-12-151-4-7-10-13-14
1-4-7-10-13-152-3-6-9-12-152-4-6-9-12-152-4-7-9-12-15
2-4-7-10-12-152-4-7-10-13-152-5-6-9-12-152-5-7-9-12-15
2-5-7-10-12-152-5-7-10-13-152-5-8-9-12-152-5-8-10-12-15
2-5-8-10-13-152-5-8-11-12-152-5-8-11-13-152-5-8-11-14-15
+ + When a node fails or is marked down, its vnodes will not be considered for coverage queries. Fallback vnodes will be created on other nodes so that PUT and GET operations can be handled, but only primary vnodes are considered for secondary index coverage queries. If a covering set cannot be found, `{error, insufficient_vnodes}` will be returned. Thus, the reply will either be complete or an error. + + When a node is `force-remove`d, it is dropped from the cluster without transferring its data to other nodes, and the remaining nodes then claim the unowned partitions, designating new primary replicas to comply with `n_val`, but they do not immediately populate the data or indexes. + + Read repair, triggered by GETs or PUTs on the individual keys, and/or Active Anti-Entropy, will eventually repopulate the data, restoring consistency. + A GET operation for a key will request the data from all of the vnodes in its preflist, by default waiting for over half of them to respond. This results in consistent responses to get even when one of the vnodes in the preflist has been compromised. + + Secondary index queries, however, consult a covering set which may include only 1 member of the preflist. If that vnode is empty due to the `force-remove` operation, none of the keys from that preflist will be returned. + + Continuing with the above example, consider if node C is force removed. + This is one possible configuration after rebalancing: + +``` +A: 0-5-10-15 +B: 1-6-11-2* +D: 3-8-12-7* +E: 4-9-14-12* +``` + + Vnodes 2,7, and 12 (marked with `*`) are newly created primary partitions that do not contain any values or index information. + + In this new 4-node configuration any coverage set that includes vnodes 2,7, or 12 will return incomplete results until consistency is restored via read-repair or AAE, because not all vnodes will contain the data that would otherwise be present. + + + So making a couple of assumptions for demonstration purposes: + + 1. The keys `a`, `b`, and `c` are stored in the following preflists: + + ``` + a - 0-1-2 + b - 6-7-8 + c - 10-11-12 + ``` + + 2. The cluster is not loaded, so no GET/PUT or other coverage queries are being performed + + 3. AAE is not enabled + + The coordinating node (the one that receives the request from the client) will attempt to spread the load by not using the same partitions for successive coverage queries. + + The results from secondary index queries that should return all 3 keys will vary depending on the nodes chosen for the coverage set. Of the 56 possible covering sets ... + + * 20 sets (35.7% of sets) will return all 3 keys `{a,b,c}`: + + + + + + +
0-2-5-8-10-130-2-5-8-11-130-2-5-8-11-140-3-5-8-10-13
0-3-5-8-11-130-3-5-8-11-140-3-6-8-10-130-3-6-8-11-13
0-3-6-8-11-140-3-6-9-10-130-3-6-9-11-130-3-6-9-11-14
1-2-5-8-11-141-3-5-8-11-141-3-6-8-11-141-3-6-9-11-14
1-4-5-8-11-141-4-6-8-11-141-4-6-9-11-141-4-7-8-11-14
+ + * 24 sets (42.9%) will return 2 of the 3 keys: + + + + + + + + + + + +
`{a,b}` (7 sets)
0-3-6-9-12-130-3-6-9-12-140-3-6-9-12-151-3-6-9-12-14
1-3-6-9-12-151-4-6-9-12-141-4-6-9-12-15 
`{a,c}` (12 sets)
0-1-4-7-10-130-2-4-7-10-130-2-5-7-10-130-3-4-7-10-13
0-3-5-7-10-130-3-6-7-10-131-4-7-10-11-141-4-7-10-12-14
1-4-7-10-12-151-4-7-10-13-141-4-7-10-13-151-4-7-9-11-14
`{b,c}` (5 sets)
2-5-8-10-12-152-5-8-10-13-152-5-8-11-12-152-5-8-11-14-15
2-5-8-11-13-15 
+ + * 10 sets (17.8%) will return only one of the 3 keys: + + + + + + + +
`{a}` (2 sets)
1-4-7-9-12-141-4-7-9-12-15 
`{b}` (4 sets)
2-3-6-9-12-152-4-6-9-12-152-5-6-9-12-152-5-8-9-12-15
`{c}` (4 sets)
2-4-7-10-12-152-4-7-10-13-152-5-7-10-12-152-5-7-10-13-15
+ + * 2 sets (3.6%) will not return any of the 3 keys + + +
2-4-7-9-12-152-5-7-9-12-15
+ +--- + +**Q: How do I load 3rd-party Javascript libraries for use in MapReduce functions?** + Is it possible to load third-party javascript libraries (like Underscore.js) to be available in MapReduce functions? + + +**A:** + Yes. For JavaScript, this can be done in `app.config` in `js_source_dir` in the `riak_kv` settings: + + ```erlang + {js_source_dir, "/etc/riak/javascript"}, + ``` + + For Erlang code (please note that you need compiled modules in this dir), set `add_paths` in the `riak_kv` section: + + ```erlang + {add_paths, "/etc/riak/erlang"}, + ``` + + You can find more details in the [Configuration Files] document. + +--- + +**Q: Is it possible to use key filtering to just return a list of keys that match a particular pattern without performing a MapReduce on it?** + When running a MapReduce query, a map phase results in Riak pulling an object off of disk. Some queries are only interested in the keys of an object and not the value. Is it possible to run a MapReduce query that does not have to pull objects off of disk? + + +**A:** + Yes. Specifying a MapReduce query with just a reduce phase will avoid any need to pull data off of disk. To return the results of a key filtering query you can do the following: + + ```json + { + "inputs": { + "bucket": "test", + "key_filters": [ + ["ends_with","1"] + ] + }, + "query": [ + { + "reduce": { + "language": "erlang", + "module": "riak_kv_mapreduce", + "function": "reduce_identity" + } + } + ] + } + ``` + + There is also a reduce function for counting inputs. This function can be used to count keys in a bucket without reading objects from disk: + + ```json + { + "inputs": { + "bucket": "test", + "key_filters": [ + [ + "ends_with","1" + ] + ] + }, + "query": [ + { + "reduce": { + "language": "erlang", + "module": "riak_kv_mapreduce", + "function": "reduce_count_inputs" + } + } + ] + } + ``` + + +--- + +**Q: How can I observe object sizes and sibling counts?** + + +**A:** + `riak-admin status` will return the following stats, which give the mean and median along with the 95th, 99th, and 100th percentile object size and sibling counts. + + ``` + node_get_fsm_siblings_mean : 0 + node_get_fsm_siblings_median : 0 + node_get_fsm_siblings_95 : 0 + node_get_fsm_siblings_99 : 0 + node_get_fsm_siblings_100 : 0 + node_get_fsm_objsize_mean : 0 + node_get_fsm_objsize_median : 0 + node_get_fsm_objsize_95 : 0 + node_get_fsm_objsize_99 : 0 + node_get_fsm_objsize_100 : 0 + ``` + + +--- + +**Q: A node left the cluster before handing off all data. How can I resolve this?** + + +**A:** + In versions of Riak earlier than Riak 1.0, there are cases in which a node that is leaving the cluster will shut down before handing off all of its data. This has been resolved in Riak 1.0. + + If you encounter this issue, you can rely upon the `read-repair` command to restore your lost replicas. Simply send a `HEAD` request for each key in your data set and Riak will restore replicas as needed. + + Alternatively, if the node that left prematurely is still installed/available, you can manually re-initiate handoff using the following sequence. This approach requires entering code directly into the Erlang console of a running Riak node, and is therefore most appropriate for users with a support contract with Basho that can ask for help if anything goes wrong. + + **Manual approach**: Restart the node that prematurely left by using `riak console`. Then copy/paste the following sequence, changing the first line to point to a node still in your cluster. Handoff should then restart, but there may be no visual indicator. Simply leave the node running for awhile. It should eventually hand off all data and then shut down. Verify handoff by once again checking the size of your data directories. + + ```erlang + ClusterNode = 'riak@127.0.0.1'. + + application:set_env(riak_core, wants_claim_fun, {riak_core_claim, never_wants_claim}). + {ok, Ring} = rpc:call(ClusterNode, riak_core_ring_manager, get_my_ring, []). + Ring2 = setelement(2, Ring, node()). + riak_core_ring_manager:set_my_ring(Ring2). + riak_core_ring_manager:write_ringfile(). + [gen_server:cast(riak_core_node_watcher, {up, Node, [riak_kv]}) || Node + ``` + + +--- + +**Q: Is there a limit on the size of files that can be stored on Riak?** + + +**A:** + There isn't a limit on object size, but we suggest you keep it to no more than 1-2MB for performance reasons. Variables such as network speed can directly affect the maximum usable object size for a given cluster. You should use a tool like [Basho Bench] to determine the performance of your cluster with a given object size before moving to production use. Or if your use case demands storing many large objects, you may want to consider the [Riak CS] object storage system, which is designed for precisely that purpose. + + +--- + +**Q: Does the bucket name impact key storage size?** + + +**A:** + The storage per key is 40 bytes plus the key size and bucket name size. + + Example: + + Key size: 15 bytes. + Bucket Name size: 10 bytes. + + Total size = 40 + 15 + 10 = **65 bytes**. + + + +--- + +**Q: Are Riak-generated keys unique within a bucket?** + + +**A:** + It's not guaranteed, but you are extremely unlikely to get collisions. Riak generates keys using an Erlang-generated unique ID and a timestamp hashed with SHA-1 and base-62 encoded for URL safety. + + +--- + +**Q: Where are bucket properties stored?** + + +**A:** + The bucket properties for the default bucket type are stored in the *ring* (metadata stored in each node about the cluster). Rings are gossipped as a single unit, so if possible you should limit your creation of custom buckets under the default bucket type. + Bucket properties for non-default bucket types are stored in the cluster metadata system. The cluster metadata system is a more efficient way of replicating this information around a Riak cluster. + + The bucket properties stay in the ring and cluster metadata even if the bucket is empty. + +--- + +**Q: Are Riak keys / buckets case sensitive?** + + +**A:** + Yes, they are case sensitive and treated as binaries (byte buffers). Thus, `mykey` is not equal to `MyKey`. + + +--- + +**Q: Can I run my own Erlang applications in the same VM as Riak?** + + +**A:** + We do not recommend running your application inside the same virtual machine as Riak for several reasons. If they are kept separate, the following will hold: + + 1. Your application and Riak will not compete for the same resources and are thus less likely to affect each other's performance and availability. + 2. You will be able to upgrade Riak and your application independently of one another. + 3. When your application or Riak need more capacity, you can scale them separately to meet your production needs. + + +--- + +**Q: Is there a simple way to reload an Erlang module for MapReduce across a cluster?** + + +**A:** + Assuming that the module is in your code path, you can run `c:nl(ModName)` from the Erlang console . + + + +--- + +**Q: How do I spread requests across---i.e. load balance---a Riak cluster?** + + +**A:** + There are at least two acceptable strategies for load balancing requests across your Riak cluster: **virtual IPs** and **reverse-proxy**. + + For further information see [System Planning]. + + +--- + + +**Q: Why does it seem that Bitcask merging is only triggered when a Riak node is restarted?** + There have been situations where the data directory for a Riak node (e.g. `data/bitcask`) grows continually and does not seem to merge. After restarting the node a series of merges are kicked off and the total size of the data directory shrinks. Why does this happen? + + +**A:** + Riak and Bitcask are operating normally. Bitcask's merge behavior is as follows: + + 1. List all of the data files in the Bitcask directory; it should be noted that a Bitcask directory exists for every vnode (e.g. `data/bitcask/0`) + 2. Remove the currently active file from the list; the active file is the one being actively written + 3. Lookup file stats for each data file; this includes percent fragmentation and number of dead bytes + 4. If any of the stats exceed the defined triggers, the Bitcask directory is merged + + The default triggers for a Bitcask directory: + + * `{frag_merge_trigger, 60}, % >= 60% fragmentation` + * `{dead_bytes_merge_trigger, 536870912}, % Dead bytes > 512 MB` + + In the described scenario, merging has not occurred because none of the data files have triggered the merge. After restarting the node, however, the previously active file is now included in the merge trigger analysis and triggers a merge on the Bitcask directory. + + If Riak was never restarted, the merge would eventually happen when writes roll over to a new data file. Bitcask rolls writes over to a new data file once the currently active file has exceeded a certain size (2 GB by default). + + +--- + +**Q: When retrieving a list of siblings I am getting the same vtag multiple times.** + When retrieving a list of siblings via the REST interface, I am seeing the same vtag appear multiple times. Is this normal? I thought vtags were unique. Are they referring to the same sibling? + + +**A:** + The vtag is calculated on a `PUT` based on the vclock and is stored as part of the object's metadata. + + It is possible to get siblings with the same vtag during vector clock pruning and read/repair. + + See [vector clocks] for more information. + + + +--- + +**Q: How should I structure larger data objects?** + I have a data object that is denormalized, with multiple child data objects, and stored as a nested JSON hash. However, retrieving and storing this object becomes increasingly costly as my application modifies and adds pieces to the object. Would breaking the object into smaller pieces improve performance? What are the tradeoffs? + + +**A:** + The factors involved in deciding whether or not to break this large object into multiple pieces are more concerned with conceptual structure than performance, although performance will be affected. Those factors include: + + 1. How tightly coupled are the child objects to the parent? That is, are they frequently updated at the same time? + 2. How likely are the objects to be updated at the same time by multiple processes? + + If the parent and child objects are not too tightly coupled (or the children are updated much more frequently), then splitting them along conceptual boundaries will improve performance in your application by decreasing payload size and reducing update conflicts. Generally, you will want to add links to connect the objects for easy fetching and traversal. + + +--- + +**Q: Is there any way in Riak to limit access to a user or a group of users?** + + +**A:** + Allowing multiple users, also known as multitenancy, is not built into Riak (though it is built into [Riak CS]). Riak has no built-in authentication. + + If you need to restrict access, consider putting an authenticating reverse-proxy server in front of it. + + +--- + +**Q: Is there a way to enforce a schema on data in a given bucket?** + Suppose I'd like to set up a bucket to store data adhering to a particular schema. Is there any way to set this up with Riak? This way, when my application attempts to store data in a particular bucket, it will check with this schema first before storing it. Otherwise, it will produce an error. + + +**A:** + Riak does not implement any form of schema validation. A pre-commit hook can be used in this scenario but would need to be written by your development team. You can read more about [commit hooks] in the docs. This document provides two pre-commit hook examples, one in Erlang that restricts objects that are too large and one in Javascript that restricts non-JSON content. + + +--- + +**Q: How does the Erlang Riak Client manage node failures?** + Does the Erlang Riak Client manage its own reconnect logic? What should a client do to maintain the connection or reconnect in case of nodes going down? + + +**A:** + The [Erlang Riak Client] gives you several options for how to manage connections. You can set these when starting a `riakc_pb_socket` process or by using the `set_options` function. + + * `queue_if_disconnected` (default: `false`) --- requests will be queued when the connection to the server is lost. + * `auto_reconnect` (default: `false`) --- if the connection is lost, `riakc_pb_socket` will attempt to reconnect automatically. This is set to `true` if `queue_if_disconnected` is set to `true`. + + If these options are both false, connection errors will be returned to the process-making requests as `{error, Reason}` tuples. + + +--- + +**Q: Is there a limiting factor for the number of buckets in a cluster?** + + +**A:** + As long as you use the default bucket properties, buckets consume no resources. Each bucket with non-default bucket properties is stored in the gossiped ring state, so the more buckets with custom properties, the more ring data must be handed off to every node. + + More on [Bucket Properties]. + + +--- + +**Q: Is it possible to configure a single bucket's properties in `app.config`?** + + +**A:** + Not a specific bucket, only the defaults. However, you should only need to change them once, since after that the settings will be reflected in the ring state. + + You can read more on `app.config` in [Configuration Files]. + + +--- + +**Q: Is there a simple command to delete a bucket?** + + +**A:** + There is no straightforward command to delete an entire bucket. You must delete all of the key/value objects individually. Thus, the following will not work: + + ```curl + curl -X DELETE http://your-host:8098/riak/your-bucket + ``` + + +--- + +**Q: Can Riak be configured to fail an update instead of generating a conflict?** + + +**A:** + No. The closest thing would be to use the `If-None-Match` header, but that is only supported in the HTTP interface and probably won't accomplish what you're trying to do. + + +--- + +**Q: How can I limit the number of keys retrieved?** + + +**A:** + You'll need to use a [MapReduce] job for this. + + You could also run `keys=stream` and close the connection when you have the designated number. This will not, however, reduce load on the Riak cluster. It will only reduce load on your client. + + +--- + +**Q: How is the real hash value for replicas calculated based on the preflist?** + + +**A:** + The hash is calculated first and then the next subsequent *N* partitions are chosen for the preflist. + + +--- + +**Q: Do client libraries support load balancing/round robin?** + + +**A:** + + * The Riak Ruby client has failure-aware load balancing. It will round-robin unless there are network errors, in which case other nodes will be preferred. + * The Java client is strictly round robin, but with retries built in. + * The Python client also follows round robin without retries. + * The Erlang client does not support any load balancing. + +## MapReduce + + +**Q: Does the number of keys in a bucket affect the performance of MapReduce?** + + +**A:** + Yes. In general, the smaller the number of keys a bucket holds, the faster MapReduce operations will run. + + +--- + +**Q: How do I filter out `not_found` from MapReduce results?** + If I want to filter out the `not_found` in my MapReduce, should I do it in the reduce phase? I have a MapReduce job that returns what I'm looking for, but I want to filter out the `not_found` entries so that I only get a list back with the keys. + + +**A:** + There is a built-in function for this that ships with Riak. Check out `Riak.filterNotFound` from the [built-in functions list]. + + +--- + +**Q: Is it possible to call a reduce function at specific intervals during a map function?** + When doing the map step on a whole bucket, can I choose how many keys to map before calling the reduce? I am generating a lot of data in memory and it could be reduced if I could call the following reduce step more often. + + +**A:** + Not currently. The reduce function is run occasionally as the bucket is processed and MapReduce doesn't wait for the whole map process to finish before running the reduce. + + +--- + +**Q: When searching over a bucket using MapReduce, is it recommended to perform the search during the map phase or the reduce phase?** + + +**A:** + Aside from the performance considerations of doing a full-bucket [MapReduce], searching is a form of filtering, which should be done in the map phase. + + +--- + +**Q: Is it possible to delete data from Riak with a JavaScript MapReduce job?** + + +**A:** + This is not currently possible. If you want to delete objects from MapReduce, use an Erlang reduce phase like the one on [contrib.basho.com]. + + +--- + +**Q: Why does MapReduce return a JSON object on occasion instead of an array?** + + +**A:** + `mochijson2` assumes that anything that looks like a proplist---a list of 2-tuples---is turned into a hash: + + ```erlang + list_to_binary(mochijson2:encode([{a , b}, {foo, bar}])). + <<"{\"a\":\"b\",\"foo\":\"bar\"}">> + ``` + + JSON has no "tuple" notion. For the time being, a recommended workaround would be to use a list of length-2 lists. diff --git a/content/riak/kv/2.2.6/developing/getting-started.md b/content/riak/kv/2.2.6/developing/getting-started.md new file mode 100644 index 0000000000..7f99bcb67a --- /dev/null +++ b/content/riak/kv/2.2.6/developing/getting-started.md @@ -0,0 +1,46 @@ +--- +title: "Getting Started Overview" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Getting Started" + identifier: "developing_getting_started" + weight: 100 + parent: "developing" +toc: true +--- + +[install index]: {{}}riak/kv/2.2.6/setup/installing +[dev client libraries]: {{}}riak/kv/2.2.6/developing/client-libraries + +Welcome, new Riak developer! This guide will get you started developing +against Riak KV with minimal fuss. + +## Installing Riak KV + +The easiest way to get started with Riak KV is to complete the +[installation][install index] process. + +## Choose Your Programming Language + +Basho officially supports a number of open-source [client libraries][dev client libraries] +for various programming languages and environments. Please select the +language with which you'd like to proceed: + +
    +
  • +
  • +
  • +
  • +
  • +
  • +
  • +
  • +
+ +### Community-supported Client Libraries + +Please see our [client libraries][dev client libraries] page for a listing of +community-supported clients. diff --git a/content/riak/kv/2.2.6/developing/getting-started/csharp.md b/content/riak/kv/2.2.6/developing/getting-started/csharp.md new file mode 100644 index 0000000000..a7cb2725bb --- /dev/null +++ b/content/riak/kv/2.2.6/developing/getting-started/csharp.md @@ -0,0 +1,82 @@ +--- +title: "Getting Started with C Sharp" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "C Sharp" + identifier: "getting_started_csharp" + weight: 103 + parent: "developing_getting_started" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/taste-of-riak/csharp + - /riak-docs/riak/kv/2.2.6/dev/taste-of-riak/csharp +--- + + + +If you haven't set up a Riak Node and started it, please visit [Running A Cluster]({{}}riak/kv/2.2.6/using/running-a-cluster) first. + +To try this flavor of Riak, a working installation of the .NET Framework or Mono is required. + +### Client Setup + +Install [the Riak .NET Client](https://github.com/basho/riak-dotnet-client/wiki/Installation) through [NuGet](http://nuget.org/packages/RiakClient) or the Visual Studio NuGet package manager. + +{{% note title="Configuring for a remote cluster" %}} +By default, the Riak .NET Client will add a section to your `app.config` file +for a four node local cluster. If you are using a remote cluster, open up +`app.config` and change the `hostAddress` values to point to nodes in your +remote cluster. +{{% /note %}} + +### Connecting to Riak + +Connecting to Riak with the Riak .NET Client requires creating a cluster object and then creating a new client object. + +```csharp +using System; +using RiakClient; + +namespace TasteOfRiak +{ + class Program + { + static void Main(string[] args) + { + // don't worry, we'll use this string later + const string contributors = "contributors"; + IRiakEndpoint cluster = RiakCluster.FromConfig("riakConfig"); + IRiakClient client = cluster.CreateClient(); + } + } +} +``` + +This creates a new `RiakCluster` which is used to create a new `RiakClient`. A `RiakCluster` object handles all the details of tracking active nodes and also provides load balancing. The `RiakClient` is used to send commands to Riak. *Note:* the `IRiakEndpoint` object implements `IDisposable` and should be correctly disposed when you're done communicating with Riak. + +Let's make sure the cluster is online. Add this to your `Main` method: + +```csharp +var pingResult = client.Ping(); + +if (pingResult.IsSuccess) +{ + Console.WriteLine("pong"); +} +else +{ + Console.WriteLine("Are you sure Riak is running?"); + Console.WriteLine("{0}: {1}", pingResult.ResultCode, pingResult.ErrorMessage); +} +``` + +This is some simple code to test that a node in a Riak cluster is online - we send a simple ping message. Even if the cluster isn't present, the Riak .NET Client will return a response message. It's important to check that your activity was successful by using the `IsSuccess` property and then checking any errors and result codes. + +We are now ready to start interacting with Riak. + +## Next Steps + +[CRUD Operations]({{}}riak/kv/2.2.6/developing/getting-started/csharp/crud-operations) diff --git a/content/riak/kv/2.2.6/developing/getting-started/csharp/crud-operations.md b/content/riak/kv/2.2.6/developing/getting-started/csharp/crud-operations.md new file mode 100644 index 0000000000..6ee7a34b05 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/getting-started/csharp/crud-operations.md @@ -0,0 +1,143 @@ +--- +title_supertext: "Getting Started:" +title: "CRUD Operations with C Sharp" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "CRUD Operations" + identifier: "getting_started_csharp_crud" + weight: 100 + parent: "getting_started_csharp" +toc: true +--- + +### Creating Objects In Riak + +Pinging a Riak cluster sounds like a lot of fun, but eventually someone is going to want us to do productive work. Let's create a class to represent some data and save some objects into Riak. + +The Riak .NET Client makes use of a `RiakObject` class to encapsulate Riak key/value objects. At the most basic, a `RiakObject` is responsible for identifying your object and for translating it into a format that can be easily saved to Riak. + +Add the `RiakClient.Models` namespace to your using directive. Your usings should look like this: + +```csharp +using System; +using System.Collections.Generic; +using RiakClient; +using RiakClient.Models; +``` + +Add the `Person` class to the `TasteOfRiak` namespace: + +```csharp +public class Person +{ + public string EmailAddress { get; set; } + public string FirstName { get; set; } + public string LastName { get; set; } +} +``` + +Now let's create some people! + +```csharp +var people = new[] +{ + new Person { + EmailAddress = "bashoman@basho.com", + FirstName = "Basho", + LastName = "Man" + }, + new Person { + EmailAddress = "johndoe@gmail.com", + FirstName = "John", + LastName = "Doe" + } +}; + +foreach (var person in people) +{ + var o = new RiakObject(contributors, person.EmailAddress, person); + var putResult = client.Put(o); + + if (putResult.IsSuccess) + { + Console.WriteLine("Successfully saved {1} to bucket {0}", o.Key, o.Bucket); + } + else + { + Console.WriteLine("Are you *really* sure Riak is running?"); + Console.WriteLine("{0}: {1}", putResult.ResultCode, putResult.ErrorMessage); + } +} +``` + +In this sample, we create a collection of `Person` objects and then save each `Person` to Riak. + +Before saving, we need to create a `RiakObject` that encapsulates the bucket, key, and object to be saved. Once we've created a `RiakObject` from our `Person` object, we can save it to Riak using `Client.Put()`. + +Once again, we check the response from Riak. If things are successful, you'll see a helpful message letting you know that your object has been saved to Riak. If things didn't go as planned, there will be an error message displaying the result code and a helpful error message. + +### Reading from Riak + +Let's find a person! + +```csharp +var result = client.Get(contributors, "bashoman@basho.com"); +if (result.IsSuccess) +{ + bashoman = result.Value.GetObject(); + Console.WriteLine("I found {0} in {1}", bashoman.EmailAddress, contributors); +} +else +{ + Console.WriteLine("Something went wrong!"); + Console.WriteLine("{0}: {1}", result.ResultCode, result.ErrorMessage); +} +``` + +We use `RiakClient.Get` to retrieve an object from Riak. This returns a `RiakResult` which, like other RiakResults, helpfully encapsulates the communication with Riak. + +After verifying that we've been able to communicate with Riak *and* that we have a successful result, we use `GetObject` to deserialize our object. + +### Modifying Existing Data + +Let's say that Basho Man has decided to be known as Riak Man: + +```csharp +bashoman.FirstName = "Riak"; + +var o = new RiakObject(contributors, bashoman.EmailAddress, bashoman); +var updateResult = client.Put(o); +if (updateResult.IsSuccess) +{ + Console.WriteLine("Successfully updated {0} in {1}", bashoman.EmailAddress, contributors); +} +else +{ + Console.WriteLine("Something went wrong!"); + Console.WriteLine("{0}: {1}", updateResult.ResultCode, updateResult.ErrorMessage); +} +``` + +Updating an object involves creating a new `RiakObject` then using `RiakClient.Put` to save the existing object. + +### Deleting Data + +```csharp +var deleteResult = client.Delete(contributors, "johndoe@gmail.com"); +if (deleteResult.IsSuccess) +{ + Console.WriteLine("Successfully got rid of John Doe"); +} +else +{ + Console.WriteLine("Something went wrong!"); + Console.WriteLine("{0}: {1}", deleteResult.ResultCode, deleteResult.ErrorMessage); +} +``` + +Just like other operations, we check the results that have come back from Riak to make sure the object was successfully deleted. + +The Riak .NET Client has a lot of additional functionality that makes it easy to build rich, complex applications with Riak. Check out the [documentation](https://github.com/basho/riak-dotnet-client/wiki) to learn more about working with the Riak .NET Client and Riak. diff --git a/content/riak/kv/2.2.6/developing/getting-started/csharp/object-modeling.md b/content/riak/kv/2.2.6/developing/getting-started/csharp/object-modeling.md new file mode 100644 index 0000000000..95ca900cea --- /dev/null +++ b/content/riak/kv/2.2.6/developing/getting-started/csharp/object-modeling.md @@ -0,0 +1,107 @@ +--- +title_supertext: "Getting Started:" +title: "Object Modeling with C Sharp" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Object Modeling" + identifier: "getting_started_csharp_object" + weight: 102 + parent: "getting_started_csharp" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/taste-of-riak/object-modeling-csharp + - /riak-docs/riak/kv/2.2.6/dev/taste-of-riak/object-modeling-csharp +--- + +To get started, refer to [this source code][1] for the models that we'll +be using. + +To use these classes to store data, we will first have to create a user. +Then, when a user creates a message, we will append that message to one +or more timelines. If it's a private message, we'll append it to the +Recipient's `Inbox` timeline and the User's own `Sent` timeline. If it's +a group message, we'll append it to the Group's timeline, as well as to +the User's `Sent` timeline. + +#### Buckets and Keys Revisited + +Now that we've worked out how we will differentiate data in the system, +let's figure out our bucket and key names. + +The bucket names are straightforward. We can use `Users`, `Msgs`, and +`Timelines`. The key names, however, are a little more tricky. In past +examples we've used sequential integers, but this presents a problem: we +would need a secondary service to hand out these IDs. This service could +easily be a future bottleneck in the system, so let's use a natural key. +Natural keys are a great fit for key/value systems because both humans +and computers can easily construct them when needed, and most of the +time they can be made unique enough for a KV store. + +| Bucket | Key Pattern | Example Key +|:-------|:------------|:----------- +| `Users` | `` | `joeuser` +| `Msgs` | `_` | `joeuser_2014-03-06T02:05:13` +| `Timelines` | `__` | `joeuser_Sent_2014-03-06`
`marketing_group_Inbox_2014-03-06` | + +For the `Users` bucket, we can be certain that we will want each +username to be unique, so let's use the `username` as the key. + +For the `Msgs` bucket, let's use a combination of the username and the +posting UTC datetime in an [ISO 8601][iso_8601] +format. This combination gives us the pattern `_`, +which produces keys like `joeuser_2014-03-05T23:20:28`. + +Now for `Timelines`, we need to differentiate between `Inbox` and `Sent` +timelines, so we can simply add that type into the key name. We will +also want to partition each collection object into some time period, +that way the object doesn't grow too large (see note below). + +For `Timelines`, let's use the pattern `__` for +users, and `_Inbox_` for groups, which will look like +`joeuser_Sent_2014-03-06` or `marketing_group_Inbox_2014-03-05`, +respectively. + +{{% note title="Note" %}} +Riak performs best with objects under 1-2MB. Objects larger than that can hurt +performance, especially when many siblings are being created. We will cover +siblings, sibling resolution, and sibling explosions in the next chapter. +{{% /note %}} + +#### Keeping our story straight with repositories + +Now that we've figured out our object model, please refer to +[this source code][2] for the repositories that we'll be using. + +[This console application][3] exercises the code that we've written. + +The repository pattern and `TimelineManager` help with a few things: + + - It helps us to see if an object exists before creating a new one + - It keeps our buckets and key names consistent + - It provides us with a consistent interface to work with. + +While this set of repositories solves many of our problems, it is very +minimal and doesn't cover all the edge cases. For instance, what happens +if two different people try to create a user with the same username? + +We can also easily "compute" key names now, but how do we quickly look +up the last 10 messages a user sent? Many of these answers will be +application dependent. If your application shows the last 10 messages in +reverse order, for example, you may want to store that set of data in +another collection object to make lookup faster. There are drawbacks to +every solution, but we recommend seeking out the key/value-based +solution first, as it will likely be the quickest. + +So to recap, in this chapter we learned: + +* How to choose bucket names +* How to choose natural keys based on how we want to partition our data + +[1]: https://github.com/basho/taste-of-riak/tree/master/csharp/Ch03-Msgy-Schema/Models +[2]: https://github.com/basho/taste-of-riak/tree/master/csharp/Ch03-Msgy-Schema/Repositories +[3]: https://github.com/basho/taste-of-riak/blob/master/csharp/Ch03-Msgy-Schema/Program.cs +[iso_8601]: http://en.wikipedia.org/wiki/ISO_8601 + diff --git a/content/riak/kv/2.2.6/developing/getting-started/csharp/querying.md b/content/riak/kv/2.2.6/developing/getting-started/csharp/querying.md new file mode 100644 index 0000000000..04be3fb546 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/getting-started/csharp/querying.md @@ -0,0 +1,210 @@ +--- +title_supertext: "Getting Started:" +title: "Querying with C Sharp" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Querying" + identifier: "getting_started_csharp_query" + weight: 101 + parent: "getting_started_csharp" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/taste-of-riak/querying-csharp + - /riak-docs/riak/kv/2.2.6/dev/taste-of-riak/querying-csharp +--- + +## C Sharp Version Setup + +For the C# version, please download the source from GitHub by either +[cloning][taste_of_riak] the source code repository or downloading the +[current zip of the master branch][master_zip]. The code for this +chapter is in `/csharp`. Open up `TasteOfRiak.sln` in Visual Studio or +your IDE of choice. + +## A Quick Note on Querying and Schemas + +_Schemas_? Yes, we said that correctly: S-C-H-E-M-A-S. It's not a dirty +word. Even in a key/value store, you will still have a logical database +schema of how all the data relates to other data. This can be as simple +as using the same key across multiple buckets for different types of +data to having fields in your data that are related by name. These +querying methods will introduce you to some ways of laying out your data +in Riak, along with how to query it back. + +## Denormalization + +If you're coming from a relational database, the easiest way to get your +application's feet wet with NoSQL is to denormalize your data into +related chunks. For example, with a customer database, you might have +separate tables for customers, addresses, preferences, etc. In Riak, +you can denormalize all that associated data into a single object and +store it into a `Customer` bucket. You can keep pulling in associated +data until you hit one of the big denormalization walls: + +* Size Limits (objects greater than 1MB) +* Shared/Referential Data (data that the object doesn't "own") +* Differences in Access Patterns (objects that get read/written once vs. + often) + +At one of these points we will have to split the model. + +## Same Keys, Different Buckets + +The simplest way to split up data would be to use the same identity key +across different buckets. A good example of this would be a `Customer` +object, an `Order` object, and an `OrderSummaries` object that keeps +rolled up info about orders such as total, etc. You can find the source +for these POCOs in `Customer.cs`, `Order.cs` and +`OrderSummaries.cs`. Let's put some data into Riak so we can play +with it. + +```csharp +Console.WriteLine("Creating Data"); +Customer customer = CreateCustomer(); +IEnumerable orders = CreateOrders(customer); +OrderSummary orderSummary = CreateOrderSummary(customer, orders); + +Console.WriteLine("Starting Client"); +using (IRiakEndPoint endpoint = RiakCluster.FromConfig("riakConfig")) +{ + IRiakClient client = endpoint.CreateClient(); + + Console.WriteLine("Storing Data"); + + client.Put(ToRiakObject(customer)); + + foreach (Order order in orders) + { + // NB: this adds secondary index data as well + client.Put(ToRiakObject(order)); + } + + client.Put(ToRiakObject(orderSummary)); + + ... + ... + ... +} +``` + +While individual `Customer` and `Order` objects don't change much (or +shouldn't change), the `OrderSummaries` object will likely change often. +It will do double duty by acting as an index for all a customer's +orders, and also holding some relevant data such as the order total, +etc. If we showed this information in our application often, it's only +one extra request to get all the info. + +```csharp +Console.WriteLine("Fetching related data by shared key"); +string key = "1"; + +var result = client.Get(customersBucketName, key); +CheckResult(result); +Console.WriteLine("Customer 1: {0}\n", GetValueAsString(result)); + +result = client.Get(orderSummariesBucketName, key); +CheckResult(result); +Console.WriteLine("OrderSummary 1: {0}\n", GetValueAsString(result)); +``` + +Which returns our amalgamated objects: + +```bash +Fetching related data by shared key +Customer 1: {"CustomerId":1,"Name":"John Smith","Address":"123 Main Street","City":"Columbus","State":"Ohio","Zip":"43210","Phone":"+1-614-555-5555","CreatedDate":"2013-10-01 14:30:26"} +OrderSummary 1: {"CustomerId":1,"Summaries":[{"OrderId":1,"Total":415.98,"OrderDate":"2013-10-01 14:42:26"},{"OrderId":2,"Total":359.99,"OrderDate":"2013-10-15 16:43:16"},{"OrderId":3,"Total":74.98,"OrderDate":"2013-11-03 17:45:28"}]} +``` + +While this pattern is very easy and extremely fast with respect to +queries and complexity, it's up to the application to know about these +intrinsic relationships. + +## Secondary Indexes + +{{% note %}} +Secondary indexes in Riak KV require a sorted backend: [Memory]({{}}riak/kv/2.2.6/setup/planning/backend/memory) or [LevelDB]({{}}riak/kv/2.2.6/setup/planning/backend/leveldb). [Bitcask]({{}}riak/kv/2.2.6/setup/planning/backend/bitcask) does not support secondary indexes. + +See [Using Secondary Indexes (2i)]({{}}riak/kv/2.2.6/developing/usage/secondary-indexes) for more information on developing with secondary indexes. +{{% /note %}} + +If you're coming from an SQL world, Secondary Indexes (2i) are a lot +like SQL indexes. They are a way to quickly look up objects based on a +secondary key, without scanning through the whole dataset. This makes it +very easy to find groups of related data by values, or even ranges of +values. To properly show this off, we will make a note of where +secondary index data is added to our model objects. + +```csharp +private static RiakObject ToRiakObject(Order order) +{ + var orderRiakObjectId = new RiakObjectId(ordersBucketName, order.Id.ToString()); + var riakObject = new RiakObject(orderRiakObjectId, order); + + IntIndex salesPersonIdIndex = riakObject.IntIndex(ordersSalesPersonIdIndexName); + salesPersonIdIndex.Add(order.SalesPersonId.ToString()); + + BinIndex orderDateIndex = riakObject.BinIndex(ordersOrderDateIndexName); + orderDateIndex.Add(order.OrderDate.ToString("yyyy-MM-dd")); + + return riakObject; +} +``` + +As you may have noticed, ordinary key/value data is opaque to 2i, so we +have to add entries to the indexes at the application level. Now let's +find all of Jane Appleseed's processed orders, we'll look up the orders +by searching the `SalespersonId` integer index for Jane's id of `9000`. + +```csharp +// Query for order keys where the SalesPersonId index is set to 9000 +var riakIndexId = new RiakIndexId(ordersBucketName, ordersSalesPersonIdIndexName); +RiakResult indexRiakResult = client.GetSecondaryIndex(riakIndexId, 9000); // NB: *must* use 9000 as integer here. +CheckResult(indexRiakResult); +RiakIndexResult indexResult = indexRiakResult.Value; +Console.WriteLine("Jane's orders (key values): {0}", string.Join(", ", indexResult.IndexKeyTerms.Select(ikt => ikt.Key))); +``` + +Which returns: + +```text +Jane's orders (key values): 1, 3 +``` + +Jane processed orders 1 and 3. We used an "integer" index to reference +Jane's ID, next let's use a "binary" index. Now, let's say that the VP +of Sales wants to know how many orders came in during October 2013. In +this case, we can exploit 2i's range queries. Let's search the +`OrderDate` binary index for entries between `2013-10-01` and +`2013-10-31`. + +```csharp +// Query for orders where the OrderDate index is between 2013-10-01 and 2013-10-31 +riakIndexId = new RiakIndexId(ordersBucketName, ordersOrderDateIndexName); +indexRiakResult = client.GetSecondaryIndex(riakIndexId, "2013-10-01", "2013-10-31"); // NB: *must* use strings here. +CheckResult(indexRiakResult); +indexResult = indexRiakResult.Value; +Console.WriteLine("October orders (key values): {0}", string.Join(", ", indexResult.IndexKeyTerms.Select(ikt => ikt.Key))); +``` + +Which returns: + +```text +October orders (key values): 1, 2 +``` + +We used 2i's range feature to search for a range of values, and demonstrated binary indexes. + +So to recap: + +* You can use Secondary Indexes to quickly look up an object based on a + secondary id other than the object's key. +* Indexes can have either Integer or Binary(String) keys +* You can search for specific values, or a range of values +* Riak will return a list of keys that match the index query + + +[taste_of_riak]: https://github.com/basho/taste-of-riak +[master_zip]: https://github.com/basho/taste-of-riak/archive/master.zip diff --git a/content/riak/kv/2.2.6/developing/getting-started/erlang.md b/content/riak/kv/2.2.6/developing/getting-started/erlang.md new file mode 100644 index 0000000000..c559d44384 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/getting-started/erlang.md @@ -0,0 +1,55 @@ +--- +title: "Getting Started with Erlang" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Erlang" + identifier: "getting_started_erlang" + weight: 105 + parent: "developing_getting_started" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/taste-of-riak/erlang + - /riak-docs/riak/kv/2.2.6/dev/taste-of-riak/erlang +--- + +If you haven't set up a Riak Node and started it, please visit [Running A Cluster]({{}}riak/kv/2.2.6/using/running-a-cluster) first. + +To try this flavor of Riak, a working installation of Erlang is +required. You can also use the `erts` Erlang installation that comes +with Riak. + +## Client Setup + +Download the latest Erlang client from GitHub +([zip](https://github.com/basho/riak-erlang-client/archive/master.zip), +[GitHub repository](https://github.com/basho/riak-erlang-client/)) and +extract it to your working directory. + +Next, open the Erlang console with the client library paths included. + +```bash +erl -pa CLIENT_LIBRARY_PATH/ebin/ CLIENT_LIBRARY_PATH/deps/*/ebin +``` + +Now let’s create a link to the Riak node. If you are using a single +local Riak node, use the following to create the link: + +```erlang +{ok, Pid} = riakc_pb_socket:start("127.0.0.1", 8087). +``` + +If you set up a local Riak cluster using the [[five-minute install]] +method, use this code snippet instead: + +```erlang +{ok, Pid} = riakc_pb_socket:start_link("127.0.0.1", 10017). +``` + +We are now ready to start interacting with Riak. + +## Next Steps + +[CRUD Operations]({{}}riak/kv/2.2.6/developing/getting-started/erlang/crud-operations) diff --git a/content/riak/kv/2.2.6/developing/getting-started/erlang/crud-operations.md b/content/riak/kv/2.2.6/developing/getting-started/erlang/crud-operations.md new file mode 100644 index 0000000000..6ee9556b79 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/getting-started/erlang/crud-operations.md @@ -0,0 +1,167 @@ +--- +title_supertext: "Getting Started:" +title: "CRUD Operations with Erlang" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "CRUD Operations" + identifier: "getting_started_erlang_crud" + weight: 100 + parent: "getting_started_erlang" +toc: true +--- + +## Creating Objects In Riak + +First, let’s create a few Riak objects. For these examples we'll be +using the bucket `test`. + +```erlang +MyBucket = <<"test">>. + +Val1 = 1. +Obj1 = riakc_obj:new(MyBucket, <<"one">>, Val1). +riakc_pb_socket:put(Pid, Obj1). +``` + +In this first example, we have stored the integer 1 with the lookup key +of `one`. Next, let’s store a simple string value of `two` with a +matching key. + +```erlang +Val2 = <<"two">>. +Obj2 = riakc_obj:new(MyBucket, <<"two">>, Val2). +riakc_pb_socket:put(Pid, Obj2). +``` + +That was easy. Finally, let’s store something more complex, a tuple this +time. You will probably recognize the pattern by now. + +```erlang +Val3 = {value, 3}. +Obj3 = riakc_obj:new(MyBucket, <<"three">>, Val3). +riakc_pb_socket:put(Pid, Obj3). +``` + +## Reading Objects From Riak + +Now that we have a few objects stored, let’s retrieve them and make sure +they contain the values we expect. + +```erlang +{ok, Fetched1} = riakc_pb_socket:get(Pid, MyBucket, <<"one">>). +{ok, Fetched2} = riakc_pb_socket:get(Pid, MyBucket, <<"two">>). +{ok, Fetched3} = riakc_pb_socket:get(Pid, MyBucket, <<"three">>). + +Val1 =:= binary_to_term(riakc_obj:get_value(Fetched1)). %% true +Val2 =:= riakc_obj:get_value(Fetched2). %% true +Val3 =:= binary_to_term(riakc_obj:get_value(Fetched3)). %% true +``` + +That was easy. We simply request the objects by bucket and key. + +## Updating Objects In Riak + +While some data may be static, other forms of data may need to be +updated. This is also easy to do. Let’s update the value in the third +example to 42, update the Riak object, and then save it. + +```erlang +NewVal3 = setelement(2, Val3, 42). +UpdatedObj3 = riakc_obj:update_value(Fetched3, NewVal3). +{ok, NewestObj3} = riakc_pb_socket:put(Pid, UpdatedObj3, [return_body]). +``` + +We can verify that our new value was saved by looking at the value +returned. + +```erlang +rp(binary_to_term(riakc_obj:get_value(NewestObj3))). +``` + +## Deleting Objects From Riak + +Nothing is complete without a delete, as they say. Fortunately, that's +easy too. + +```erlang +riakc_pb_socket:delete(Pid, MyBucket, <<"one">>). +riakc_pb_socket:delete(Pid, MyBucket, <<"two">>). +riakc_pb_socket:delete(Pid, MyBucket, <<"three">>). +``` + +Now we can verify that the objects have been removed from Riak. + +```erlang +{error,notfound} =:= riakc_pb_socket:get(Pid, MyBucket, <<"one">>). +{error,notfound} =:= riakc_pb_socket:get(Pid, MyBucket, <<"two">>). +{error,notfound} =:= riakc_pb_socket:get(Pid, MyBucket, <<"three">>). +``` + +## Working With Complex Objects + +Since the world is a little more complicated than simple integers and +bits of strings, let’s see how we can work with more complex objects. +Take, for example, this record that encapsulates some information about +a book. + +```erlang +rd(book, {title, author, body, isbn, copies_owned}). + +MobyDickBook = #book{title="Moby Dick", + isbn="1111979723", + author="Herman Melville", + body="Call me Ishmael. Some years ago...", + copies_owned=3}. +``` + +So we have some information about our Moby Dick collection that we want +to save. Storing this to Riak should look familiar by now: + +```erlang +MobyObj = riakc_obj:new(<<"books">>, + list_to_binary(MobyDickBook#book.isbn), + MobyDickBook). + +riakc_pb_socket:put(Pid, MobyObj). +``` + +Some of you may be thinking: "How does the Erlang Riak client +encode/decode my object?" If we fetch our book back and print the value, +we shall know: + +```erlang +{ok, FetchedBook} = riakc_pb_socket:get(Pid, + <<"books">>, + <<"1111979723">>). + +rp(riakc_obj:get_value(FetchedBook)). +``` + +The response: + +``` +<<131,104,6,100,0,4,98,111,111,107,107,0,9,77,111,98,121, + 32,68,105,99,107,107,0,15,72,101,114,109,97,110,32,77, + 101,108,118,105,108,108,101,107,0,34,67,97,108,108,32, + 109,101,32,73,115,104,109,97,101,108,46,32,83,111,109, + 101,32,121,101,97,114,115,32,97,103,111,46,46,46,107,0, + 10,49,49,49,49,57,55,57,55,50,51,97,3>> +``` + +Erlang binaries! The Riak Erlang client library encodes everything as +binaries. If we wanted to get a `book` object back we could use +`binary_to_term/1` to get our original object back: + +```erlang +rp(binary_to_term(riakc_obj:get_value(FetchedBook))). +``` + +Next let’s clean up our mess: + +```erlang +riakc_pb_socket:delete(Pid, <<"books">>, <<"1111979723">>). +riakc_pb_socket:stop(Pid). +``` diff --git a/content/riak/kv/2.2.6/developing/getting-started/erlang/object-modeling.md b/content/riak/kv/2.2.6/developing/getting-started/erlang/object-modeling.md new file mode 100644 index 0000000000..d1c3a543bf --- /dev/null +++ b/content/riak/kv/2.2.6/developing/getting-started/erlang/object-modeling.md @@ -0,0 +1,338 @@ +--- +title_supertext: "Getting Started:" +title: "Object Modeling with Erlang" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Object Modeling" + identifier: "getting_started_erlang_object" + weight: 102 + parent: "getting_started_erlang" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/taste-of-riak/object-modeling-erlang + - /riak-docs/riak/kv/2.2.6/dev/taste-of-riak/object-modeling-erlang +--- + +To get started, let's create the records that we'll be using. + +{{% note title="Code Download" %}} +You can also download the code for this chapter at +[Github](https://github.com/basho/taste-of-riak/tree/am-dem-erlang-modules/erlang/Ch03-Msgy-Schema). + +The Github version includes Erlang type specifications which have been omitted +here for brevity. +{{% /note %}} + + +```erlang +%% msgy.hrl + +-define(USER_BUCKET, <<"Users">>). +-define(MSG_BUCKET, <<"Msgs">>). +-define(TIMELINE_BUCKET, <<"Timelines">>). +-define(INBOX, "Inbox"). +-define(SENT, "Sent"). + +-record(user, {user_name, full_name, email}). + +-record(msg, {sender, recipient, created, text}). + +-record(timeline, {owner, msg_type, msgs}). +``` + +We'll be using the bucket `Users` to store our data. We won't be [using bucket types]({{}}riak/kv/2.2.6/developing/usage/bucket-types) here, so we don't need to specify one. + +To use these records to store data, we will first have to create a user +record. Then, when a user creates a message, we will append that message +to one or more timelines. If it's a private message, we'll append it to +the Recipient's `Inbox` timeline and to the User's own `Sent` timeline. +If it's a group message, we'll append it to the Group's timeline, as +well as to the User's `Sent` timeline. + +#### Buckets and keys revisited + +Now that we've worked out how we will differentiate data in the system, +let's figure out our bucket and key names. + +The bucket names are straightforward. We can use `Users`, `Msgs`, and +`Timelines`. The key names, however, are a little more tricky. In past +examples we've used sequential integers, but this presents a problem: we +would need a secondary service to hand out these IDs. This service could +easily be a future bottleneck in the system, so let's use a natural key. +Natural keys are a great fit for key/value systems because both humans +and computers can easily construct them when needed, and most of the +time they can be made unique enough for a KV store. + + +Bucket | Key Pattern | Example Key +:------|:------------|:----------- +`Users` | `` | `joeuser` +`Msgs` | `_` | `joeuser_2014-03-06T02:05:13.223556Z` +`Timelines` | `__` | `joeuser_Sent_2014-03-06Z`
`marketing_group_Inbox_2014-03-06Z` | + +For the `Users` bucket, we can be certain that we will want each +username to be unique, so let's use the `username` as the key. For the +`Msgs` bucket, let's use a combination of the username and the posting +datetime in an [ISO 8601 Long](http://en.wikipedia.org/wiki/ISO_8601) +format. This combination gives us the pattern `_`, +which produces keys like `joeuser_2014-03-05T23:20:28Z`. + +Now for `Timelines`, we need to differentiate between `Inbox` and `Sent` +timelines, so we can simply add that type into the key name. We will +also want to partition each collection object into some time period, +that way the object doesn't grow too large (see note below). + +For `Timelines`, let's use the pattern `__` for +users, and `_Inbox_` for groups, which will look like +`joeuser_Sent_2014-03-06Z` or `marketing_group_Inbox_2014-03-05Z`, +respectively. + +{{% note title="Note" %}} +Riak performs best with objects under 1-2 MB. Objects larger than that can +hurt performance, especially if many siblings are being created. We will cover +siblings, sibling resolution, and sibling explosions in the next chapter. +{{% /note %}} + +#### Keeping our story straight with repositories + +Now that we've figured out our object model, let's write some modules to +act as repositories that will help us create and work with these records +in Riak: + +```erlang +%% user_repository.erl + +-module(user_repository). +-export([save_user/2, + get_user/2]). +-include("msgy.hrl"). + +save_user(ClientPid, User) -> + RUser = riakc_obj:new(?USER_BUCKET, + list_to_binary(User#user.user_name), + User), + riakc_pb_socket:put(ClientPid, RUser). + +get_user(ClientPid, UserName) -> + {ok, RUser} = riakc_pb_socket:get(ClientPid, + ?USER_BUCKET, + list_to_binary(UserName)), + binary_to_term(riakc_obj:get_value(RUser)). +``` + +
+ +```erlang +%% msg_repository.erl + +-module(msg_repository). +-export([create_msg/3, + get_msg/2]). +-include("msgy.hrl"). + +-spec create_msg(user_name(), user_name(), text()) -> msg(). +create_msg(Sender, Recipient, Text) -> + #msg{sender=Sender, + recipient=Recipient, + created=get_current_iso_timestamp(), + text = Text}. + +-spec get_msg(pid(), riakc_obj:key()) -> msg(). +get_msg(ClientPid, MsgKey) -> + {ok, RMsg} = riakc_pb_socket:get(ClientPid, + ?MSG_BUCKET, + MsgKey), + binary_to_term(riakc_obj:get_value(RMsg)). + +%% @private +-spec get_current_iso_timestamp() -> datetimestamp(). +get_current_iso_timestamp() -> + {_,_,MicroSec} = DateTime = erlang:now(), + {{Year,Month,Day},{Hour,Min,Sec}} = calendar:now_to_universal_time(DateTime), + lists:flatten( + io_lib:format("~4..0B-~2..0B-~2..0BT~2..0B:~2..0B:~2..0B.~6..0B", + [Year, Month, Day, Hour, Min, Sec, MicroSec])). + +``` + +
+ +```erlang +%% timeline_repository.erl + +-module(timeline_repository). +-export([post_msg/2, + get_timeline/4]). +-include("msgy.hrl"). + +post_msg(ClientPid, Msg) -> + %% Save the canonical copy + SavedMsg = save_msg(ClientPid, Msg), + MsgKey = binary_to_list(riakc_obj:key(SavedMsg)), + + %% Post to sender's Sent timeline + add_to_timeline(ClientPid, Msg, sent, MsgKey), + + %% Post to recipient's Inbox timeline + add_to_timeline(ClientPid, Msg, inbox, MsgKey), + ok. + +get_timeline(ClientPid, Owner, MsgType, Date) -> + TimelineKey = generate_key(Owner, MsgType, Date), + {ok, RTimeline} = riakc_pb_socket:get(ClientPid, + ?TIMELINE_BUCKET, + list_to_binary(TimelineKey)), + binary_to_term(riakc_obj:get_value(RTimeline)). + +%% -------------------------------------------------------------------- + +%% @private +save_msg(ClientPid, Msg) -> + MsgKey = Msg#msg.sender ++ "_" ++ Msg#msg.created, + ExistingMsg = riakc_pb_socket:get(ClientPid, + ?MSG_BUCKET, + list_to_binary(MsgKey)), + SavedMsg = case ExistingMsg of + {error, notfound} -> + NewMsg = riakc_obj:new(?MSG_BUCKET, list_to_binary(MsgKey), Msg), + {ok, NewSaved} = riakc_pb_socket:put(ClientPid, + NewMsg, + [if_none_match, return_body]), + NewSaved; + {ok, Existing} -> Existing + end, + SavedMsg. + +%% @private +add_to_timeline(ClientPid, Msg, MsgType, MsgKey) -> + TimelineKey = generate_key_from_msg(Msg, MsgType), + ExistingTimeline = riakc_pb_socket:get(ClientPid, + ?TIMELINE_BUCKET, + list_to_binary(TimelineKey)), + UpdatedTimeline = case ExistingTimeline of + {error, notfound} -> + create_new_timeline(Msg, MsgType, MsgKey, TimelineKey); + {ok, Existing} -> + add_to_existing_timeline(Existing, MsgKey) + end, + + {ok, SavedTimeline} = riakc_pb_socket:put(ClientPid, + UpdatedTimeline, + [return_body]), + SavedTimeline. + +%% @private +create_new_timeline(Msg, MsgType, MsgKey, TimelineKey) -> + Owner = get_owner(Msg, MsgType), + Timeline = #timeline{owner=Owner, + msg_type=MsgType, + msgs=[MsgKey]}, + riakc_obj:new(?TIMELINE_BUCKET, list_to_binary(TimelineKey), Timeline). + +%% @private +add_to_existing_timeline(ExistingRiakObj, MsgKey) -> + ExistingTimeline = binary_to_term(riakc_obj:get_value(ExistingRiakObj)), + ExistingMsgList = ExistingTimeline#timeline.msgs, + UpdatedTimeline = ExistingTimeline#timeline{msgs=[MsgKey|ExistingMsgList]}, + riakc_obj:update_value(ExistingRiakObj, UpdatedTimeline). + +%% @private +get_owner(Msg, inbox) -> Msg#msg.recipient; +get_owner(Msg, sent) -> Msg#msg.sender. + +%% @private +generate_key_from_msg(Msg, MsgType) -> + Owner = get_owner(Msg, MsgType), + generate_key(Owner, MsgType, Msg#msg.created). + +%% @private +generate_key(Owner, MsgType, Date) when is_tuple(Date) -> + DateString = get_iso_datestamp_from_date(Date), + generate_key(Owner, MsgType, DateString); + +generate_key(Owner, MsgType, Datetimestamp) -> + DateString = get_iso_datestamp_from_iso_timestamp(Datetimestamp), + MsgTypeString = case MsgType of + inbox -> ?INBOX; + sent -> ?SENT + end, + Owner ++ "_" ++ MsgTypeString ++ "_" ++ DateString. + +%% @private +get_iso_datestamp_from_date(Date) -> + {Year,Month,Day} = Date, + lists:flatten(io_lib:format("~4..0B-~2..0B-~2..0B", [Year, Month, Day])). + +%% @private +get_iso_datestamp_from_iso_timestamp(CreatedString) -> + {Date, _} = lists:split(10,CreatedString), + Date. + +``` + +Finally, let's test them: + +```erlang +%% msgy.erl + +-module(msgy). +-export([main/0]). +-include("msgy.hrl"). + +main() -> + %% Setup our repositories + {ok, Pid} = riakc_pb_socket:start_link("127.0.0.1", 10017), + + %% Create and save users + Joe = #user{user_name="joeuser", + full_name="Joe User", + email="joe.user@basho.com"}, + + Marleen = #user{user_name="marleenmgr", + full_name="Marleen Manager", + email="marleen.manager@basho.com"}, + + user_repository:save_user(Pid, Joe), + user_repository:save_user(Pid, Marleen), + + %% Create new Msg, post to timelines + Msg = msg_repository:create_msg(Marleen#user.user_name, Joe#user.user_name, "Welcome to the company!"), + timeline_repository:post_msg(Pid, Msg), + + + %% Get Joe's inbox for today, get first message + {TodaysDate,_} = calendar:now_to_universal_time(erlang:now()), + JoesInboxToday = timeline_repository:get_timeline(Pid, Joe#user.user_name, inbox, TodaysDate), + + JoesFirstMessage = msg_repository:get_msg(Pid, hd(JoesInboxToday#timeline.msgs)), + + io:format("From: ~s~nMsg : ~s~n~n", [JoesFirstMessage#msg.sender, JoesFirstMessage#msg.text]), + ok. +``` + +As you can see, the repository pattern helps us with a few things: + +* It helps us to see if an object exists before creating a new one +* It keeps our buckets and key names consistent +* It provides us with a consistent interface to work with. + +While this set of repositories solves many of our problems, it is very +minimal and doesn't cover all the edge cases. For instance, what happens +if two different people try to create a user with the same username? + +We can also easily "compute" key names now, but how do we quickly look +up the last 10 messages a user sent? Many of these answers will be +application dependent. If your application shows the last 10 messages in +reverse order, for example, you may want to store that set of data in +another collection object to make lookup faster. There are drawbacks to +every solution, but we recommend seeking out the key/value-based +solution first, as it will likely be the quickest. + +So to recap, in this chapter we learned: + +* How to choose bucket names +* How to choose natural keys based on how we want to partition our data. + diff --git a/content/riak/kv/2.2.6/developing/getting-started/erlang/querying.md b/content/riak/kv/2.2.6/developing/getting-started/erlang/querying.md new file mode 100644 index 0000000000..0f4ba26178 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/getting-started/erlang/querying.md @@ -0,0 +1,303 @@ +--- +title_supertext: "Getting Started:" +title: "Querying with Erlang" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Querying" + identifier: "getting_started_erlang_query" + weight: 101 + parent: "getting_started_erlang" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/taste-of-riak/querying-erlang + - /riak-docs/riak/kv/2.2.6/dev/taste-of-riak/querying-erlang +--- + +## A Quick Note on Querying and Schemas + +_Schemas_? Yes, we said that correctly: S-C-H-E-M-A-S. It's not a dirty +word. Even in a key/value store, you will still have a logical database +schema of how all the data relates to other data. This can be as simple +as using the same key across multiple buckets for different types of +data to having fields in your data that are related by name. These +querying methods will introduce you to some ways of laying out your data +in Riak, along with how to query it back. + +A more comprehensive discussion can be found in [Key/Value Modeling]({{}}riak/kv/2.2.6/developing/key-value-modeling). + +## Denormalization + +If you're coming from a relational database, the easiest way to get your +application's feet wet with NoSQL is to denormalize your data into +related chunks. For example, with a customer database, you might have +separate tables for customers, addresses, preferences, etc. In Riak, you +can denormalize all that associated data into a single object and store +it into a `Customer` bucket. You can keep pulling in associated +data until you hit one of the big denormalization walls: + +* Size limits (objects greater than 1MB) +* Shared/referential Data (data that the object doesn't "own") +* Differences in access patterns (objects that get read/written once vs. + often) + +At one of these points we will have to split the model. + +## Same Keys, Different Buckets + +The simplest way to split up data would be to use the same identity key +across different buckets. A good example of this would be a `Customer` +object, an `Order` object, and an `OrderSummaries` object that keeps +rolled up info about orders such as total, etc. + +Let's put some data into Riak so we can play with it. Fire up your +Erlang REPL with the client library in the path, and enter in the +following: + +```erlang +rd(customer, {customer_id, name, address, city, state, zip, phone, created_date}). +rd(item, {item_id, title, price}). +rd(order, {order_id, customer_id, salesperson_id, items, total, order_date}). +rd(order_summary_entry, {order_id, total, order_date}). +rd(order_summary, {customer_id, summaries}). + + +Customer = #customer{ customer_id= 1, + name= "John Smith", + address= "123 Main Street", + city= "Columbus", + state= "Ohio", + zip= "43210", + phone= "+1-614-555-5555", + created_date= {{2013,10,1},{14,30,26}}}. + +Orders = [ #order{ + order_id= 1, + customer_id= 1, + salesperson_id= 9000, + items= [ + #item{ + item_id= "TCV37GIT4NJ", + title= "USB 3.0 Coffee Warmer", + price= 15.99 }, + #item{ + item_id= "PEG10BBF2PP", + title= "eTablet Pro, 24GB, Grey", + price= 399.99 }], + total= 415.98, + order_date= {{2013,10,1},{14,42,26}}}, + + #order{ + order_id= 2, + customer_id= 1, + salesperson_id= 9001, + items= [ + #item{ + item_id= "OAX19XWN0QP", + title= "GoSlo Digital Camera", + price= 359.99 }], + total= 359.99, + order_date= {{2013,10,15},{16,43,16}}}, + + #order { + order_id= 3, + customer_id= 1, + salesperson_id= 9000, + items= [ + #item{ + item_id= "WYK12EPU5EZ", + title= "Call of Battle= Goats - Gamesphere 4", + price= 69.99 }, + #item{ + item_id= "TJB84HAA8OA", + title= "Bricko Building Blocks", + price= 4.99 }], + total= 74.98, + order_date= {{2013,11,3},{17,45,28}}} + ]. + +OrderSummary = #order_summary{ + customer_id= 1, + summaries= [ + #order_summary_entry{ + order_id= 1, + total= 415.98, + order_date= {{2013,10,1},{14,42,26}} + }, + #order_summary_entry{ + order_id= 2, + total= 359.99, + order_date= {{2013,10,15},{16,43,16}} + }, + #order_summary_entry{ + order_id= 3, + total= 74.98, + order_date= {{2013,11,3},{17,45,28}}}]}. + +## Remember to replace the ip and port parameters with those that match your cluster. +{ok, Pid} = riakc_pb_socket:start_link("127.0.0.1", 10017). + +CustomerBucket = <<"Customers">>. +OrderBucket = <<"Orders">>. +OrderSummariesBucket = <<"OrderSummaries">>. + +CustObj = riakc_obj:new(CustomerBucket, + list_to_binary( + integer_to_list( + Customer#customer.customer_id)), + Customer). + +riakc_pb_socket:put(Pid, CustObj). + +StoreOrder = fun(Order) -> + OrderObj = riakc_obj:new(OrderBucket, + list_to_binary( + integer_to_list( + Order#order.order_id)), + Order), + riakc_pb_socket:put(Pid, OrderObj) +end. + +lists:foreach(StoreOrder, Orders). + + +OrderSummaryObj = riakc_obj:new(OrderSummariesBucket, + list_to_binary( + integer_to_list( + OrderSummary#order_summary.customer_id)), + OrderSummary). + +riakc_pb_socket:put(Pid, OrderSummaryObj). + +``` + +While individual `Customer` and `Order` objects don't change much (or +shouldn't change), the `OrderSummaries` object will likely change often. +It will do double duty by acting as an index for all a customer's +orders, and also holding some relevant data such as the order total, +etc. If we showed this information in our application often, it's only +one extra request to get all the info. + +```erlang +{ok, FetchedCustomer} = riakc_pb_socket:get(Pid, + CustomerBucket, + <<"1">>). +{ok, FetchedSummary} = riakc_pb_socket:get(Pid, + OrderSummariesBucket, + <<"1">>). +rp({binary_to_term(riakc_obj:get_value(FetchedCustomer)), + binary_to_term(riakc_obj:get_value(FetchedSummary))}). +``` + +Which returns our amalgamated objects: + +```erlang +{#customer{customer_id = 1,name = "John Smith", + address = "123 Main Street",city = "Columbus", + state = "Ohio",zip = "43210",phone = "+1-614-555-5555", + created_date = {{2013,10,1},{14,30,26}}}, + #order_summary{customer_id = 1, + summaries = [#order_summary_entry{order_id = 1, + total = 415.98, + order_date = {{2013,10,1},{14,42,26}}}, + #order_summary_entry{order_id = 2,total = 359.99, + order_date = {{2013,10,15},{16,43,16}}}, + #order_summary_entry{order_id = 3,total = 74.98, + order_date = {{2013,11,3},{17,45,28}}}]}} +``` + +While this pattern is very easy and extremely fast with respect to +queries and complexity, it's up to the application to know about these +intrinsic relationships. + +## Secondary Indexes + +{{% note %}} +Secondary indexes in Riak KV require a sorted backend: [Memory]({{}}riak/kv/2.2.6/setup/planning/backend/memory) or [LevelDB]({{}}riak/kv/2.2.6/setup/planning/backend/leveldb). [Bitcask]({{}}riak/kv/2.2.6/setup/planning/backend/bitcask) does not support secondary indexes. + +See [Using Secondary Indexes (2i)]({{}}riak/kv/2.2.6/developing/usage/secondary-indexes) for more information on developing with secondary indexes. +{{% /note %}} + +If you're coming from an SQL world, Secondary Indexes (2i) are a lot +like SQL indexes. They are a way to quickly look up objects based on a +secondary key, without scanning through the whole dataset. This makes it +very easy to find groups of related data by values, or even ranges of +values. To properly show this off, we will now add some more data to our +application, and add some secondary index entries at the same time. + +```erlang +FormatDate = fun(DateTime) -> + {{Year, Month, Day}, {Hour, Min, Sec}} = DateTime, + lists:concat([Year,Month,Day,Hour,Min,Sec]) +end. + +AddIndicesToOrder = fun(OrderKey) -> + {ok, Order} = riakc_pb_socket:get(Pid, OrderBucket, + list_to_binary(integer_to_list(OrderKey))), + + OrderData = binary_to_term(riakc_obj:get_value(Order)), + OrderMetadata = riakc_obj:get_update_metadata(Order), + + MD1 = riakc_obj:set_secondary_index(OrderMetadata, + [{{binary_index, "order_date"}, + [FormatDate(OrderData#order.order_date)]}]), + + MD2 = riakc_obj:set_secondary_index(MD1, + [{{integer_index, "salesperson_id"}, + [OrderData#order.salesperson_id]}]), + + Order2 = riakc_obj:update_metadata(Order,MD2), + riakc_pb_socket:put(Pid,Order2) +end. + +lists:foreach(AddIndicesToOrder, [1,2,3]). + +``` + +As you may have noticed, ordinary Key/Value data is opaque to 2i, so we +have to add entries to the indices at the application level. Now let's +find all of Jane Appleseed's processed orders, we'll lookup the orders +by searching the `saleperson_id_int` index for Jane's id of `9000`. + +```erlang +riakc_pb_socket:get_index_eq(Pid, OrderBucket, {integer_index, "salesperson_id"}, 9000). +``` + +Which returns: + +```erlang +{ok,{index_results_v1,[<<"1">>,<<"3">>], + undefined,undefined}} +``` + +Jane processed orders 1 and 3. We used an "integer" index to reference +Jane's id, next let's use a "binary" index. Now, let's say that the VP +of Sales wants to know how many orders came in during October 2013. In +this case, we can exploit 2i's range queries. Let's search the +`order_date_bin` index for entries between `20131001` and `20131031`. + +```erlang +riakc_pb_socket:get_index_range(Pid, OrderBucket, + {binary_index, "order_date"}, + <<"20131001">>, <<"20131031">>). +``` + +Which returns: + +```erlang +{ok,{index_results_v1,[<<"1">>,<<"2">>], + undefined,undefined}} +``` + +Boom! Easy-peasy. We used 2i's range feature to search for a range of +values, and demonstrated binary indexes. + +So, to recap: + +* You can use Secondary Indexes to quickly lookup an object based on a + secondary id other than the object's key. +* Indices can have either Integer or Binary(String) keys +* You can search for specific values, or a range of values +* Riak will return a list of keys that match the index query diff --git a/content/riak/kv/2.2.6/developing/getting-started/golang.md b/content/riak/kv/2.2.6/developing/getting-started/golang.md new file mode 100644 index 0000000000..14bd669d05 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/getting-started/golang.md @@ -0,0 +1,78 @@ +--- +title: "Getting Started with Go" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Go" + identifier: "getting_started_go" + weight: 107 + parent: "developing_getting_started" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/taste-of-riak/golang + - /riak-docs/riak/kv/2.2.6/dev/taste-of-riak/golang +--- + +If you haven't set up a Riak Node and started it, please visit [Running A Cluster]({{}}riak/kv/2.2.6/using/running-a-cluster) first and ensure you have +[a working installation of Go](http://golang.org/doc/install). + +## Client Setup + +First install the [Riak Go client](https://github.com/basho/riak-go-client): + +```bash +go get github.com/basho/riak-go-client +``` + +Next download the [Taste of Riak - Go](https://github.com/basho/taste-of-riak/tree/master/go) utilities: + +```bash +go get github.com/basho/taste-of-riak/go/util +``` + +If you are using a single local Riak node, use the following to create a +new client instance: + +```golang +package main + +import ( + "encoding/binary" + "encoding/json" + "sync" + + riak "github.com/basho/riak-go-client" + util "github.com/basho/taste-of-riak/go/util" +) + +func main() { + var err error + + // un-comment-out to enable debug logging + // riak.EnableDebugLogging = true + + o := &riak.NewClientOptions{ + RemoteAddresses: []string{util.GetRiakAddress()}, + } + + var c *riak.Client + c, err = riak.NewClient(o) + if err != nil { + util.ErrExit(err) + } + + defer func() { + if err := c.Stop(); err != nil { + util.ErrExit(err) + } + }() +} +``` + +We are now ready to start interacting with Riak. + +## Next Steps + +[CRUD Operations]({{}}riak/kv/2.2.6/developing/getting-started/golang/crud-operations) diff --git a/content/riak/kv/2.2.6/developing/getting-started/golang/crud-operations.md b/content/riak/kv/2.2.6/developing/getting-started/golang/crud-operations.md new file mode 100644 index 0000000000..afe9ee80ea --- /dev/null +++ b/content/riak/kv/2.2.6/developing/getting-started/golang/crud-operations.md @@ -0,0 +1,370 @@ +--- +title_supertext: "Getting Started:" +tiGetting Started: CRUD Operations with Go" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "CRUD Operations" + identifier: "getting_started_go_crud" + weight: 100 + parent: "getting_started_go" +toc: true +--- + +## Creating Objects + +First let’s create a few objects and a bucket to keep them in: + +```golang + val1 := uint32(1) + val1buf := make([]byte, 4) + binary.LittleEndian.PutUint32(val1buf, val1) + + val2 := "two" + + val3 := struct{ MyValue int }{3} // NB: ensure that members are exported (i.e. capitalized) + var val3json []byte + val3json, err = json.Marshal(val3) + if err != nil { + util.ErrExit(err) + } + + bucket := "test" + + util.Log.Println("Creating Objects In Riak...") + + objs := []*riak.Object{ + { + Bucket: bucket, + Key: "one", + ContentType: "application/octet-stream", + Value: val1buf, + }, + { + Bucket: bucket, + Key: "two", + ContentType: "text/plain", + Value: []byte(val2), + }, + { + Bucket: bucket, + Key: "three", + ContentType: "application/json", + Value: val3json, + }, + } + + var cmd riak.Command + wg := &sync.WaitGroup{} + + for _, o := range objs { + cmd, err = riak.NewStoreValueCommandBuilder(). + WithContent(o). + Build() + if err != nil { + util.ErrLog.Println(err) + continue + } + a := &riak.Async{ + Command: cmd, + Wait: wg, + } + if err := c.ExecuteAsync(a); err != nil { + util.ErrLog.Println(err) + } + } + + wg.Wait() +``` + +In our first object, we have stored the integer 1 with the lookup key +of `one`: + +```golang +{ + Bucket: bucket, + Key: "one", + ContentType: "application/octet-stream", + Value: val1buf, +} +``` + +For our second object, we stored a simple string value of `two` with a +matching key: + +```golang +{ + Bucket: bucket, + Key: "two", + ContentType: "text/plain", + Value: []byte(val2), +} +``` + +Finally, the third object we stored was a bit of JSON: + +```golang +{ + Bucket: bucket, + Key: "three", + ContentType: "application/json", + Value: val3json, +} +``` + +## Reading Objects + +Now that we have a few objects stored, let’s retrieve them and make sure +they contain the values we expect. + +Requesting the objects by key: + +```golang +var cmd riak.Command +wg := &sync.WaitGroup{} + +for _, o := range objs { + cmd, err = riak.NewStoreValueCommandBuilder(). + WithContent(o). + Build() + if err != nil { + util.ErrLog.Println(err) + continue + } + a := &riak.Async{ + Command: cmd, + Wait: wg, + } + if err := c.ExecuteAsync(a); err != nil { + util.ErrLog.Println(err) + } +} + +wg.Wait() + +util.Log.Println("Reading Objects From Riak...") + +d := make(chan riak.Command, len(objs)) + +for _, o := range objs { + cmd, err = riak.NewFetchValueCommandBuilder(). + WithBucket(bucket). + WithKey(o.Key). + Build() + if err != nil { + util.ErrLog.Println(err) + continue + } + a := &riak.Async{ + Command: cmd, + Wait: wg, + Done: d, + } + if err := c.ExecuteAsync(a); err != nil { + util.ErrLog.Println(err) + } +} + +wg.Wait() +close(d) +``` + +Converting to JSON to compare a string key to a symbol +key: + +```golang +for done := range d { + f := done.(*riak.FetchValueCommand) + /* un-comment to dump fetched object as JSON + if json, jerr := json.MarshalIndent(f.Response, "", " "); err != nil { + util.ErrLog.Println(jerr) + } else { + util.Log.Println("fetched value: ", string(json)) + } + */ + obj := f.Response.Values[0] + switch obj.Key { + case "one": + if actual, expected := binary.LittleEndian.Uint32(obj.Value), val1; actual != expected { + util.ErrLog.Printf("key: %s, actual %v, expected %v", obj.Key, actual, expected) + } + case "two": + if actual, expected := string(obj.Value), val2; actual != expected { + util.ErrLog.Printf("key: %s, actual %v, expected %v", obj.Key, actual, expected) + } + case "three": + obj3 = obj + val3.MyValue = 0 + if jerr := json.Unmarshal(obj.Value, &val3); jerr != nil { + util.ErrLog.Println(jerr) + } else { + if actual, expected := val3.MyValue, int(3); actual != expected { + util.ErrLog.Printf("key: %s, actual %v, expected %v", obj.Key, actual, expected) + } + } + default: + util.ErrLog.Printf("unrecognized key: %s", obj.Key) + } +} +``` + +## Updating Objects + +While some data may be static, other forms of data need to be +updated. + +Let’s update some values: + +```golang +util.Log.Println("Updating Object Three In Riak...") + +val3.MyValue = 42 +obj3.Value, err = json.Marshal(val3) +if err != nil { + util.ErrExit(err) +} + +cmd, err = riak.NewStoreValueCommandBuilder(). + WithContent(obj3). + WithReturnBody(true). + Build() +if err != nil { + util.ErrLog.Println(err) +} else { + if err := c.Execute(cmd); err != nil { + util.ErrLog.Println(err) + } +} + +svcmd := cmd.(*riak.StoreValueCommand) +svrsp := svcmd.Response +obj3 = svrsp.Values[0] +val3.MyValue = 0 +if jerr := json.Unmarshal(obj3.Value, &val3); jerr != nil { + util.ErrLog.Println(jerr) +} else { + if actual, expected := val3.MyValue, int(42); actual != expected { + util.ErrLog.Printf("key: %s, actual %v, expected %v", obj3.Key, actual, expected) + } +} +util.Log.Println("updated object key: ", obj3.Key) +util.Log.Println("updated object value: ", val3.MyValue) +``` + +## Deleting Objects + +As a last step, we’ll demonstrate how to delete data. You’ll see that +the delete message can be called against either the bucket or the +object. + +```golang +for _, o := range objs { + cmd, err = riak.NewDeleteValueCommandBuilder(). + WithBucket(o.Bucket). + WithKey(o.Key). + Build() + if err != nil { + util.ErrLog.Println(err) + continue + } + a := &riak.Async{ + Command: cmd, + Wait: wg, + } + if err := c.ExecuteAsync(a); err != nil { + util.ErrLog.Println(err) + } +} + +wg.Wait() +``` + +## Working With Complex Objects + +Since the world is a little more complicated than simple integers and +bits of strings, let’s see how we can work with more complex objects. + +For example, this `struct` that represents some information about +a book: + +```golang +type Book struct { + ISBN string + Title string + Author string + Body string + CopiesOwned uint16 +} + +book := &Book{ + ISBN: "1111979723", + Title: "Moby Dick", + Author: "Herman Melville", + Body: "Call me Ishmael. Some years ago...", + CopiesOwned: 3, +} +``` + +We now have some information about our Moby Dick collection +that we want to save. Storing this to Riak should look familiar by now: + +```golang +var jbook []byte +jbook, err = json.Marshal(book) +if err != nil { + util.ErrExit(err) +} + +bookObj := &riak.Object{ + Bucket: "books", + Key: book.ISBN, + ContentType: "application/json", + Value: jbook, +} + +cmd, err = riak.NewStoreValueCommandBuilder(). + WithContent(bookObj). + WithReturnBody(false). + Build() +if err != nil { + util.ErrLog.Println(err) +} else { + if err := c.Execute(cmd); err != nil { + util.ErrLog.Println(err) + } +} +``` + +If we fetch our book back and print the data: + +```golang +cmd, err = riak.NewFetchValueCommandBuilder(). + WithBucket("books"). + WithKey(book.ISBN). + Build() +if err != nil { + util.ErrExit(err) +} +if err := c.Execute(cmd); err != nil { + util.ErrLog.Println(err) +} + +fcmd := cmd.(*riak.FetchValueCommand) +bookObj = fcmd.Response.Values[0] +util.Log.Println(string(bookObj.Value)) +``` + +The result is: + +```json +{"isbn":"1111979723","title":"Moby Dick","author":"Herman Melville", +"body":"Call me Ishmael. Some years ago...","copies_owned":3} +``` + +Now, let’s delete the book: + +```golang +... +``` diff --git a/content/riak/kv/2.2.6/developing/getting-started/golang/object-modeling.md b/content/riak/kv/2.2.6/developing/getting-started/golang/object-modeling.md new file mode 100644 index 0000000000..35f1b5103f --- /dev/null +++ b/content/riak/kv/2.2.6/developing/getting-started/golang/object-modeling.md @@ -0,0 +1,548 @@ +--- +title_supertext: "Getting Started:" +title: "Object Modeling with Go" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Object Modeling" + identifier: "getting_started_go_object" + weight: 102 + parent: "getting_started_go" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/taste-of-riak/object-modeling-golang + - /riak-docs/riak/kv/2.2.6/dev/taste-of-riak/object-modeling-golang +--- + +{{% note title="Code Download" %}} +You can download the code for this chapter at +[Github](https://github.com/basho/taste-of-riak/tree/master/go/ch03/models). +{{% /note %}} + +To get started, let's create the models that we'll be using: + +```model.go +package models + +type Model interface { + GetId() string + SetId(id string) +} + +type modelImpl struct { + id string +} + +func (m *modelImpl) SetId(id string) { + m.id = id +} +``` + +Our user model: + +```user.go +package models + +type User struct { + modelImpl + UserName string + FullName string + Email string +} + +func NewUser(userName, fullName, email string) *User { + u := &User{ + UserName: userName, + FullName: fullName, + Email: email, + } + u.SetId(userName) + return u +} + +func (u *User) GetId() string { + return u.UserName +} +``` + +And our message model: + +```msg.go +package models + +import ( + "fmt" + "time" + + util "github.com/basho/taste-of-riak/go/util" +) + +type Msg struct { + modelImpl + Sender string + Recipient string + Text string + Created time.Time +} + +func NewMsg(sender, recipient, text string) *Msg { + m := &Msg{ + Sender: sender, + Recipient: recipient, + Text: text, + Created: time.Now(), + } + m.SetId(m.GetId()) + return m +} + +func (m *Msg) GetId() string { + return fmt.Sprintf("%s_%v", m.Sender, util.Iso8601(m.Created)) +} +``` + +Our timeline model: + +```timeline.go +package models + +type Timeline struct { + modelImpl + MsgKeys []string +} + +type TimelineType byte + +const ( + TimelineType_INBOX TimelineType = iota + TimelineType_SENT +) + +func NewTimeline(id string) *Timeline { + t := &Timeline{} + t.id = id + return t +} + +func (t *Timeline) AddMsg(msgKey string) { + t.MsgKeys = append(t.MsgKeys, msgKey) +} + +func (t *Timeline) GetId() string { + return t.id +} +```` + +We'll be using the bucket `Users` to store our data. We won't be [using bucket types]({{}}riak/kv/2.2.6/developing/usage/bucket-types) here, so we don't need to specify one. + +To use these records to store data, we will first have to create a user +record. Then, when a user creates a message, we will append that message +to one or more timelines. If it's a private message, we'll append it to +the Recipient's `Inbox` timeline and to the User's own `Sent` timeline. +If it's a group message, we'll append it to the Group's timeline, as +well as to the User's `Sent` timeline. + +#### Buckets and keys revisited + +Now that we've worked out how we will differentiate data in the system, +let's figure out our bucket and key names. + +The bucket names are straightforward. We can use `Users`, `Msgs`, and +`Timelines`. The key names, however, are a little trickier. In past +examples we've used sequential integers, but this presents a problem: we +would need a secondary service to hand out these IDs. This service could +easily be a future bottleneck in the system, so let's use a natural key. +Natural keys are a great fit for key/value systems because both humans +and computers can easily construct them when needed, and most of the +time they can be made unique enough for a KV store. + + +Bucket | Key Pattern | Example Key +:------|:------------|:----------- +`Users` | `` | `joeuser` +`Msgs` | `_` | `joeuser_2014-03-06T02:05:13.223556Z` +`Timelines` | `__` | `joeuser_Sent_2014-03-06Z`
`marketing_group_Inbox_2014-03-06Z` | + +For the `Users` bucket, we can be certain that we will want each +username to be unique, so let's use the `username` as the key. For the +`Msgs` bucket, let's use a combination of the username and the posting +datetime in an [ISO 8601 Long](http://en.wikipedia.org/wiki/ISO_8601) +format. This combination gives us the pattern `_`, +which produces keys like `joeuser_2014-03-05T23:20:28Z`. + +Now for `Timelines`, we need to differentiate between `Inbox` and `Sent` +timelines, so we can simply add that type into the key name. We will +also want to partition each collection object into some time period, +that way the object doesn't grow too large (see note below). + +For `Timelines`, let's use the pattern `__` for +users and `_Inbox_` for groups, which will look like +`joeuser_Sent_2014-03-06Z` or `marketing_group_Inbox_2014-03-05Z`, +respectively. + +{{% note title="Note" %}} +Riak performs best with objects under 1-2 MB. Objects larger than that can +hurt performance, especially if many siblings are being created. We will cover +siblings, sibling resolution, and sibling explosions in the next chapter. +{{% /note %}} + +#### Keeping our story straight with repositories + +Now that we've figured out our object model, let's write some modules to +act as repositories that will help us create and work with these records +in Riak: + +```repository.go +package repositories + +import ( + "encoding/json" + "errors" + + riak "github.com/basho/riak-go-client" + models "github.com/basho/taste-of-riak/go/ch03/models" +) + +var ErrUnexpectedSiblings = errors.New("Unexpected siblings in response!") + +type Repository interface { + Get(key string, notFoundOk bool) (models.Model, error) + Save(models.Model) (models.Model, error) + getBucketName() string + getModel() models.Model + getClient() *riak.Client +} + +type repositoryImpl struct { + client *riak.Client +} + +func (ri *repositoryImpl) getClient() *riak.Client { + return ri.client +} + +func get(r Repository, key string, notFoundOk bool) (models.Model, error) { + client := r.getClient() + bucket := r.getBucketName() + cmd, err := riak.NewFetchValueCommandBuilder(). + WithBucket(bucket). + WithKey(key). + WithNotFoundOk(notFoundOk). + Build() + if err != nil { + return nil, err + } + if err = client.Execute(cmd); err != nil { + return nil, err + } + + fcmd := cmd.(*riak.FetchValueCommand) + + if notFoundOk && len(fcmd.Response.Values) == 0 { + return nil, nil + } + + if len(fcmd.Response.Values) > 1 { + // Siblings present that need resolution + // Here we'll just return an unexpected error + return nil, ErrUnexpectedSiblings + } else { + return buildModel(r.getModel(), fcmd.Response.Values[0]) + } +} + +func save(r Repository, m models.Model) (models.Model, error) { + client := r.getClient() + bucket := r.getBucketName() + key := m.GetId() + + cmd, err := riak.NewFetchValueCommandBuilder(). + WithBucket(bucket). + WithKey(key). + WithNotFoundOk(true). + Build() + if err != nil { + return nil, err + } + if err = client.Execute(cmd); err != nil { + return nil, err + } + + modelJson, err := json.Marshal(m) + if err != nil { + return nil, err + } + + var objToInsertOrUpdate *riak.Object + fcmd := cmd.(*riak.FetchValueCommand) + if len(fcmd.Response.Values) > 1 { + // Siblings present that need resolution + // Here we'll just assume the first sibling is the "correct" one + // with which to update with the new Model data + // A conflict resolver can also be part of the options to fetchValue above + objToInsertOrUpdate = fcmd.Response.Values[0] + objToInsertOrUpdate.Value = modelJson + } else { + objToInsertOrUpdate = &riak.Object{ + Bucket: bucket, + Key: key, + ContentType: "application/json", + Charset: "utf8", + Value: modelJson, + } + } + + cmd, err = riak.NewStoreValueCommandBuilder(). + WithContent(objToInsertOrUpdate). + WithReturnBody(true). + Build() + if err != nil { + return nil, err + } + if err = client.Execute(cmd); err != nil { + return nil, err + } + + scmd := cmd.(*riak.StoreValueCommand) + if len(scmd.Response.Values) > 1 { + return nil, ErrUnexpectedSiblings + } + obj := scmd.Response.Values[0] + return buildModel(r.getModel(), obj) +} + +func buildModel(m models.Model, obj *riak.Object) (models.Model, error) { + err := json.Unmarshal(obj.Value, m) + m.SetId(obj.Key) + return m, err +} +``` + +
+ +```user-repository.go +package repositories + +import ( + riak "github.com/basho/riak-go-client" + models "github.com/basho/taste-of-riak/go/ch03/models" +) + +type UserRepository struct { + repositoryImpl +} + +func NewUserRepository(c *riak.Client) *UserRepository { + r := &UserRepository{} + r.client = c + return r +} + +func (u *UserRepository) Get(key string, notFoundOk bool) (models.Model, error) { + return get(u, key, notFoundOk) +} + +func (u *UserRepository) Save(m models.Model) (models.Model, error) { + return save(u, m) +} + +func (u *UserRepository) getBucketName() string { + return "Users" +} + +func (u *UserRepository) getModel() models.Model { + return &models.User{} +} +``` + +
+ +```msg-repository.go +package repositories + +import ( + riak "github.com/basho/riak-go-client" + models "github.com/basho/taste-of-riak/go/ch03/models" +) + +type MsgRepository struct { + repositoryImpl +} + +func NewMsgRepository(c *riak.Client) *MsgRepository { + m := &MsgRepository{} + m.client = c + return m +} + +func (m *MsgRepository) Get(key string, notFoundOk bool) (models.Model, error) { + return get(m, key, notFoundOk) +} + +func (m *MsgRepository) Save(model models.Model) (models.Model, error) { + return save(m, model) +} + +func (m *MsgRepository) getBucketName() string { + return "Msgs" +} + +func (m *MsgRepository) getModel() models.Model { + return &models.Msg{} +} +``` + +
+ +```timeline-repository.go +package repositories + +import ( + riak "github.com/basho/riak-go-client" + models "github.com/basho/taste-of-riak/go/ch03/models" +) + +type TimelineRepository struct { + repositoryImpl +} + +func NewTimelineRepository(c *riak.Client) *TimelineRepository { + t := &TimelineRepository{} + t.client = c + return t +} + +func (t *TimelineRepository) Get(key string, notFoundOk bool) (models.Model, error) { + return get(t, key, notFoundOk) +} + +func (t *TimelineRepository) Save(m models.Model) (models.Model, error) { + return save(t, m) +} + +func (t *TimelineRepository) getBucketName() string { + return "Timelines" +} + +func (t *TimelineRepository) getModel() models.Model { + return &models.Timeline{} +} +``` + +Finally, let's test them: + +```golang +package main + +import ( + "time" + + mgrs "github.com/basho/taste-of-riak/go/ch03/managers" + models "github.com/basho/taste-of-riak/go/ch03/models" + repos "github.com/basho/taste-of-riak/go/ch03/repositories" + + riak "github.com/basho/riak-go-client" + util "github.com/basho/taste-of-riak/go/util" +) + +func main() { + var err error + + // un-comment-out to enable debug logging + // riak.EnableDebugLogging = true + + util.Log.Println("Starting Client") + + o := &riak.NewClientOptions{ + RemoteAddresses: util.GetRiakAddresses(), + } + + var client *riak.Client + client, err = riak.NewClient(o) + if err != nil { + util.ErrExit(err) + } + + defer func() { + if err := client.Stop(); err != nil { + util.ErrExit(err) + } + }() + + userRepo := repos.NewUserRepository(client) + msgRepo := repos.NewMsgRepository(client) + timelineRepo := repos.NewTimelineRepository(client) + timelineMgr := mgrs.NewTimelineManager(timelineRepo, msgRepo) + + util.Log.Println("Creating and saving users") + + marleen := models.NewUser("marleenmgr", "Marleen Manager", "marleen.manager@basho.com") + joe := models.NewUser("joeuser", "Joe User", "joe.user@basho.com") + + var m models.Model + m, err = userRepo.Save(marleen) + if err != nil { + util.ErrExit(err) + } + marleen = m.(*models.User) + + m, err = userRepo.Save(joe) + if err != nil { + util.ErrExit(err) + } + joe = m.(*models.User) + + util.Log.Println("Posting message") + + msg := models.NewMsg(marleen.UserName, joe.UserName, "Welcome to the company!") + if terr := timelineMgr.PostMsg(msg); terr != nil { + util.ErrExit(terr) + } + + util.Log.Println("Getting Joe's inbox for today") + + // Get Joe's inbox for today, get first message + now := time.Now() + joe_tl, terr := timelineMgr.GetTimeline(joe.UserName, models.TimelineType_INBOX, now) + if terr != nil { + util.ErrExit(terr) + } + + for _, msgKey := range joe_tl.MsgKeys { + m, merr := msgRepo.Get(msgKey, false) + if merr != nil { + util.ErrExit(merr) + } + tl_msg := m.(*models.Msg) + util.Log.Println("From: ", tl_msg.Sender) + util.Log.Println("Msg: ", tl_msg.Text) + } +} +``` + +As you can see, the repository pattern helps us with a few things: + +* It helps us to see if an object exists before creating a new one. +* It keeps our buckets and key names consistent. +* It provides us with a consistent interface to work with. + +While this set of repositories solves many of our problems, it is very +minimal and doesn't cover all the edge cases. For instance, what happens +if two different people try to create a user with the same username? + +Also, we can easily compute key names now, but how do we quickly look +up the last 10 messages a user sent? Many of these answers will be +application-dependent. If your application shows the last 10 messages in +reverse order, for example, you may want to store that set of data in +another collection object to make lookup faster. There are drawbacks to +every solution, but we recommend seeking out the key/value-based +solution first, as it will likely be the quickest. + +So to recap, in this chapter we learned: + +* How to choose bucket names. +* How to choose natural keys based on how we want to partition our data. + + diff --git a/content/riak/kv/2.2.6/developing/getting-started/golang/querying.md b/content/riak/kv/2.2.6/developing/getting-started/golang/querying.md new file mode 100644 index 0000000000..1a4a4f92e2 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/getting-started/golang/querying.md @@ -0,0 +1,576 @@ +--- +title_supertext: "Getting Started:" +title: "Querying with Go" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Querying" + identifier: "getting_started_go_query" + weight: 101 + parent: "getting_started_go" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/taste-of-riak/querying-golang + - /riak-docs/riak/kv/2.2.6/dev/taste-of-riak/querying-golang +--- + +## Go Version Setup + +For the Go version, please download the source from GitHub by either [cloning](https://github.com/basho/taste-of-riak) the source code repository or downloading the [current zip of the master branch](https://github.com/basho/taste-of-riak/archive/master.zip). Ensure that the source is located in your `GOPATH`. The code for this chapter is in `go/ch02/ch02.go`. You may import this code into your favorite editor, or just run it from the command line using the `Makefile` if you are running on a *nix* OS. + +>A Quick Note on Querying and Schemas: +> +>Even with a key/value store, you will still have a logical database schema of how all the data relates to one another. This can be as simple as using the same key across multiple buckets for different types of data, to having fields in your data that are related by name. These querying methods will introduce you to some ways of laying out your data in Riak, along with how to query it back. + +### Denormalization + +If you're coming from a relational database, the easiest way to get your application started with NoSQL is to denormalize your data into related chunks. For example with a customer database, you might have separate tables for Customers, Addresses, Preferences, etc. In Riak KV, you can denormalize all that associated data into a single object and store it into a `Customer` bucket. You can keep pulling in associated data until you hit one of the big denormalization walls: + +* Size Limits (objects greater than 1MB) +* Shared/Referential Data (data that the object doesn't "own") +* Differences in Access Patterns (objects that get read/written once vs. often) + +At one of these points we will have to split the model. + +### Same Keys - Different Buckets + +The simplest way to split up data would be to use the same identity key across different buckets. A good example of this would be a `Customer` object, an `Order` object, and an `OrderSummaries` object that keeps rolled up info about orders such as Total, etc. Let's put some data into Riak KV so we can play with it. + +```golang +package main + +import ( + "encoding/json" + "errors" + "fmt" + "reflect" + "sync" + "time" + + riak "github.com/basho/riak-go-client" + util "github.com/basho/taste-of-riak/go/util" +) + +const ( + timeFmt = "2006-01-02 15:04:05" + customersBucket = "Customers" + ordersBucket = "Orders" + orderSummariesBucket = "OrderSummaries" +) + +type Customer struct { + Name string + Address string + City string + State string + Zip string + Phone string + CreatedDate time.Time +} + +type Order struct { + Id string + CustomerId string + SalespersonId string + Items []*OrderItem + Total float32 + Date time.Time +} + +type OrderItem struct { + Id string + Title string + Price float32 +} + +type OrderSummary struct { + CustomerId string + Summaries []*OrderSummaryItem +} + +type OrderSummaryItem struct { + Id string + Total float32 + Date time.Time +} + +func main() { + var err error + var customerId string + + util.Log.Println("Creating Data") + + var cd time.Time + cd, err = time.Parse(timeFmt, "2013-10-01 14:30:26") + if err != nil { + util.ErrExit(err) + } + + customer := &Customer{ + Name: "John Smith", + Address: "123 Main Street", + City: "Columbus", + State: "Ohio", + Zip: "43210", + Phone: "+1-614-555-5555", + CreatedDate: cd, + } + + util.Log.Printf("customer: %v", customer) + + util.Log.Println("Starting Client") + + // un-comment-out to enable debug logging + // riak.EnableDebugLogging = true + + o := &riak.NewClientOptions{ + RemoteAddresses: []string{util.GetRiakAddress()}, + } + + var c *riak.Client + c, err = riak.NewClient(o) + if err != nil { + util.ErrExit(err) + } + + defer func() { + if err := c.Stop(); err != nil { + util.ErrExit(err) + } + }() + + util.Log.Println("Storing Customer") + + var cmd riak.Command + var customerJson []byte + + customerJson, err = json.Marshal(customer) + if err != nil { + util.ErrExit(err) + } + + obj := &riak.Object{ + Bucket: customersBucket, + ContentType: "application/json", + Value: customerJson, + } + + cmd, err = riak.NewStoreValueCommandBuilder(). + WithContent(obj). + WithReturnBody(true). + Build() + if err != nil { + util.ErrExit(err) + } + if eerr := c.Execute(cmd); eerr != nil { + util.ErrExit(eerr) + } + + svc := cmd.(*riak.StoreValueCommand) + customerId = svc.Response.GeneratedKey + if customerId == "" { + util.ErrExit(errors.New("expected generated customer Id")) + } else { + util.Log.Println("Customer ID:", customerId) + } + + util.Log.Println("Storing Data") + + var orders []*Order + orders, err = createOrders(customerId) + if err != nil { + util.ErrExit(err) + } + + var orderSummary *OrderSummary + var orderSummaryJson []byte + orderSummary = createOrderSummary(customerId, orders) + + ccmds := 1 + len(orders) + cmds := make([]riak.Command, ccmds) + + // command to store OrderSummary + orderSummaryJson, err = json.Marshal(orderSummary) + if err != nil { + util.ErrExit(err) + } + obj = &riak.Object{ + Bucket: orderSummariesBucket, + Key: customerId, + ContentType: "application/json", + Value: orderSummaryJson, + } + cmds[0], err = riak.NewStoreValueCommandBuilder(). + WithContent(obj). + Build() + if err != nil { + util.ErrExit(err) + } + + for i, order := range orders { + // command to store Order + var orderJson []byte + orderJson, err = json.Marshal(order) + if err != nil { + util.ErrExit(err) + } + obj = &riak.Object{ + Bucket: ordersBucket, + Key: order.Id, + ContentType: "application/json", + Value: orderJson, + } + cmds[i+1], err = riak.NewStoreValueCommandBuilder(). + WithContent(obj). + Build() + if err != nil { + util.ErrExit(err) + } + } + + errored := false + wg := &sync.WaitGroup{} + for _, cmd := range cmds { + a := &riak.Async{ + Command: cmd, + Wait: wg, + } + if eerr := c.ExecuteAsync(a); eerr != nil { + errored = true + util.ErrLog.Println(eerr) + } + } + wg.Wait() + if errored { + util.ErrExit(errors.New("error, exiting!")) + } +} + +func createOrders(customerId string) ([]*Order, error) { + o := make([]*Order, 3) + + d, err := time.Parse(timeFmt, "2013-10-01 14:42:26") + if err != nil { + return nil, err + } + o[0] = &Order{ + Id: "1", + CustomerId: customerId, + SalespersonId: "9000", + Items: []*OrderItem{ + { + Id: "TCV37GIT4NJ", + Title: "USB 3.0 Coffee Warmer", + Price: 15.99, + }, + { + Id: "PEG10BBF2PP", + Title: "eTablet Pro, 24GB; Grey", + Price: 399.99, + }, + }, + Total: 415.98, + Date: d, + } + + d, err = time.Parse(timeFmt, "2013-10-15 16:43:16") + if err != nil { + return nil, err + } + o[1] = &Order{ + Id: "2", + CustomerId: customerId, + SalespersonId: "9001", + Items: []*OrderItem{ + { + Id: "OAX19XWN0QP", + Title: "GoSlo Digital Camera", + Price: 359.99, + }, + }, + Total: 359.99, + Date: d, + } + + d, err = time.Parse(timeFmt, "2013-11-03 17:45:28") + if err != nil { + return nil, err + } + o[2] = &Order{ + Id: "3", + CustomerId: customerId, + SalespersonId: "9000", + Items: []*OrderItem{ + { + Id: "WYK12EPU5EZ", + Title: "Call of Battle : Goats - Gamesphere 4", + Price: 69.99, + }, + { + Id: "TJB84HAA8OA", + Title: "Bricko Building Blocks", + Price: 4.99, + }, + }, + Total: 74.98, + Date: d, + } + + return o, nil +} + +func createOrderSummary(customerId string, orders []*Order) *OrderSummary { + + s := &OrderSummary{ + CustomerId: customerId, + Summaries: make([]*OrderSummaryItem, len(orders)), + } + + for i, o := range orders { + s.Summaries[i] = &OrderSummaryItem{ + Id: o.Id, + Total: o.Total, + Date: o.Date, + } + } + + return s +} +``` + +While individual `Customer` and `Order` objects don't change much (or shouldn't change), the `Order Summaries` object will likely change often. It will do double duty by acting as an index for all a customer's orders and also holding some relevant data, such as the order total, etc. If we showed this information in our application often, it's only one extra request to get all the info. + +```golang +util.Log.Println("Fetching related data by shared key") + +cmds = cmds[:0] + +// fetch customer +cmd, err = riak.NewFetchValueCommandBuilder(). + WithBucket(customersBucket). + WithKey(customerId). + Build() +if err != nil { + util.ErrExit(err) +} +cmds = append(cmds, cmd) + +// fetch OrderSummary +cmd, err = riak.NewFetchValueCommandBuilder(). + WithBucket(orderSummariesBucket). + WithKey(customerId). + Build() +if err != nil { + util.ErrExit(err) +} +cmds = append(cmds, cmd) + +doneChan := make(chan riak.Command) +errored = false +for _, cmd := range cmds { + a := &riak.Async{ + Command: cmd, + Done: doneChan, + } + if eerr := c.ExecuteAsync(a); eerr != nil { + errored = true + util.ErrLog.Println(eerr) + } +} +if errored { + util.ErrExit(errors.New("error, exiting!")) +} + +for i := 0; i < len(cmds); i++ { + select { + case d := <-doneChan: + if fv, ok := d.(*riak.FetchValueCommand); ok { + obj := fv.Response.Values[0] + switch obj.Bucket { + case customersBucket: + util.Log.Printf("Customer 1: %v", string(obj.Value)) + case orderSummariesBucket: + util.Log.Printf("OrderSummary 1: %v", string(obj.Value)) + } + } else { + util.ErrExit(fmt.Errorf("unknown response command type: %v", reflect.TypeOf(d))) + } + case <-time.After(5 * time.Second): + util.ErrExit(errors.New("fetch operations took too long")) + } +} +``` + +Which returns our amalgamated objects: + +```sh +2015/12/29 09:44:10 OrderSummary 1: {"CustomerId":"I4R9AdTpJ7RL13qj14ED9Qjzbyy","Summaries":[{"Id":"1","Total":415.98,"Date":"2013-10-01T14:42:26Z"},{"Id":"2","Total":359.99,"Date":"2013-10-15T16:43:16Z"},{"Id":"3","Total":74.98,"Date":"2013-11-03T17:45:28Z"}]} +2015/12/29 09:44:10 Customer 1: {"Name":"John Smith","Address":"123 Main Street","City":"Columbus","State":"Ohio","Zip":"43210","Phone":"+1-614-555-5555","CreatedDate":"2013-10-01T14:30:26Z" +``` + +While this pattern is very easy and extremely fast with respect to queries and complexity, it's up to the application to know about these intrinsic relationships. + + +### Secondary Indexes + +{{% note %}} +Secondary indexes in Riak KV require a sorted backend: [Memory]({{}}riak/kv/2.2.6/setup/planning/backend/memory) or [LevelDB]({{}}riak/kv/2.2.6/setup/planning/backend/leveldb). [Bitcask]({{}}riak/kv/2.2.6/setup/planning/backend/bitcask) does not support secondary indexes. + +See [Using Secondary Indexes (2i)]({{}}riak/kv/2.2.6/developing/usage/secondary-indexes) for more information on developing with secondary indexes. +{{% /note %}} + +If you're coming from a SQL world, Secondary Indexes (2i) are a lot like SQL indexes. They are a way to quickly look up objects based on a secondary key, without scanning through the whole dataset. This makes it very easy to find groups of related data by values or ranges of values. To properly show this off, we will add some more data to our application, and add some secondary index entries at the same time: + +```golang +util.Log.Println("Adding Index Data") + +// fetch orders to add index data +cmds = cmds[:0] + +for _, order := range orders { + cmd, err = riak.NewFetchValueCommandBuilder(). + WithBucket(ordersBucket). + WithKey(order.Id). + Build() + if err != nil { + util.ErrExit(err) + } + cmds = append(cmds, cmd) +} + +errored = false +for _, cmd := range cmds { + a := &riak.Async{ + Command: cmd, + Done: doneChan, + } + if eerr := c.ExecuteAsync(a); eerr != nil { + errored = true + util.ErrLog.Println(eerr) + } +} +if errored { + util.ErrExit(errors.New("error, exiting!")) +} + +errored = false +for i := 0; i < len(cmds); i++ { + select { + case d := <-doneChan: + if fv, ok := d.(*riak.FetchValueCommand); ok { + obj := fv.Response.Values[0] + switch obj.Key { + case "1": + obj.AddToIntIndex("SalespersonId_int", 9000) + obj.AddToIndex("OrderDate_bin", "2013-10-01") + case "2": + obj.AddToIntIndex("SalespersonId_int", 9001) + obj.AddToIndex("OrderDate_bin", "2013-10-15") + case "3": + obj.AddToIntIndex("SalespersonId_int", 9000) + obj.AddToIndex("OrderDate_bin", "2013-11-03") + } + scmd, serr := riak.NewStoreValueCommandBuilder(). + WithContent(obj). + Build() + if serr != nil { + util.ErrExit(serr) + } + a := &riak.Async{ + Command: scmd, + Wait: wg, + } + if eerr := c.ExecuteAsync(a); eerr != nil { + errored = true + util.ErrLog.Println(eerr) + } + } else { + util.ErrExit(fmt.Errorf("unknown response command type: %v", reflect.TypeOf(d))) + } + case <-time.After(5 * time.Second): + util.ErrExit(errors.New("fetch operations took too long")) + } +} + +if errored { + util.ErrExit(errors.New("error, exiting!")) +} + +wg.Wait() +close(doneChan) +``` + +As you may have noticed, ordinary key/value data is opaque to 2i, so we have to add entries to the indexes at the application level. + +Now let's find all of Jane Appleseed's processed orders. We'll lookup the orders by searching the `saleperson_id_int` index for Jane's id of `9000`: + +```golang +util.Log.Println("Index Queries") + +cmd, err = riak.NewSecondaryIndexQueryCommandBuilder(). + WithBucket(ordersBucket). + WithIndexName("SalespersonId_int"). + WithIndexKey("9000"). + Build() +if err != nil { + util.ErrExit(err) +} + +if eerr := c.Execute(cmd); eerr != nil { + util.ErrExit(eerr) +} + +qcmd := cmd.(*riak.SecondaryIndexQueryCommand) +for _, rslt := range qcmd.Response.Results { + util.Log.Println("Jane's Orders, key: ", string(rslt.ObjectKey)) +} +``` + +Which returns: + +```sh +2015/12/29 09:44:10 Jane's Orders, key: 3 +2015/12/29 09:44:10 Jane's Orders, key: 1 +``` + +Jane processed orders 1 and 3. We used an *integer* index to reference Jane's id, next let's use a *binary* index. + +Let's say that the VP of Sales wants to know how many orders came in during October 2013. In this case, we can exploit 2i's range queries. Let's search the `order_date_bin` index for entries between `20131001` and `20131031`: + +```golang +cmd, err = riak.NewSecondaryIndexQueryCommandBuilder(). + WithBucket(ordersBucket). + WithIndexName("OrderDate_bin"). + WithRange("2013-10-01", "2013-10-31"). + Build() +if err != nil { + util.ErrExit(err) +} + +if eerr := c.Execute(cmd); eerr != nil { + util.ErrExit(eerr) +} + +qcmd = cmd.(*riak.SecondaryIndexQueryCommand) +for _, rslt := range qcmd.Response.Results { + util.Log.Println("October's Orders, key: ", string(rslt.ObjectKey)) +} +``` + +Which returns: + +```sh +2015/12/29 09:44:10 October's Orders, key: 1 +2015/12/29 09:44:10 October's Orders, key: 2 +``` + +Easy! We used 2i's range feature to search for a range of values, and demonstrated binary indexes. + +So to recap: + +* You can use Secondary Indexes to quickly lookup an object based on a secondary id other than the object's key. +* Indexes can have either Integer or Binary(String) keys. +* You can search for specific values or a range of values. +* Riak will return a list of keys that match the index query. diff --git a/content/riak/kv/2.2.6/developing/getting-started/java.md b/content/riak/kv/2.2.6/developing/getting-started/java.md new file mode 100644 index 0000000000..b9d59523df --- /dev/null +++ b/content/riak/kv/2.2.6/developing/getting-started/java.md @@ -0,0 +1,89 @@ +--- +title: "Getting Started with Java" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Java" + identifier: "getting_started_java" + weight: 100 + parent: "developing_getting_started" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/taste-of-riak/java + - /riak-docs/riak/kv/2.2.6/dev/taste-of-riak/java +--- + + + +If you haven't set up a Riak Node and started it, please visit [Running A Cluster]({{}}riak/kv/2.2.6/using/running-a-cluster) first. + +To try this flavor of Riak, a working installation of Java is required. + +## Client Setup + +To include the Riak Java client in your project, add it to your +project's dependencies. Here is a Maven example: + +```xml + + + com.basho.riak + riak-client + 2.1.1 + +``` + +Next, download +[`TasteOfRiak.java`](https://github.com/basho/basho_docs/raw/master/extras/code-examples/TasteOfRiak.java) +source code for this tutorial, and save it to your working directory. + +{{% note title="Configuring for a local cluster" %}} +The `TasteOfRiak.java` file that you downloaded is set up to communicate with +a 1-node Riak cluster listening on `localhost` port 10017. We recommend +modifying the connection info directly within the `setUpCluster()` method. +{{% /note %}} + +If you execute the `TasteOfRiak.java` file within your IDE, you should +see the following: + +``` +Basic object created +Location object created for quote object +StoreValue operation created +Client object successfully created +Object storage operation successfully completed +Success! The object we created and the object we fetched have the same value +Quote object successfully deleted +Book object created +Moby Dick information now stored in Riak +Book object successfully fetched +Success! All of our tests check out +``` + +Since Java doesn’t have a REPL environment, let's walk through the code +to see what it actually did at each step. + +## Setting Up the Cluster + +The first step in using the Riak Java client is to create a cluster +object to facilitate all interactions with Riak. You'll see this on line +72: + +```java +RiakCluster cluster = setUpCluster(); +``` + +This calls the private `setUpCluster` method which begins on line 25. +Using that `cluster` object, we can instantiate a client object which +will execute all Riak interactions: + +```java +RiakClient client = new RiakClient(cluster); +``` + +## Next Steps + +[CRUD Operations]({{}}riak/kv/2.2.6/developing/getting-started/java/crud-operations) diff --git a/content/riak/kv/2.2.6/developing/getting-started/java/crud-operations.md b/content/riak/kv/2.2.6/developing/getting-started/java/crud-operations.md new file mode 100644 index 0000000000..1a9b00b6f8 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/getting-started/java/crud-operations.md @@ -0,0 +1,201 @@ +--- +title_supertext: "Getting Started:" +title: "CRUD Operations with Java" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "CRUD Operations" + identifier: "getting_started_java_crud" + weight: 100 + parent: "getting_started_java" +toc: true +--- + +## Creating Objects in Riak + +The first object that we create is a very basic object with a content +type of `text/plain`. Once that object is created, we create a +`StoreValue` operation that will store the object later on down the line + +```java +RiakObject quoteObject = new RiakObject() + .setContentType("text/plain") + .setValue(BinaryValue.create("You're dangerous, Maverick")); +Namespace quotesBucket = new Namespace("quotes"); +Location quoteObjectLocation = new Location(quotesBucket, "Icemand"); +StoreValue storeOp = new StoreValue.Builder(quoteObject) + .withLocation(quoteObjectLocation) + .build(); +``` + +We then use our `client` object to execute the storage operation: + +```java +StoreValue.Response response = client.execute(storeOp); +``` + +## Reading Objects from Riak + +After that, we check to make sure that the stored object has the same +value as the object that we created. This requires us to fetch the +object by way of a `FetchValue` operation: + +```java +FetchValue fetchOp = new FetchValue.Builder(quoteObjectLocation) + .build(); +RiakObject fetchedObject = client.execute(fetchOp).getValue(RiakObject.class); +assert(fetchedObject.getValue.equals(quoteObject.getValue())); +``` + +If the values are equal, as they should be, the Java client will say +`Success! The object we created and the object we fetched have the same +value`. If not, then the client will throw an exception. + +## Updating Objects + +Once we've read the object back in from Riak, we can update the object +and store it back as we did before with the `StoreValue` object: + +```java +fetchedObject.setValue(BinaryValue.create("You can be my wingman any time.")); +StoreValue updateOp = new StoreValue.Builder(fetchedObject) + .withLocation(quoteObjectLocation) + .build(); +StoreValue.Response updateOpResp = client.execute(updateOp); +``` + +For more in depth information on updating objects and sibling resolution in +Riak, see [Updating Objects]({{}}riak/kv/2.2.6/developing/usage/updating-objects/) +and [Conflict Resolution]({{}}riak/kv/2.2.6/developing/usage/conflict-resolution/) +documentation. + +## Updating Objects + +Once we've read the object back in from Riak, we can update the object +and store it back as we did before with the `StoreValue` object: + +```java +fetchedObject.setValue(BinaryValue.create("You can be my wingman any time.")); +StoreValue updateOp = new StoreValue.Builder(fetchedObject) + .withLocation(quoteObjectLocation) + .build(); +StoreValue.Response updateOpResp = client.execute(updateOp); +``` + +For more in depth information on updating objects and sibling resolution in +Riak, see [Updating Objects]({{}}riak/kv/2.2.6/developing/usage/updating-objects/) +and [Conflict Resolution]({{}}riak/kv/2.2.6/developing/usage/conflict-resolution/) +documentation. + +## Deleting Objects + +Now that we've stored and then fetched the object, we can delete it by +creating and executing a `DeleteValue` operation: + +```java +DeleteValue deleteOp = new DeleteValue.Builder(quoteObjectLocation) + .build(); +client.execute(deleteOp); +``` + +## Working With Complex Objects + +Since the world is a little more complicated than simple integers and +bits of strings, let’s see how we can work with more complex objects. +Take for example, this plain old Java object (POJO) that encapsulates +some knowledge about a book. + +```java +public class Book { + public String title; + public String author; + public String body; + public String isbn; + publict Integer copiesOwned; +} +``` + +By default, the Java Riak client serializes POJOs as JSON. Let's create +a new `Book` object to store: + +```java +Book mobyDick = new Book(); +modyDick.title = "Moby Dick"; +mobyDick.author = "Herman Melville"; +mobyDick.body = "Call me Ishmael. Some years ago..."; +mobyDick.isbn = "11119799723"; +mobyDick.copiesOwned = 3; +``` + +Now we can store that POJO object just like we stored the more simple +object earlier: + +```java +Namespace booksBucket = new Namespace("books"); +Location mobyDickLocation = new Location(booksBucket, "moby_dick"); +StoreValue storeBookOp = new StoreValue.Builder(mobyDick) + .withLocation(mobyDickLocation) + .build(); +client.execute(storeBookOp); +``` + +If we fetch the object (using the same method we showed up above and in +`TasteOfRiak.java`), we should get the following: + +```json +{ + "title": "Moby Dick", + "author": "Herman Melville", + "body": "Call me Ishmael. Some years ago...", + "isbn": "1111979723", + "copiesOwned": 3 +} +``` + +Since we really like Moby Dick, let's buy a couple more copies +and update the POJO. + +To update the POJO, we would use `UpdateValue` by +extending a new `BookUpdate` class as follows: + +```java +public static class BookUpdate extends UpdateValue.Update { + private final Book update; + public BookUpdate(Book update){ + this.update = update; + } + + @Override + public Book apply(Book t) { + if(t == null) { + t = new Book(); + } + + t.author = update.author; + t.body = update.body; + t.copiesOwned = update.copiesOwned; + t.isbn = update.isbn; + t.title = update.title; + + return t; + } +} +``` + +Then using the `BookUpdate` class with our `mobyDick` object: + +```java +mobyDick.copiesOwned = 5; +BookUpdate updatedBook = new BookUpdate(mobyDick); + +UpdateValue updateValue = new UpdateValue.Builder(mobyDickLocation) + .withUpdate(updatedBook).build(); +UpdateValue.Response response = client.execute(updateValue); +``` + +For more in depth information on updating objects and sibling resolution in +Riak, see [Updating Objects]({{}}riak/kv/2.2.6/developing/usage/updating-objects/) +and [Conflict Resolution]({{}}riak/kv/2.2.6/developing/usage/conflict-resolution/) +documention. diff --git a/content/riak/kv/2.2.6/developing/getting-started/java/object-modeling.md b/content/riak/kv/2.2.6/developing/getting-started/java/object-modeling.md new file mode 100644 index 0000000000..5fbe93b8ed --- /dev/null +++ b/content/riak/kv/2.2.6/developing/getting-started/java/object-modeling.md @@ -0,0 +1,428 @@ +--- +title_supertext: "Getting Started:" +title: "Object Modeling with Java" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Object Modeling" + identifier: "getting_started_java_object" + weight: 102 + parent: "getting_started_java" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/taste-of-riak/object-modeling-java + - /riak-docs/riak/kv/2.2.6/dev/taste-of-riak/object-modeling-java +--- + +To get started, let's create the models that we'll be using. + +```java +package com.basho.msgy.Models; + +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.TimeZone; + +public class Msg { + public String Sender; + public String Recipient; + public String Created; + public String Text; + + public static Msg createNew(String sender, String recipient, String text) { + Msg msg = new Msg(); + msg.Sender = sender; + msg.Recipient = recipient; + msg.Text = text; + msg.Created = GetCurrentISO8601Timestamp(); + return msg; + } + + private static String GetCurrentISO8601Timestamp() { + TimeZone tz = TimeZone.getTimeZone("UTC"); + // Java Dates don't have microsecond resolution :( + // Pad out to microseconds to match other examples. + DateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'000'"); + df.setTimeZone(tz); + return df.format(new Date()); + } +} + +// ---------------------------------------------------------------------------- + +import java.util.ArrayList; + +public class Timeline { + + public enum TimelineType + { + Inbox, + Sent; + + @Override + public String toString() { + if(this == Inbox) + return "Inbox"; + else + return "Sent"; + } + } + + public Timeline() { + Msgs = new ArrayList(); + } + + public String Owner; + public String Type; + public ArrayList Msgs; +} + +// ---------------------------------------------------------------------------- + +package com.basho.msgy.Models; + +import com.basho.riak.client.convert.RiakKey; + +public class User { + @RiakKey + public String UserName; + + @RiakBucketName + final String bucketName = "msgs"; + + public String FullName; + public String Email; + + public User() {} + + public User(String userName, String fullName, String email) { + this.UserName = userName; + this.FullName = fullName; + this.Email = email; + } +} +``` + +To use these classes to store data, we will first have to create a user. +Then, when a user creates a message, we will append that message to one +or more timelines. If it's a private message, we'll append it to the +Recipient's `Inbox` timeline and the User's own `Sent` timeline. If it's +a group message, we'll append it to the Group's timeline, as well as to +the User's `Sent` timeline. + +#### Buckets and Keys Revisited + +Now that we've worked out how we will differentiate data in the system, +let's figure out our bucket and key names. + +The bucket names are straightforward. We can use `Users`, `Msgs`, and +`Timelines`. The key names, however, are a little more tricky. In past +examples we've used sequential integers, but this presents a problem: we +would need a secondary service to hand out these IDs. This service could +easily be a future bottleneck in the system, so let's use a natural key. +Natural keys are a great fit for key/value systems because both humans +and computers can easily construct them when needed, and most of the +time they can be made unique enough for a KV store. + + +| Bucket | Key Pattern | Example Key +|:-------|:------------|:----------- +| `Users` | `` | `joeuser` +| `Msgs` | `_` | `joeuser_2014-03-06T02:05:13.223556Z` +| `Timelines` | `__` | `joeuser_Sent_2014-03-06Z`
`marketing_group_Inbox_2014-03-06Z` | + +For the `Users` bucket, we can be certain that we will want each +username to be unique, so let's use the `username` as the key. With the +Java client, we can use the `@RiakKey` annotation to tell the client +that we want to use the `UserName` member as the key. It will +automatically use that value in the future, instead of having to pass the +key in as another parameter when storing a value. + +For the `Msgs` bucket, let's use a combination of the username and the +posting datetime in an [ISO 8601 +Long](http://en.wikipedia.org/wiki/ISO_8601) format. This combination +gives us the pattern `_`, which produces keys like +`joeuser_2014-03-05T23:20:28Z`. + +Now for `Timelines`, we need to differentiate between `Inbox` and `Sent` +timelines, so we can simply add that type into the key name. We will +also want to partition each collection object into some time period, +that way the object doesn't grow too large (see note below). + +For `Timelines`, let's use the pattern `__` for +users, and `_Inbox_` for groups, which will look like +`joeuser_Sent_2014-03-06Z` or `marketing_group_Inbox_2014-03-05Z`, +respectively. + +{{% note title="Note" %}} +Riak performs best with objects under 1-2MB. Objects larger than that can hurt +performance, especially many siblings are being created. We will cover +siblings, sibling resolution, and sibling explosions in the next chapter. +{{% /note %}} + +#### Keeping our story straight with repositories + +Now that we've figured out our object model, let's write some +repositories to help create and work with these objects in Riak: + +```java +package com.basho.msgy.Repositories; + +import com.basho.msgy.Models.Msg; +import com.basho.riak.client.IRiakClient; +import com.basho.riak.client.RiakRetryFailedException; +import com.basho.riak.client.bucket.Bucket; + +public class MsgRepository { + + static final String BUCKET_NAME = "Msgs"; + protected RiakClient client; + + public MsgRepository(RiakClient client) { + this.client = client; + } + + public Msg get(String msgKey) throws Exception { + Location key = new Location(new Namespace(BUCKET_NAME), msgKey); + FetchValue fetch = new FetchValue.Builder(key).build(); + FetchValue.Response response = client.execute(fetch); + return response.getValue(Msg.class); + } + + public String save(Msg msg) throws Exception { + StoreValue store = new StoreValue.Builder(msg).build(); + client.execute(store); + return generateKey(msg); + } + + private String generateKey(Msg msg) { + return msg.Sender + "_" + msg.Created; + } +} + +// ---------------------------------------------------------------------------- + +package com.basho.msgy.Repositories; + +import com.basho.msgy.Models.Msg; +import com.basho.msgy.Models.Timeline; +import com.basho.riak.client.IRiakClient; +import com.basho.riak.client.RiakRetryFailedException; +import com.basho.riak.client.bucket.Bucket; + +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.TimeZone; + +public class TimelineRepository { + + static final String BUCKET_NAME = "Timelines"; + protected RiakClient client; + protected MsgRepository msgRepo; + + public TimelineRepository(RiakClient client) { + this.client = client; + this.msgRepo = new MsgRepository(this.client); + } + + public void postMsg(Msg msg) throws Exception { + String msgKey = msgRepo.save(msg); + + // Post to recipient's Inbox timeline + addToTimeline(msg, Timeline.TimelineType.Inbox, msgKey); + + // Post to sender's Sent timeline + addToTimeline(msg, Timeline.TimelineType.Sent, msgKey); + } + + + private void addToTimeline(Msg msg, Timeline.TimelineType type, String msgKey) throws Exception { + String timelineKey = generateKeyFromMsg(msg, type); + + Location loc = new Location(new Namespace(BUCKET_NAME), timelineKey); + FetchValue fetch = new FetchValue.Builder(loc).build(); + Timeline timeline = client.execute(fetch).getValue(Timeline.class); + + if (timeline != null) { + timeline = addToExistingTimeline(timeline,msgKey); + } else { + timeline = createNewTimeline(msg, type, msgKey); + } + + StoreValue store = new StoreValue.Builder(timeline).build(); + client.execute(store); + } + + public Timeline createNewTimeline(Msg msg, Timeline.TimelineType type, String msgKey) { + String owner = getOwner(msg, type); + + Timeline newTimeline = new Timeline(); + newTimeline.Owner = owner; + newTimeline.Type = type.toString(); + newTimeline.Msgs.add(msgKey); + + return newTimeline; + } + + public Timeline addToExistingTimeline(Timeline timeline, String msgKey) { + timeline.Msgs.add(msgKey); + return timeline; + } + + public Timeline getTimeline(String ownerUsername, Timeline.TimelineType type, Date date) throws RiakRetryFailedException { + String timelineKey = generateKey(ownerUsername, type, date); + Bucket bucket = client.fetchBucket(BUCKET_NAME).execute(); + return bucket.fetch(timelineKey, Timeline.class).execute(); + } + + private String generateKeyFromMsg(Msg msg, Timeline.TimelineType type) { + String owner = getOwner(msg, type); + String dateString = msg.Created.substring(0, 10); + return generateKey(owner, type, dateString); + } + + private String getOwner(Msg msg, Timeline.TimelineType type) { + if(type == Timeline.TimelineType.Inbox) + return msg.Recipient; + else + return msg.Sender; + } + + private String generateKey(String ownerUsername, Timeline.TimelineType type, Date date) { + String dateString = getIso8601DateStringFromDate(date); + return generateKey(ownerUsername, type, dateString); + } + + private String generateKey(String ownerUsername, Timeline.TimelineType type, String dateString) { + return ownerUsername + "_" + type.toString() + "_" + dateString; + } + + private String getIso8601DateStringFromDate(Date date) { + TimeZone tz = TimeZone.getTimeZone("UTC"); + DateFormat df = new SimpleDateFormat("yyyy-MM-dd"); + df.setTimeZone(tz); + return df.format(date); + } + + +} + +// ---------------------------------------------------------------------------- + +package com.basho.msgy.Repositories; + +import com.basho.msgy.Models.User; +import com.basho.riak.client.IRiakClient; +import com.basho.riak.client.RiakRetryFailedException; +import com.basho.riak.client.bucket.Bucket; + +public class UserRepository { + static final String BUCKET_NAME = "Users"; + protected IRiakClient client; + + public UserRepository(IRiakClient client) { + this.client = client; + } + + public void save(User user) throws RiakRetryFailedException { + Bucket bucket = client.fetchBucket(BUCKET_NAME).execute(); + bucket.store(user).execute(); + } + + public User get(String UserName) throws RiakRetryFailedException { + Bucket bucket = client.fetchBucket(BUCKET_NAME).execute(); + return bucket.fetch(UserName, User.class).execute(); + } +} + +``` + +Finally, let's test them: + +```java +package com.basho.msgy; + +import com.basho.msgy.Models.Msg; +import com.basho.msgy.Models.Timeline; +import com.basho.msgy.Models.User; +import com.basho.msgy.Repositories.MsgRepository; +import com.basho.msgy.Repositories.TimelineRepository; +import com.basho.msgy.Repositories.UserRepository; +import com.basho.riak.client.IRiakClient; +import com.basho.riak.client.RiakException; +import com.basho.riak.client.RiakFactory; + +import java.util.Date; + +public class MsgyMain { + + public static void main(String[] args) throws RiakException { + // Setup our repositories + IRiakClient client = RiakFactory.pbcClient("127.0.0.1", 10017); + + UserRepository userRepo = new UserRepository(client); + MsgRepository msgRepo = new MsgRepository(client); + TimelineRepository timelineRepo = new TimelineRepository(client); + + // Create and save users + User marleen = new User("marleenmgr", + "Marleen Manager", + "marleen.manager@basho.com"); + + User joe = new User("joeuser", + "Joe User", + "joe.user@basho.com"); + + userRepo.save(marleen); + userRepo.save(joe); + + // Create new Msg, post to timelines + Msg msg = Msg.createNew(marleen.UserName, + joe.UserName, + "Welcome to the company!"); + + timelineRepo.postMsg(msg); + + + // Get Joe's inbox for today, get first message + Timeline joesInboxToday = timelineRepo.getTimeline(joe.UserName, + Timeline.TimelineType.Inbox, + new Date()); + + Msg joesFirstMsg = msgRepo.get(joesInboxToday.Msgs.get(0)); + + System.out.println("From: " + joesFirstMsg.Sender); + System.out.println("Msg : " + joesFirstMsg.Text); + System.out.println(""); + + client.shutdown(); + } +} +``` + +As you can see, the repository pattern helps us with a few things: + + - It helps us to see if an object exists before creating a new one + - It keeps our buckets and key names consistent + - It provides us with a consistent interface to work with. + +While this set of repositories solves many of our problems, it is very +minimal and doesn't cover all the edge cases. For instance, what happens +if two different people try to create a user with the same username? + +We can also easily "compute" key names now, but how do we quickly look +up the last 10 messages a user sent? Many of these answers will be +application dependent. If your application shows the last 10 messages in +reverse order, for example, you may want to store that set of data in +another collection object to make lookup faster. There are drawbacks to +every solution, but we recommend seeking out the key/value-based +solution first, as it will likely be the quickest. + +So to recap, in this chapter we learned: + +* How to choose bucket names +* How to choose natural keys based on how we want to partition our data diff --git a/content/riak/kv/2.2.6/developing/getting-started/java/querying.md b/content/riak/kv/2.2.6/developing/getting-started/java/querying.md new file mode 100644 index 0000000000..c37a3a8551 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/getting-started/java/querying.md @@ -0,0 +1,276 @@ +--- +title_supertext: "Getting Started:" +title: "Querying with Java" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Querying" + identifier: "getting_started_java_query" + weight: 101 + parent: "getting_started_java" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/taste-of-riak/querying-java + - /riak-docs/riak/kv/2.2.6/dev/taste-of-riak/querying-java +--- + +## Java Version Setup + +For the Java version, please download the source from GitHub by either +[cloning](https://github.com/basho/taste-of-riak) the source code +repository or downloading the [current zip of the master +branch](https://github.com/basho/taste-of-riak/archive/master.zip). +The code for this chapter is in `/java/Ch02-Schemas-and-Indexes`. You +may import this code into your favorite editor, or just run it from the +command line using the commands in `BuildAndRun.sh` if you are running +on a *nix* OS. + +## A Quick Note on Querying and Schemas + +_Schemas_? Yes, we said that correctly: S-C-H-E-M-A-S. It's not a dirty +word. Even in a key/value store, you will still have a logical database +schema of how all the data relates to other data. This can be as simple +as using the same key across multiple buckets for different types of +data to having fields in your data that are related by name. These +querying methods will introduce you to some ways of laying out your data +in Riak, along with how to query it back. + +## Denormalization + +If you're coming from a relational database, the easiest way to get your +application's feet wet with NoSQL is to denormalize your data into +related chunks. For example, with a customer database, you might have +separate tables for customers, addresses, preferences, etc. In Riak, +you can denormalize all that associated data into a single object and +store it into a `Customer` bucket. You can keep pulling in associated +data until you hit one of the big denormalization walls: + +* Size Limits (objects greater than 1MB) +* Shared/Referential Data (data that the object doesn't "own") +* Differences in Access Patterns (objects that get read/written once vs. + often) + +At one of these points we will have to split the model. + +## Same Keys, Different Buckets + +The simplest way to split up data would be to use the same identity key +across different buckets. A good example of this would be a `Customer` +object, an `Order` object, and an `OrderSummaries` object that keeps +rolled up info about orders such as total, etc. You can find the source +for these POJO's in `Customer.java`, `Order.java` and +`OrderSummaries.java`. Let's put some data into Riak so we can play +with it. + +```java +// From SipOfRiak.java + +private static Customer createCustomer() { + Customer customer = new Customer(); + customer.CustomerId = 1; + customer.Name = "John Smith"; + customer.Address = "123 Main Street"; + customer.City = "Columbus"; + customer.State = "Ohio"; + customer.Zip = "43210"; + customer.Phone = "+1-614-555-5555"; + customer.CreatedDate = "2013-10-01 14:30:26"; + return customer; +} + +private static ArrayList createOrders() { + ArrayList orders = new ArrayList(); + + Order order1 = new Order(); + order1.OrderId = 1; + order1.CustomerId = 1; + order1.SalespersonId = 9000; + order1.Items.add( + new Item("TCV37GIT4NJ", + "USB 3.0 Coffee Warmer", + 15.99)); + order1.Items.add( + new Item("PEG10BBF2PP", + "eTablet Pro; 24GB; Grey", + 399.99)); + order1.Total = 415.98; + order1.OrderDate = "2013-10-01 14:42:26"; + orders.add(order1); + + Order order2 = new Order(); + order2.OrderId = 2; + order2.CustomerId = 1; + order2.SalespersonId = 9001; + order2.Items.add( + new Item("OAX19XWN0QP", + "GoSlo Digital Camera", + 359.99)); + order2.Total = 359.99; + order2.OrderDate = "2013-10-15 16:43:16"; + orders.add(order2); + + Order order3 = new Order(); + order3.OrderId = 3; + order3.CustomerId = 1; + order3.SalespersonId = 9000; + order3.Items.add( + new Item("WYK12EPU5EZ", + "Call of Battle = Goats - Gamesphere 4", + 69.99)); + order3.Items.add( + new Item("TJB84HAA8OA", + "Bricko Building Blocks", + 4.99)); + order3.Total = 74.98; + order3.OrderDate = "2013-11-03 17:45:28"; + orders.add(order3); + return orders; +} + +private static OrderSummary createOrderSummary(ArrayList orders) { + OrderSummary orderSummary = new OrderSummary(); + orderSummary.CustomerId = 1; + for(Order order: orders) + { + orderSummary.Summaries.add(new OrderSummaryItem(order)); + } + return orderSummary; +} + +public static void main(String[] args) throws RiakException { + + System.out.println("Creating Data"); + Customer customer = createCustomer(); + ArrayList orders = createOrders(); + OrderSummary orderSummary = createOrderSummary(orders); + + System.out.println("Starting Client"); + IRiakClient client = RiakFactory.pbcClient("127.0.0.1", 10017); + + + System.out.println("Creating Buckets"); + Bucket customersBucket = client.fetchBucket("Customers").lazyLoadBucketProperties().execute(); + Bucket ordersBucket = client.fetchBucket("Orders").lazyLoadBucketProperties().execute(); + Bucket orderSummariesBucket = client.fetchBucket("OrderSummaries").lazyLoadBucketProperties().execute(); + + System.out.println("Storing Data"); + customersBucket.store(String.valueOf(customer.CustomerId), customer).execute(); + for (Order order : orders) { + ordersBucket.store(String.valueOf(order.OrderId), order).execute(); + } + orderSummariesBucket.store(String.valueOf(orderSummary.CustomerId), orderSummary).execute(); +``` + +While individual `Customer` and `Order` objects don't change much (or +shouldn't change), the `OrderSummaries` object will likely change often. +It will do double duty by acting as an index for all a customer's +orders, and also holding some relevant data such as the order total, +etc. If we showed this information in our application often, it's only +one extra request to get all the info. + +```java + System.out.println("Fetching related data by shared key"); + String key = "1"; + String fetchedCust = customersBucket.fetch(key).execute().getValueAsString(); + String fetchedOrdSum = orderSummariesBucket.fetch(key).execute().getValueAsString(); + System.out.format("Customer 1: %s\n", fetchedCust); + System.out.format("OrderSummary 1: %s\n", fetchedOrdSum); +``` + +Which returns our amalgamated objects: + +```bash +Fetching related data by shared key +Customer 1: {"CustomerId":1,"Name":"John Smith","Address":"123 Main Street","City":"Columbus","State":"Ohio","Zip":"43210","Phone":"+1-614-555-5555","CreatedDate":"2013-10-01 14:30:26"} +OrderSummary 1: {"CustomerId":1,"Summaries":[{"OrderId":1,"Total":415.98,"OrderDate":"2013-10-01 14:42:26"},{"OrderId":2,"Total":359.99,"OrderDate":"2013-10-15 16:43:16"},{"OrderId":3,"Total":74.98,"OrderDate":"2013-11-03 17:45:28"}]} +``` + +While this pattern is very easy and extremely fast with respect to +queries and complexity, it's up to the application to know about these +intrinsic relationships. + +## Secondary Indexes + +{{% note %}} +Secondary indexes in Riak KV require a sorted backend: [Memory]({{}}riak/kv/2.2.6/setup/planning/backend/memory) or [LevelDB]({{}}riak/kv/2.2.6/setup/planning/backend/leveldb). [Bitcask]({{}}riak/kv/2.2.6/setup/planning/backend/bitcask) does not support secondary indexes. + +See [Using Secondary Indexes (2i)]({{}}riak/kv/2.2.6/developing/usage/secondary-indexes) for more information on developing with secondary indexes. +{{% /note %}} + +If you're coming from an SQL world, Secondary Indexes (2i) are a lot +like SQL indexes. They are a way to quickly look up objects based on a +secondary key, without scanning through the whole dataset. This makes it +very easy to find groups of related data by values, or even ranges of +values. To properly show this off, we will now add some more data to our +application, and add some secondary index entries at the same time. + +```java + System.out.println("Adding Index Data"); + IRiakObject riakObj = ordersBucket.fetch("1").execute(); + riakObj.addIndex("SalespersonId", 9000); + riakObj.addIndex("OrderDate", "2013-10-01"); + ordersBucket.store(riakObj).execute(); + + IRiakObject riakObj2 = ordersBucket.fetch("2").execute(); + riakObj2.addIndex("SalespersonId", 9001); + riakObj2.addIndex("OrderDate", "2013-10-15"); + ordersBucket.store(riakObj2).execute(); + + IRiakObject riakObj3 = ordersBucket.fetch("3").execute(); + riakObj3.addIndex("SalespersonId", 9000); + riakObj3.addIndex("OrderDate", "2013-11-03"); + ordersBucket.store(riakObj3).execute(); +``` + +As you may have noticed, ordinary key/value data is opaque to 2i, so we +have to add entries to the indexes at the application level. Now let's +find all of Jane Appleseed's processed orders, we'll look up the orders +by searching the `SalespersonId` integer index for Jane's id of `9000`. + +```java + // Query for orders where the SalespersonId index is set to 9000 + List janesOrders = ordersBucket.fetchIndex(IntIndex.named("SalespersonId")) + .withValue(9000).execute(); + + System.out.format("Jane's Orders: %s\n", StringUtil.Join(", ", janesOrders)); +``` + +Which returns: + +```text +Jane's Orders: 1, 3 +``` + +Jane processed orders 1 and 3. We used an "integer" index to reference +Jane's ID, next let's use a "binary" index. Now, let's say that the VP +of Sales wants to know how many orders came in during October 2013. In +this case, we can exploit 2i's range queries. Let's search the +`OrderDate` binary index for entries between `2013-10-01` and +`2013-10-31`. + +```java + // Query for orders where the OrderDate index is between 2013-10-01 and 2013-10-31 + List octoberOrders = ordersBucket.fetchIndex(BinIndex.named("OrderDate")) + .from("2013-10-01").to("2013-10-31").execute(); + + System.out.format("October's Orders: %s\n", StringUtil.Join(", ", octoberOrders)); +``` + +Which returns: + +```text +October's Orders: 1, 2 +``` + +Boom! Easy-peasy. We used 2i's range feature to search for a range of +values, and demonstrated binary indexes. + +So to recap: + +* You can use Secondary Indexes to quickly look up an object based on a + secondary id other than the object's key. +* Indexes can have either Integer or Binary(String) keys +* You can search for specific values, or a range of values +* Riak will return a list of keys that match the index query diff --git a/content/riak/kv/2.2.6/developing/getting-started/nodejs.md b/content/riak/kv/2.2.6/developing/getting-started/nodejs.md new file mode 100644 index 0000000000..c46a18bbfc --- /dev/null +++ b/content/riak/kv/2.2.6/developing/getting-started/nodejs.md @@ -0,0 +1,100 @@ +--- +title: "Getting Started with NodeJS" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "NodeJS" + identifier: "getting_started_nodejs" + weight: 104 + parent: "developing_getting_started" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/taste-of-riak/nodejs + - /riak-docs/riak/kv/2.2.6/dev/taste-of-riak/nodejs +--- + +[introduction.js]: https://github.com/basho/riak-nodejs-client-examples/blob/master/dev/taste-of-riak/introduction.js +[npm]: https://www.npmjs.com/package/basho-riak-client +[node_js_installation]: https://github.com/basho/riak-nodejs-client/wiki/Installation +[nodejs_wiki]: https://github.com/basho/riak-nodejs-client/wiki + + +If you haven't set up a Riak Node and started it, please visit [Running A Cluster]({{}}riak/kv/2.2.6/using/running-a-cluster) first. + +To try this flavor of Riak, a working installation of Node.js 0.12 or later is +required. + +Code for these examples is available [here][introduction.js]. To run, follow +these directions: + +```bash +git clone git://github.com/basho/riak-nodejs-client-examples +cd riak-nodejs-client-examples +npm install +node ./app.js +``` + +### Client Setup + +Install [the Riak Node.js Client][node_js_installation] through [NPM][npm]. + +### Connecting to Riak + +Connecting to Riak with the Riak Node.js Client requires creating a new client +object and using the callback argument to know when the client is fully +initialized: + +```javascript +var Riak = require('basho-riak-client'); +var nodes = [ + 'riak-test:10017', + 'riak-test:10027', + 'riak-test:10037', + 'riak-test:10047' +]; +var client = new Riak.Client(nodes, function (err, c) { + // NB: at this point the client is fully initialized, and + // 'client' and 'c' are the same object +}); +``` + +This creates a new `Riak.Client` object which handles all the details of +tracking active nodes and also provides load balancing. The `Riak.Client` object +is used to send commands to Riak. When your application is completely done with +Riak communications, the following method can be used to gracefully shut the +client down and exit Node.js: + +```javascript +client.stop(function (err, rslt) { + // NB: you may wish to check err + process.exit(); +}); +``` + +Let's make sure the cluster is online with a `Ping` request: + +```javascript +var assert = require('assert'); + +client.ping(function (err, rslt) { + if (err) { + throw new Error(err); + } else { + // On success, ping returns true + assert(rslt === true); + } +}); +``` + +This is some simple code to test that a node in a Riak cluster is online - we +send a simple ping message. Even if the cluster isn't present, the Riak Node.js +Client will return a response message. In the callback it is important to check +that your activity was successful by checking the `err` variable. + +We are now ready to start interacting with Riak. + +## Next Steps + +[CRUD Operations]({{}}riak/kv/2.2.6/developing/getting-started/nodejs/crud-operations) diff --git a/content/riak/kv/2.2.6/developing/getting-started/nodejs/crud-operations.md b/content/riak/kv/2.2.6/developing/getting-started/nodejs/crud-operations.md new file mode 100644 index 0000000000..9f22f9475a --- /dev/null +++ b/content/riak/kv/2.2.6/developing/getting-started/nodejs/crud-operations.md @@ -0,0 +1,133 @@ +--- +title_supertext: "Getting Started:" +title: "CRUD Operations with NodeJS" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "CRUD Operations" + identifier: "getting_started_nodejs_crud" + weight: 100 + parent: "getting_started_nodejs" +toc: true +--- + +[nodejs_wiki]: https://github.com/basho/riak-nodejs-client/wiki + +### Creating Objects In Riak KV + +Pinging a Riak cluster sounds like a lot of fun, but eventually someone is going +to want us to do productive work. Let's create some data to save in Riak. + +The Riak Node.js Client makes use of a `RiakObject` class to encapsulate Riak +key/value objects. At the most basic, a `RiakObject` is responsible for +identifying your object and for translating it into a format that can be easily +saved to Riak. + +```javascript +var async = require('async'); + +var people = [ + { + emailAddress: "bashoman@basho.com", + firstName: "Basho", + lastName: "Man" + }, + { + emailAddress: "johndoe@gmail.com", + firstName: "John", + lastName: "Doe" + } +]; + +var storeFuncs = []; +people.forEach(function (person) { + // Create functions to execute in parallel to store people + storeFuncs.push(function (async_cb) { + client.storeValue({ + bucket: 'contributors', + key: person.emailAddress, + value: person + }, + function(err, rslt) { + async_cb(err, rslt); + } + ); + }); +}); + +async.parallel(storeFuncs, function (err, rslts) { + if (err) { + throw new Error(err); + } +}); +``` + +In this sample, we create a collection of `Person` objects and then save each +`Person` to Riak. Once again, we check the response from Riak. + +### Reading from Riak + +Let's find a person! + +```javascript +var logger = require('winston'); + +client.fetchValue({ bucket: 'contributors', key: 'bashoman@basho.com', convertToJs: true }, + function (err, rslt) { + if (err) { + throw new Error(err); + } else { + var riakObj = rslt.values.shift(); + var bashoman = riakObj.value; + logger.info("I found %s in 'contributors'", bashoman.emailAddress); + } + } +); +``` + +We use `client.fetchValue` to retrieve an object from Riak. This returns an +array of `RiakObject` objects which helpfully encapsulates the communication +with Riak. + +After verifying that we've been able to communicate with Riak *and* that we have +a successful result, we use the `value` property to get the object, which has +already been converted to a javascript object due to the use of `convertToJs: +true` in the options. + +### Modifying Existing Data + +Let's say that Basho Man has decided to be known as Riak Man: + +```javascript +bashoman.FirstName = "Riak"; +riakObj.setValue(bashoman); + +client.storeValue({ value: riakObj }, function (err, rslt) { + if (err) { + throw new Error(err); + } +}); +``` + +Updating an object involves modifying a `RiakObject` then using +`client.storeValue` to save the existing object. + +### Deleting Data + +```javascript +client.deleteValue({ bucket: 'contributors', key: 'johndoe@gmail.com' }, function (err, rslt) { + if (err) { + throw new Error(err); + } +}); +``` + +Just like other operations, we check the results that have come back from Riak +to make sure the object was successfully deleted. + +The Riak Node.js Client has a lot of additional functionality that makes it easy +to build rich, complex applications with Riak. Check out the +[documentation][nodejs_wiki] to learn more about working with the Riak Node.js +Client and Riak. diff --git a/content/riak/kv/2.2.6/developing/getting-started/nodejs/object-modeling.md b/content/riak/kv/2.2.6/developing/getting-started/nodejs/object-modeling.md new file mode 100644 index 0000000000..2c3fcd5263 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/getting-started/nodejs/object-modeling.md @@ -0,0 +1,119 @@ +--- +title_supertext: "Getting Started:" +title: "Object Modeling with NodeJS" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Object Modeling" + identifier: "getting_started_nodejs_object" + weight: 102 + parent: "getting_started_nodejs" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/taste-of-riak/object-modeling-nodejs + - /riak-docs/riak/kv/2.2.6/dev/taste-of-riak/object-modeling-nodejs +--- + +To get started, let's create the models that we'll be using. + +* [`Msg`](https://github.com/basho/taste-of-riak/blob/master/nodejs/Ch03-Msgy-Schema/models/msg.js) +* [`Timeline`](https://github.com/basho/taste-of-riak/blob/master/nodejs/Ch03-Msgy-Schema/models/timeline.js) +* [`User`](https://github.com/basho/taste-of-riak/blob/master/nodejs/Ch03-Msgy-Schema/models/user.js) + +To use these classes to store data, we will first have to create a user. +Then, when a user creates a message, we will append that message to one +or more timelines. If it's a private message, we'll append it to the +Recipient's `Inbox` timeline and the User's own `Sent` timeline. If it's +a group message, we'll append it to the Group's timeline, as well as to +the User's `Sent` timeline. + +#### Buckets and Keys Revisited + +Now that we've worked out how we will differentiate data in the system, +let's figure out our bucket and key names. + +The bucket names are straightforward. We can use `Users`, `Msgs`, and +`Timelines`. The key names, however, are a little more tricky. In past +examples we've used sequential integers, but this presents a problem: we +would need a secondary service to hand out these IDs. This service could +easily be a future bottleneck in the system, so let's use a natural key. +Natural keys are a great fit for key/value systems because both humans +and computers can easily construct them when needed, and most of the +time they can be made unique enough for a KV store. + + +| Bucket | Key Pattern | Example Key +|:-------|:------------|:----------- +| `Users` | `` | `joeuser` +| `Msgs` | `_` | `joeuser_2014-03-06T02:05:13.556Z` +| `Timelines` | `__` | `joeuser_SENT_2014-03-06`
`marketing_group_INBOX_2014-03-06` | + +For the `Users` bucket, we can be certain that we will want each +username to be unique, so let's use the `userName` as the key. + +[*Example:* `userName` as key](https://github.com/basho/taste-of-riak/blob/master/nodejs/Ch03-Msgy-Schema/models/user.js#L19-L20) + +For the `Msgs` bucket, let's use a combination of the username and the +posting datetime in an [ISO 8601 +Long](http://en.wikipedia.org/wiki/ISO_8601) format. This combination +gives us the pattern `_`, which produces keys like +`joeuser_2014-03-05T23:20:28Z`. + +[*Example:* `Msg` key](https://github.com/basho/taste-of-riak/blob/master/nodejs/Ch03-Msgy-Schema/models/msg.js#L25-L27) + +Now for `Timelines`, we need to differentiate between `Inbox` and `Sent` +timelines, so we can simply add that type into the key name. We will +also want to partition each collection object into some time period, +that way the object doesn't grow too large (see note below). + +For `Timelines`, let's use the pattern `__` for +users, and `_Inbox_` for groups, which will look like +`joeuser_SENT_2014-03-06` or `marketing_group_INBOX_2014-03-05`, +respectively. + +{{% note title="Note" %}} +Riak performs best with objects under 1-2MB. Objects larger than that can hurt +performance, especially many siblings are being created. We will cover +siblings, sibling resolution, and sibling explosions in the next chapter. +{{% /note %}} + +#### Keeping our story straight with repositories + +Now that we've figured out our object model, let's write some +repositories to help create and work with these objects in Riak: + +* [Base `Repository` class](https://github.com/basho/taste-of-riak/blob/master/nodejs/Ch03-Msgy-Schema/repositories/repository.js) +* [`UserRepository` class](https://github.com/basho/taste-of-riak/blob/master/nodejs/Ch03-Msgy-Schema/repositories/user-repository.js) +* [`MsgRepository` class](https://github.com/basho/taste-of-riak/blob/master/nodejs/Ch03-Msgy-Schema/repositories/msg-repository.js) +* [`TimelineRepository` class](https://github.com/basho/taste-of-riak/blob/master/nodejs/Ch03-Msgy-Schema/repositories/timeline-repository.js) +* [`TimelineManager` class that manages `Msg` and `Timeline` objects](https://github.com/basho/taste-of-riak/blob/master/nodejs/Ch03-Msgy-Schema/timeline-manager.js) + +Finally, let's test them: + +[*Example:* Putting it all together](https://github.com/basho/taste-of-riak/blob/master/nodejs/Ch03-Msgy-Schema/app.js) + +As you can see, the repository pattern helps us with a few things: + + - It helps us to see if an object exists before creating a new one + - It keeps our buckets and key names consistent + - It provides us with a consistent interface to work with. + +While this set of repositories solves many of our problems, it is very +minimal and doesn't cover all the edge cases. For instance, what happens +if two different people try to create a user with the same username? + +We can also easily "compute" key names now, but how do we quickly look +up the last 10 messages a user sent? Many of these answers will be +application dependent. If your application shows the last 10 messages in +reverse order, for example, you may want to store that set of data in +another collection object to make lookup faster. There are drawbacks to +every solution, but we recommend seeking out the key/value-based +solution first, as it will likely be the quickest. + +So to recap, in this chapter we learned: + +* How to choose bucket names +* How to choose natural keys based on how we want to partition our data + diff --git a/content/riak/kv/2.2.6/developing/getting-started/nodejs/querying.md b/content/riak/kv/2.2.6/developing/getting-started/nodejs/querying.md new file mode 100644 index 0000000000..c01204da0b --- /dev/null +++ b/content/riak/kv/2.2.6/developing/getting-started/nodejs/querying.md @@ -0,0 +1,142 @@ +--- +title_supertext: "Getting Started:" +title: "Querying with NodeJS" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Querying" + identifier: "getting_started_nodejs_query" + weight: 101 + parent: "getting_started_nodejs" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/taste-of-riak/querying-nodejs + - /riak-docs/riak/kv/2.2.6/dev/taste-of-riak/querying-nodejs +--- + +## Node.js Version Setup + +For the Node.js version, please download the source from GitHub by either +[cloning](https://github.com/basho/taste-of-riak) the source code +repository or downloading the [current zip of the master +branch](https://github.com/basho/taste-of-riak/archive/master.zip). +The code for this chapter is in `nodejs/Ch02-Schemas-and-Indexes`. Be +sure to run `npm install` in this directory prior to running `node +./app.js` to run the code. + +## A Quick Note on Querying and Schemas + +_Schemas_? Yes, we said that correctly: S-C-H-E-M-A-S. It's not a dirty +word. Even in a key/value store, you will still have a logical database +schema of how all the data relates to other data. This can be as simple +as using the same key across multiple buckets for different types of +data to having fields in your data that are related by name. These +querying methods will introduce you to some ways of laying out your data +in Riak, along with how to query it back. + +## Denormalization + +If you're coming from a relational database, the easiest way to get your +application's feet wet with NoSQL is to denormalize your data into +related chunks. For example, with a customer database, you might have +separate tables for customers, addresses, preferences, etc. In Riak, +you can denormalize all that associated data into a single object and +store it into a `Customer` bucket. You can keep pulling in associated +data until you hit one of the big denormalization walls: + +* Size Limits (objects greater than 1MB) +* Shared/Referential Data (data that the object doesn't "own") +* Differences in Access Patterns (objects that get read/written once vs. + often) + +At one of these points we will have to split the model. + +## Same Keys, Different Buckets + +The simplest way to split up data would be to use the same identity key +across different buckets. A good example of this would be a `Customer` +object, an `Order` object, and an `OrderSummaries` object that keeps +rolled up info about orders such as total, etc. Let's put some data into +Riak so we can play with it. + +* [*Example:* Creating a customer](https://github.com/basho/taste-of-riak/blob/master/nodejs/Ch02-Schemas-and-Indexes/app.js#L24-L33) +* [*Example:* Creating orders and order summaries](https://github.com/basho/taste-of-riak/blob/master/nodejs/Ch02-Schemas-and-Indexes/app.js#L193-L262) + +While individual Customer and Order objects don't change much (or +shouldn't change), the "Order Summary" object will likely change often. +It will do double duty by acting as an index for all a customer's +orders, and also holding some relevant data such as the order total, +etc. If we showed this information in our application often, it's only +one extra request to get all the info. + +[*Example:* Fetching by shared key](https://github.com/basho/taste-of-riak/blob/master/nodejs/Ch02-Schemas-and-Indexes/app.js#L78-L96) + +Which returns our amalgamated objects: + +```bash +info: Customer 1: {"id":"1","name":"John Smith","address":"123 Main Street","city":"Columbus","state":"Ohio","zip":"43210","phone":"+1-614-555-5555","createdDate":"2013-10-01 14:30:26"} +info: OrderSummary 1: {"customerId":"1","summaries":[{"orderId":"1","total":415.98,"orderDate":"2013-10-01 14:42:26"},{"orderId":"2","total":359.99,"orderDate":"2013-10-15 16:43:16"},{"orderId":"3","total":74.98,"orderDate":"2013-11-03 17:45:28"}]} +``` + +While this pattern is very easy and extremely fast with respect to +queries and complexity, it's up to the application to know about these +intrinsic relationships. + +## Secondary Indexes + +{{% note %}} +Secondary indexes in Riak KV require a sorted backend: [Memory]({{}}riak/kv/2.2.6/setup/planning/backend/memory) or [LevelDB]({{}}riak/kv/2.2.6/setup/planning/backend/leveldb). [Bitcask]({{}}riak/kv/2.2.6/setup/planning/backend/bitcask) does not support secondary indexes. + +See [Using Secondary Indexes (2i)]({{}}riak/kv/2.2.6/developing/usage/secondary-indexes) for more information on developing with secondary indexes. +{{% /note %}} + +If you're coming from an SQL world, Secondary Indexes (2i) are a lot +like SQL indexes. They are a way to quickly look up objects based on a +secondary key, without scanning through the whole dataset. This makes it +very easy to find groups of related data by values, or even ranges of +values. To properly show this off, we will now add some more data to our +application, and add some secondary index entries at the same time. + +[*Example:* Adding index data](https://github.com/basho/taste-of-riak/blob/master/nodejs/Ch02-Schemas-and-Indexes/app.js#L98-L141) + +As you may have noticed, ordinary key/value data is opaque to 2i, so we +have to add entries to the indexes at the application level. Now let's +find all of Jane Appleseed's processed orders, we'll look up the orders +by searching the `SalespersonId` integer index for Jane's id of `9000`. + +[*Example:* Query for orders where the SalespersonId index is set to 9000](https://github.com/basho/taste-of-riak/blob/master/nodejs/Ch02-Schemas-and-Indexes/app.js#L143-L159) + +Which returns: + +```text +Jane's Orders: 1, 3 +``` + +Jane processed orders 1 and 3. We used an "integer" index to reference +Jane's ID, next let's use a "binary" index. Now, let's say that the VP +of Sales wants to know how many orders came in during October 2013. In +this case, we can exploit 2i's range queries. Let's search the +`OrderDate` binary index for entries between `2013-10-01` and +`2013-10-31`. + +[*Example:* Query for orders where the OrderDate index is between 2013-10-01 and +2013-10-31](https://github.com/basho/taste-of-riak/blob/master/nodejs/Ch02-Schemas-and-Indexes/app.js#L161-175) + +Which returns: + +```text +October's Orders: 1, 2 +``` + +Boom! Easy-peasy. We used 2i's range feature to search for a range of +values, and demonstrated binary indexes. + +So to recap: + +* You can use Secondary Indexes to quickly look up an object based on a + secondary id other than the object's key. +* Indexes can have either Integer or Binary(String) keys +* You can search for specific values, or a range of values +* Riak will return a list of keys that match the index query diff --git a/content/riak/kv/2.2.6/developing/getting-started/php.md b/content/riak/kv/2.2.6/developing/getting-started/php.md new file mode 100644 index 0000000000..9d9c37afb0 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/getting-started/php.md @@ -0,0 +1,76 @@ +--- +title: "Getting Started with PHP" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "PHP" + identifier: "getting_started_php" + weight: 107 + parent: "developing_getting_started" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/taste-of-riak/php + - /riak-docs/riak/kv/2.2.6/dev/taste-of-riak/php +--- + +If you haven't set up a Riak Node and started it, please visit [Running A Cluster]({{}}riak/kv/2.2.6/using/running-a-cluster) first. + +To try this flavor of Riak, a working installation of PHP is required, and [Composer](https://getcomposer.org/) is required to be installed to fetch the client library package. + +## Client Setup +Download and unzip, or clone the Taste of Riak Sample Code Repository from GitHub ([zip](https://github.com/basho/taste-of-riak/archive/master.zip), [github repository](https://github.com/basho/taste-of-riak)). + +From the `taste-of-riak` directory, use composer to install the Riak PHP 2.0 Client`. + +```bash +php path/to/your/composer.phar install + +# If you did a global install of composer, run this instead: +composer install +``` + +If you set up a local Riak cluster using the [[five minute install]] method, change line 11 from `->onPort(8098)` to `->onPort(10018)`. + +Next, run `php Ch01-CRUD/taste-of-riak.php` to run this chapter's example code. It should output: + +```json +Reading Objects From Riak... +Updating Objects In Riak... +Deleting Objects From Riak... +Working With Complex Objects... +Serialized Object: +{"title":"Moby Dick","author":"Herman Melville","body":"Call me Ishmael. Some years ago...","isbn":"1111979723","copiesOwned":3} +``` + +Yay, success! + +Since we didn't use PHP's REPL environment, let's walk through the code +to see what it actually did at each step. + +## Setting up the PHP Client and connections + +```php +include_once 'vendor/autoload.php'; + +use Basho\Riak; +use Basho\Riak\Node; +use Basho\Riak\Command; + +$node = (new Node\Builder) + ->atHost('127.0.0.1') + ->onPort(8098) + ->build(); + +$riak = new Riak([$node]); +``` + +This code will load the library, declare the necessary `use` statements for our code, and then initialize and configure a [Node Builder](http://basho.github.io/riak-php-client/class-Basho.Riak.Node.Builder.html). +Once we call `build()` on the builder, it will return to us a [Node](http://basho.github.io/riak-php-client/class-Basho.Riak.Node.html) object, which we use when building our Riak commands. + +We are now ready to start interacting with Riak. + +## Next Steps + +[CRUD Operations]({{}}riak/kv/2.2.6/developing/getting-started/php/crud-operations) diff --git a/content/riak/kv/2.2.6/developing/getting-started/php/crud-operations.md b/content/riak/kv/2.2.6/developing/getting-started/php/crud-operations.md new file mode 100644 index 0000000000..2a4c011e84 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/getting-started/php/crud-operations.md @@ -0,0 +1,182 @@ +--- +title_supertext: "Getting Started:" +title: "CRUD Operations with PHP" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "CRUD Operations" + identifier: "getting_started_php_crud" + weight: 100 + parent: "getting_started_php" +toc: true +--- + +## Creating Objects In Riak +First, let’s create a few objects and a bucket to keep them in. + +```php +$bucket = new Riak\Bucket('testBucket'); + +$val1 = 1; +$location1 = new Riak\Location('one', $bucket); + +$storeCommand1 = (new Command\Builder\StoreObject($riak)) + ->buildObject($val1) + ->atLocation($location1) + ->build(); +$storeCommand1->execute(); +``` + +In this first example we have stored the integer 1 with the lookup key of ‘one’. Next let’s store a simple string value of “two” with a matching key. + +```php +$val2 = 'two'; +$location2 = new Riak\Location('two', $bucket); + +$storeCommand2 = (new Command\Builder\StoreObject($riak)) + ->buildObject($val2) + ->atLocation($location2) + ->build(); +$storeCommand2->execute(); +``` + +That was easy. Finally, let’s store an associative array. You will probably recognize the pattern by now. + +```php +$val3 = ['myValue' => 3]; +$location3 = new Riak\Location('three', $bucket); + +$storeCommand3 = (new Command\Builder\StoreObject($riak)) + ->buildJsonObject($val3) + ->atLocation($location3) + ->build(); +$storeCommand3->execute(); +``` + +## Reading Objects From Riak +Now that we have a few objects stored, let’s retrieve them and make sure they contain the values we expect. + +```php +$response1 = (new Command\Builder\FetchObject($riak)) + ->atLocation($location1) + ->build() + ->execute(); + +$response2 = (new Command\Builder\FetchObject($riak)) + ->atLocation($location2) + ->build() + ->execute(); + +$response3 = (new Command\Builder\FetchObject($riak)) + ->atLocation($location3) + ->withDecodeAsAssociative() + ->build() + ->execute(); + +print_r($response1->getObject()->getData()); +print_r($response2->getObject()->getData()); +print_r($response3->getObject()->getData()); +``` + +That was easy. We create a [Fetch Command](http://basho.github.io/riak-php-client/class-Basho.Riak.Command.Object.Fetch.html) from a [FetchObject Builder](http://basho.github.io/riak-php-client/class-Basho.Riak.Command.Builder.FetchObject.html). +For our object that is an associative array, we also add [`withDecodeAsAssociative()`](http://basho.github.io/riak-php-client/class-Basho.Riak.Command.Builder.FetchObject.html#_withDecodeAsAssociative) to the builder so it returns the object as an associative array instead of an stdClass object. + +In either case, we'll get a [Response](http://basho.github.io/riak-php-client/class-Basho.Riak.Command.Object.Response.html) object back, which holds information about the operation, and the result data. + +## Updating Objects In Riak +While some data may be static, other forms of data may need to be updated. This is also easy to accomplish. Let’s update the value of myValue in the 3rd example to 42. + +```php +$object3 = $response3->getObject(); +$data3 = $object3->getData(); + +$data3['myValue'] = 42; +$object3 = $object3->setData(json_encode($data3)); + +$updateCommand = (new Command\Builder\StoreObject($riak)) + ->withObject($object3) + ->atLocation($location3) + ->build(); + +$updateCommand->execute(); +``` + +First we get the Riak [Object](http://basho.github.io/riak-php-client/class-Basho.Riak.Object.html) from the [Response](http://basho.github.io/riak-php-client/class-Basho.Riak.Command.Object.Response.html), then we get the stored data with [`getData()`](http://basho.github.io/riak-php-client/class-Basho.Riak.Object.html#_getData). We update the data to our liking, then use [`setData()`](http://basho.github.io/riak-php-client/class-Basho.Riak.Object.html#_setData) to set the new data back to the Riak Object. +To store it we use the same pattern as before, but this time we use the [`withObject()`](http://basho.github.io/riak-php-client/class-Basho.Riak.Command.Builder.ObjectTrait.html#_withObject) method to tell it to store our updated Riak Object. + +## Deleting Objects From Riak +As a last step, we’ll demonstrate how to delete data. We just build a [Delete Command](http://basho.github.io/riak-php-client/class-Basho.Riak.Command.Object.Delete.html) from a [DeleteObject Builder](http://basho.github.io/riak-php-client/class-Basho.Riak.Command.Builder.DeleteObject.html), and execute it. + +```php +(new Command\Builder\DeleteObject($riak))->atLocation($location1)->build()->execute(); +(new Command\Builder\DeleteObject($riak))->atLocation($location2)->build()->execute(); +(new Command\Builder\DeleteObject($riak))->atLocation($location3)->build()->execute(); +``` + +### Working With Complex Objects +Since the world is a little more complicated than simple integers and bits of strings, let’s see how we can work with more complex objects. Take for example, this plain old PHP object(POPO) that encapsulates some knowledge about a book. + +```php +class Book +{ + var $title; + var $author; + var $body; + var $isbn; + var $copiesOwned; +} + +$book = new Book(); +$book->isbn = '1111979723'; +$book->title = 'Moby Dick'; +$book->author = 'Herman Melville'; +$book->body = 'Call me Ishmael. Some years ago...'; +$book->copiesOwned = 3; +``` + +Ok, so we have some information about our Moby Dick collection that we want to save. Storing this to Riak should look familiar by now: + +```php +$bookLocation = new Riak\Location($book->isbn, new Riak\Bucket('books')); + +$storeCommand1 = (new Command\Builder\StoreObject($riak)) + ->buildJsonObject($book) + ->atLocation($bookLocation) + ->build(); + +$storeCommand1->execute(); +``` + +Some of you may be thinking “But how does the Riak client encode/decode my object”? If we fetch the binary version of our book back and print it as a string, we shall know: + +```php +$fetchBookResponse = (new Command\Builder\FetchObject($riak)) + ->atLocation($bookLocation) + ->build() + ->execute(); + +print('Serialized Object:' . PHP_EOL); +print($fetchBookResponse->getBody() . PHP_EOL); +``` + +```json +Serialized Object: +{"title":"Moby Dick","author":"Herman Melville","body":"Call me Ishmael. Some years ago...","isbn":"1111979723","copiesOwned":3} +``` + +JSON! The library encodes PHP objects as JSON strings when you use the [`buildJsonObject()`](http://basho.github.io/riak-php-client/class-Basho.Riak.Command.Builder.ObjectTrait.html#_buildJsonObject) method on the StoreObject builder. + +Now that we’ve ruined the magic of object encoding, let’s clean up our mess: + +```php +(new Command\Builder\DeleteObject($riak)) + ->atLocation($bookLocation) + ->build() + ->execute(); +``` + +## Next Steps + +More complex use cases can be composed from these initial create, read, update, and delete (CRUD) operations. [In the next chapter]({{}}riak/kv/2.2.6/developing/getting-started/php/querying) we will look at how to store and query more complicated and interconnected data, such as documents. diff --git a/content/riak/kv/2.2.6/developing/getting-started/php/querying.md b/content/riak/kv/2.2.6/developing/getting-started/php/querying.md new file mode 100644 index 0000000000..18c0223418 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/getting-started/php/querying.md @@ -0,0 +1,404 @@ +--- +title_supertext: "Getting Started:" +title: "Querying with PHP" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Querying" + identifier: "getting_started_php_query" + weight: 101 + parent: "getting_started_php" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/taste-of-riak/querying-php + - /riak-docs/riak/kv/2.2.6/dev/taste-of-riak/querying-php +--- + +## A Quick Note on Querying and Schemas +_Schemas_? Yes we said that correctly, S-C-H-E-M-A-S. It's not a dirty word. +Even with a Key/Value store, you will still have a logical database schema of how all the data relates to one another. This can be as simple as using the same key across multiple buckets for different types of data, to having fields in your data that are related by name. These querying methods will introduce you to some ways of laying out your data in Riak, along with how to query it back. + +## Denormalization + +If you're coming from a relational database, the easiest way to get your application's feet wet with NoSQL is to denormalize your data into related chunks. For example with a customer database, you might have separate tables for Customers, Addresses, Preferences, etc. In Riak, you can denormalize all that associated data into a single object and store it into a `Customer` bucket. You can keep pulling in associated data until you hit one of the big denormalization walls: + +* Size Limits (objects greater than 1MB) +* Shared/Referential Data (data that the object doesn't "own") +* Differences in Access Patterns (objects that get read/written once vs. often) + +At one of these points we will have to split the model. + +## Same Keys - Different Buckets + +The simplest way to split up data would be to use the same identity key across different buckets. A good example of this would be a `Customer` object, an `Order` object, and an `OrderSummaries` object that keeps rolled up info about orders such as Total, etc. Let's put some data into Riak so we can play with it. + +```php +atHost('127.0.0.1') + ->onPort(8098) + ->build(); + +$riak = new Riak([$node]); + +// Class definitions for our models + +class Customer +{ + var $customerId; + var $name; + var $address; + var $city; + var $state; + var $zip; + var $phone; + var $createdDate; +} + +class Order +{ + public function __construct() + { + $this->items = array(); + } + var $orderId; + var $customerId; + var $salespersonId; + var $items; + var $total; + var $orderDate; +} + +class Item +{ + public function __construct($itemId, $title, $price) + { + $this->itemId = $itemId; + $this->title = $title; + $this->price = $price; + } + var $itemId; + var $title; + var $price; +} + +class OrderSummary +{ + public function __construct() + { + $this->summaries = array(); + } + var $customerId; + var $summaries; +} + +class OrderSummaryItem +{ + public function __construct(Order $order) + { + $this->orderId = $order->orderId; + $this->total = $order->total; + $this->orderDate = $order->orderDate; + } + var $orderId; + var $total; + var $orderDate; +} + + +// Creating Data +$customer = new Customer(); +$customer->customerId = 1; +$customer->name = 'John Smith'; +$customer->address = '123 Main Street'; +$customer->city = 'Columbus'; +$customer->state = 'Ohio'; +$customer->zip = '43210'; +$customer->phone = '+1-614-555-5555'; +$customer->createdDate = '2013-10-01 14:30:26'; + + +$orders = []; + +$order1 = new Order(); +$order1->orderId = 1; +$order1->customerId = 1; +$order1->salespersonId = 9000; +$order1->items = [ + new Item( + 'TCV37GIT4NJ', + 'USB 3.0 Coffee Warmer', + 15.99 + ), + new Item( + 'PEG10BBF2PP', + 'eTablet Pro; 24GB; Grey', + 399.99 + ) +]; +$order1->total = 415.98; +$order1->orderDate = '2013-10-01 14:42:26'; +$orders[] = $order1; + +$order2 = new Order(); +$order2->orderId = 2; +$order2->customerId = 1; +$order2->salespersonId = 9001; +$order2->items = [ + new Item( + 'OAX19XWN0QP', + 'GoSlo Digital Camera', + 359.99 + ) +]; +$order2->total = 359.99; +$order2->orderDate = '2013-10-15 16:43:16'; +$orders[] = $order2; + +$order3 = new Order(); +$order3->orderId = 3; +$order3->customerId = 1; +$order3->salespersonId = 9000; +$order3->items = [ + new Item( + 'WYK12EPU5EZ', + 'Call of Battle = Goats - Gamesphere 4', + 69.99 + ), + new Item( + 'TJB84HAA8OA', + 'Bricko Building Blocks', + 4.99 + ) +]; +$order3->total = 74.98; +$order3->orderDate = '2013-11-03 17:45:28'; +$orders[] = $order3; + + +$orderSummary = new OrderSummary(); +$orderSummary->customerId = 1; +foreach ($orders as $order) { + $orderSummary->summaries[] = new OrderSummaryItem($order); +} +unset($order); + + + +// Starting Client +$node = (new Node\Builder) + ->atHost('127.0.0.1') + ->onPort(8098) + ->build(); + +$riak = new Riak([$node]); + +// Creating Buckets +$customersBucket = new Riak\Bucket('Customers'); +$ordersBucket = new Riak\Bucket('Orders'); +$orderSummariesBucket = new Riak\Bucket('OrderSummaries'); + +// Storing Data +$storeCustomer = (new Command\Builder\StoreObject($riak)) + ->buildJsonObject($customer) + ->atLocation(new Location($customer->customerId, $customersBucket)) + ->build(); +$storeCustomer->execute(); + +foreach ($orders as $order) { + $storeOrder = (new Command\Builder\StoreObject($riak)) + ->buildJsonObject($order) + ->atLocation(new Location($order->orderId, $ordersBucket)) + ->build(); + $storeOrder->execute(); +} +unset($order); + +$storeSummary = (new Command\Builder\StoreObject($riak)) + ->buildJsonObject($orderSummary) + ->atLocation(new Location($orderSummary->customerId, $orderSummariesBucket)) + ->build(); +$storeSummary->execute(); +``` + + While individual `Customer` and `Order` objects don't change much (or shouldn't change), the `Order Summaries` object will likely change often. It will do double duty by acting as an index for all a customer's orders, and also holding some relevant data such as the order total, etc. If we showed this information in our application often, it's only one extra request to get all the info. + +```php +// Fetching related data by shared key +$fetched_customer = (new Command\Builder\FetchObject($riak)) + ->atLocation(new Location('1', $customersBucket)) + ->build()->execute()->getObject()->getData(); + +$fetched_customer->orderSummary = + (new Command\Builder\FetchObject($riak)) + ->atLocation(new Location('1', $orderSummariesBucket)) + ->build()->execute()->getObject()->getData(); + +print("Customer with OrderSummary data: \n"); +print_r($fetched_customer); +``` + +Which returns our amalgamated objects: + +```text +Customer with OrderSummary data: +stdClass Object +( + [customerId] => 1 + [name] => John Smith + [address] => 123 Main Street + [city] => Columbus + [state] => Ohio + [zip] => 43210 + [phone] => +1-614-555-5555 + [createdDate] => 2013-10-01 14:30:26 + [orderSummary] => stdClass Object + ( + [customerId] => 1 + [summaries] => Array + ( + [0] => stdClass Object + ( + [orderId] => 1 + [total] => 415.98 + [orderDate] => 2013-10-01 14:42:26 + ) + + [1] => stdClass Object + ( + [orderId] => 2 + [total] => 359.99 + [orderDate] => 2013-10-15 16:43:16 + ) + + [2] => stdClass Object + ( + [orderId] => 3 + [total] => 74.98 + [orderDate] => 2013-11-03 17:45:28 + ) + ) + ) +) +``` + +While this pattern is very easy and extremely fast with respect to queries and complexity, it's up to the application to know about these intrinsic relationships. + + +## Secondary Indexes + +{{% note %}} +Secondary indexes in Riak KV require a sorted backend: [Memory]({{}}riak/kv/2.2.6/setup/planning/backend/memory) or [LevelDB]({{}}riak/kv/2.2.6/setup/planning/backend/leveldb). [Bitcask]({{}}riak/kv/2.2.6/setup/planning/backend/bitcask) does not support secondary indexes. + +See [Using Secondary Indexes (2i)]({{}}riak/kv/2.2.6/developing/usage/secondary-indexes) for more information on developing with secondary indexes. +{{% /note %}} + +If you're coming from a SQL world, Secondary Indexes (2i) are a lot like SQL indexes. They are a way to quickly lookup objects based on a secondary key, without scanning through the whole dataset. This makes it very easy to find groups of related data by values, or even ranges of values. To properly show this off, we will now add some more data to our application, and add some secondary index entries at the same time. + +```php +// Adding Index Data +$keys = array(1,2,3); +foreach ($keys as $key) { + $orderLocation = new Location($key, $ordersBucket); + $orderObject = (new Command\Builder\FetchObject($riak)) + ->atLocation($orderLocation) + ->build()->execute()->getObject(); + + $order = $orderObject->getData(); + + $orderObject->addValueToIndex('SalespersonId_int', $order->salespersonId); + $orderObject->addValueToIndex('OrderDate_bin', $order->orderDate); + + $storeOrder = (new Command\Builder\StoreObject($riak)) + ->withObject($orderObject) + ->atLocation($orderLocation) + ->build(); + $storeOrder->execute(); +} +unset($key); + +``` + +As you may have noticed, ordinary Key/Value data is opaque to 2i, so we have to add entries to the indexes at the application level. +Now let's find all of Jane Appleseed's processed orders, we'll lookup the orders by searching the `saleperson_id_int` index for Jane's id of `9000`. + +```php +// Query for orders where the SalespersonId int index is set to 9000 +$fetchIndex = (new Command\Builder\QueryIndex($riak)) + ->inBucket($ordersBucket) + ->withIndexName('SalespersonId_int') + ->withScalarValue(9000)->build(); +$janes_orders = $fetchIndex->execute()->getResults(); + +print("\n\nJane's Orders: \n"); +print_r($janes_orders); +``` + +Which returns: + +```text +Jane's Orders: +Array +( + [0] => 3 + [1] => 1 +) + +``` + +Jane processed orders 1 and 3. We used an "integer" index to reference Jane's id, next let's use a "binary" index. +Now, let's say that the VP of Sales wants to know how many orders came in during October 2013. In this case, we can exploit 2i's range queries. Let's search the `order_date_bin` index for entries between `20131001` and `20131031`. + +```php +// Query for orders where the OrderDate bin index is +// between 2013-10-01 and 2013-10-31 +$fetchOctoberOrders = (new Command\Builder\QueryIndex($riak)) + ->inBucket($ordersBucket) + ->withIndexName('OrderDate_bin') + ->withRangeValue('2013-10-01','2013-10-31') + ->withReturnTerms(true) + ->build(); + +$octobers_orders = $fetchOctoberOrders->execute()->getResults(); + +print("\n\nOctober's Orders: \n"); +print_r($octobers_orders); +?> +``` + +Which returns: + +```text +October's Orders: +Array +( + [0] => Array + ( + [2013-10-01 14:42:26] => 1 + ) + + [1] => Array + ( + [2013-10-15 16:43:16] => 2 + ) +) +``` + +Boom, easy-peasy. We used 2i's range feature to search for a range of values, and demonstrated binary indexes. With the October's Orders query we also used the `->withReturnTerms(true)` option, which as you can see will return the values of the matching 2i terms. + +So to recap: + +* You can use Secondary Indexes to quickly lookup an object based on a secondary id other than the object's key. +* Indexes can have either Integer or Binary(String) keys +* You can search for specific values, or a range of values +* Riak will return a list of keys (and terms if needed) that match the index query diff --git a/content/riak/kv/2.2.6/developing/getting-started/python.md b/content/riak/kv/2.2.6/developing/getting-started/python.md new file mode 100644 index 0000000000..13e0ccecae --- /dev/null +++ b/content/riak/kv/2.2.6/developing/getting-started/python.md @@ -0,0 +1,99 @@ +--- +title: "Getting Started with Python" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Python" + identifier: "getting_started_python" + weight: 102 + parent: "developing_getting_started" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/taste-of-riak/python + - /riak-docs/riak/kv/2.2.6/dev/taste-of-riak/python +--- + + + +If you haven't set up a Riak Node and started it, please visit [Running A Cluster]({{}}riak/kv/2.2.6/using/running-a-cluster) first. + +To try this flavor of Riak, a working installation of Python is +required, with Python 2.7 preferred. One of the Python package managers, +e.g. `setuptools` or `pip`, is also required to install the client +package. + +You may install `setuptools` on OS X through MacPorts by running `sudo +port install py-distribute`. `setuptools` and `pip` are included in the +Homebrew formula for Python on OS X as well. Just run `brew install +python`. + +## Prerequisites + +First, you must install some packages needed by the Riak Python client: + +* `python-dev` --- Header files and a static library for Python +* `libffi-dev` --- Foreign function interface library +* `libssl-dev` --- libssl and libcrypto development libraries + +### Ubuntu (12.04 & 14.04) + +```bash +sudo apt-get install python-dev libffi-dev libssl-dev +``` + +## Client Setup + +The easiest way to install the client is with `easy_install` or `pip`. +Either of the commands below will ensure that the client and all its +dependencies are installed and on the load path. Depending on where your +Python libraries are held, these may require `sudo`. + +```bash +easy_install riak +pip install riak +``` + +To install from source, download the latest Python client from GitHub +([zip](https://github.com/basho/riak-python-client/archive/master.zip), +[GitHub repository](https://github.com/basho/riak-python-client)), and +extract it to your working directory. + +Now, let's build the client. + +```bash +python setup.py install +``` + +## Connecting to Riak + +Now, let's start the Python REPL and get set up. Enter the following +into the Python REPL: + +```python +import riak +``` +If you are using a single local Riak node, use the following to create a +new client instance: + +```python +myClient = riak.RiakClient(pb_port=8087, protocol='pbc') + +# Because the Python client uses the Protocol Buffers interface by +# default, the following will work the same: +myClient = riak.RiakClient(pb_port=8087) +``` + +If you set up a local Riak cluster using the [[five-minute install]] +method, use this code snippet instead: + +```python +myClient = riak.RiakClient(pb_port=10017, protocol='pbc') +``` + +We are now ready to start interacting with Riak. + +## Next Steps + +[CRUD Operations]({{}}riak/kv/2.2.6/developing/getting-started/python/crud-operations) diff --git a/content/riak/kv/2.2.6/developing/getting-started/python/crud-operations.md b/content/riak/kv/2.2.6/developing/getting-started/python/crud-operations.md new file mode 100644 index 0000000000..fbf6cf2e4a --- /dev/null +++ b/content/riak/kv/2.2.6/developing/getting-started/python/crud-operations.md @@ -0,0 +1,145 @@ +--- +title_supertext: "Getting Started:" +title: "CRUD Operations with Python" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "CRUD Operations" + identifier: "getting_started_python_crud" + weight: 100 + parent: "getting_started_python" +toc: true +--- + +## Creating Objects In Riak + +First, let’s create a few objects and a bucket to keep them in. + +```python +myBucket = myClient.bucket('test') + +val1 = 1 +key1 = myBucket.new('one', data=val1) +key1.store() +``` + +In this first example, we have stored the integer 1 with the lookup key +of `one`. Next let’s store a simple string value of `two` with a +matching key. + +```python +val2 = "two" +key2 = myBucket.new('two', data=val2) +key2.store() +``` + +That was easy. Finally, let’s store a bit of JSON. You will probably +recognize the pattern by now. + +```python +val3 = {"myValue": 3} +key3 = myBucket.new('three', data=val3) +key3.store() +``` + +## Reading Objects From Riak + +Now that we have a few objects stored, let’s retrieve them and make sure +they contain the values we expect. + +```python +fetched1 = myBucket.get('one') +fetched2 = myBucket.get('two') +fetched3 = myBucket.get('three') + +assert val1 == fetched1.data +assert val2 == fetched2.data +assert val3 == fetched3.data +``` + +That was easy. We simply request the objects by key. + +## Updating Objects In Riak + +While some data may be static, other forms of data may need to be +updated. This is also easy to accomplish. Let’s update the value of +myValue in the 3rd example to `42`. + +```python +fetched3.data["myValue"] = 42 +fetched3.store() +``` + +## Deleting Objects From Riak + +Nothing is complete without a delete. Fortunately, that's easy too. + +```python +fetched1.delete() +fetched2.delete() +fetched3.delete() +``` + +Now we can verify that the objects have been removed from Riak. + +```python +assert myBucket.get('one').exists == False +assert myBucket.get('two').exists == False +assert myBucket.get('three').exists == False +``` + + +## Working With Complex Objects + +Since the world is a little more complicated than simple integers and +bits of strings, let’s see how we can work with more complex objects. +Take for example, this object that encapsulates some knowledge about a +book. + +```python +book = { + 'isbn': "1111979723", + 'title': "Moby Dick", + 'author': "Herman Melville", + 'body': "Call me Ishmael. Some years ago...", + 'copies_owned': 3 +} +``` + +All right, so we have some information about our Moby Dick collection +that we want to save. Storing this to Riak should look familiar by now: + +```python +booksBucket = myClient.bucket('books') +newBook = booksBucket.new(book['isbn'], data=book) +newBook.store() +``` + +Some of you may be thinking, "But how does the Python Riak client +encode/decode my object?" If we fetch our book back and print the raw +encoded data, we shall know: + +```python +fetchedBook = booksBucket.get(book['isbn']) + +print(fetchedBook.encoded_data) +``` + +JSON! The Riak Python client library encodes things as JSON when it can. + +```json +{"body": "Call me Ishmael. Some years ago...", +"author": "Herman Melville", "isbn": "1111979723", +"copies_owned": 3, "title": "Moby Dick"} +``` + +If we wanted to get a deserialized object back we would just use the +regular `fetchedBook.data` method. + +Finally, let’s clean up our mess: + +```python +fetchedBook.delete() +``` diff --git a/content/riak/kv/2.2.6/developing/getting-started/python/object-modeling.md b/content/riak/kv/2.2.6/developing/getting-started/python/object-modeling.md new file mode 100644 index 0000000000..929ccd1301 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/getting-started/python/object-modeling.md @@ -0,0 +1,260 @@ +--- +title_supertext: "Getting Started:" +title: "Object Modeling with Python" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Object Modeling" + identifier: "getting_started_python_object" + weight: 102 + parent: "getting_started_python" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/taste-of-riak/object-modeling-python + - /riak-docs/riak/kv/2.2.6/dev/taste-of-riak/object-modeling-python +--- + +To get started, let's create the data structures that we'll be using. + +```python +from datetime import datetime +import string +import riak + + +marleen = {'user_name': 'marleenmgr', + 'full_name': 'Marleen Manager', + 'email': 'marleen.manager@basho.com'} + +joe = {'user_name': 'joeuser', + 'full_name': 'Joe User', + 'email': 'joe.user@basho.com'} + +msg = {'sender': marleen['user_name'], + 'recipient': joe['user_name'], + 'created': datetime.utcnow().isoformat(), + 'text': 'Welcome to the company!'} +``` + +As you can see, we first create a user, and then we can use that user to +create a message. To send this message we can append it to one or more +`Timeline`s. If it's a private message, we'll append it to the +Recipient's `Inbox` timeline and the User's own `Sent` timeline. If it's +a group message, we'll append it to the Group's timeline, as well as to +the User's `Sent` timeline. + +#### Buckets and Keys Revisited + +Now that we've worked out how we will differentiate data in the system, +let's figure out our bucket and key names. + +The bucket names are straightforward. We can use `Users`, `Msgs`, and +`Timelines`. The key names, however, are a little more tricky. In past +examples we've used sequential integers, but this presents a problem: we +would need a secondary service to hand out these IDs. This service could +easily be a future bottleneck in the system, so let's use a natural key. +Natural keys are a great fit for key/value systems because both humans +and computers can easily construct them when needed, and most of the +time they can be made unique enough for a KV store. + + +Bucket | Key Pattern | Example Key +:------|:------------|:----------- +`Users` | `` | `joeuser` +`Msgs` | `_` | `joeuser_2014-03-06T02:05:13.223556Z` +`Timelines` | `__` | `joeuser_Sent_2014-03-06`
`marketing_group_Inbox_2014-03-06` | + +For the `Users` bucket, we can be certain that we will want each +username to be unique, so let's use the `username` as the key. For the +`Msgs` bucket, let's use a combination of the username and the posting +datetime in an [ISO 8601 Long](http://en.wikipedia.org/wiki/ISO_8601) +format. This combination gives us the pattern `_`, +which produces keys like `joeuser_2014-03-05T23:20:28`. + +Now for `Timelines`, we need to differentiate between `Inbox` and `Sent` +timelines, so we can simply add that type into the key name. We will +also want to partition each collection object into some time period, +that way the object doesn't grow too large (see note below). + +For `Timelines`, let's use the pattern `__` for +users, and `__` for groups, which will look like +`joeuser_Sent_2014-03-06` or `marketing_group_Inbox_2014-03-06`, +respectively. + +{{% note title="Note" %}} +Riak performs best with objects under 1-2MB. Objects larger than that can hurt +performance, especially if many siblings are being created. We will cover +siblings, sibling resolution, and sibling explosions in the next chapter. +{{% /note %}} + +#### Keeping our story straight with repositories + +Now that we've figured out our object model, let's write some +repositories to help create and work with these objects in Riak: + +```python +class UserRepository: + BUCKET = 'Users' + + def __init__(self, client): + self.client = client + + def save(self, user): + riak_obj = self.client.bucket(self.BUCKET).get(user['user_name']) + riak_obj.data = user + return riak_obj.store() + + def get(self, user_name): + riak_obj = self.client.bucket(self.BUCKET).get(user_name) + return riak_obj.data + + +class MsgRepository: + BUCKET = 'Msgs' + + def __init__(self, client): + self.client = client + + def save(self, msg): + msgs = self.client.bucket(self.BUCKET) + key = self._generate_key(msg) + + riak_obj = msgs.get(key) + + if not riak_obj.exists: + riak_obj.data = msg + riak_obj.store(if_none_match=True) + + return riak_obj + + def get(self, key): + riak_obj = self.client.bucket(self.BUCKET).get(key) + return riak_obj.data + + def _generate_key(self, msg): + return msg['sender'] + '_' + msg['created'] + + +class TimelineRepository: + BUCKET = 'Timelines' + SENT = 'Sent' + INBOX = 'Inbox' + + def __init__(self, client): + self.client = client + self.msg_repo = MsgRepository(client) + + def post_message(self, msg): + # Save the canonical copy + saved_message = self.msg_repo.save(msg) + msg_key = saved_message.key + + # Post to sender's Sent timeline + self._add_to_timeline(msg, self.SENT, msg_key) + + # Post to recipient's Inbox timeline + self._add_to_timeline(msg, self.INBOX, msg_key) + + def get_timeline(self, owner, msg_type, date): + key = self._generate_key(owner, msg_type, date) + riak_obj = self.client.bucket(self.BUCKET).get(key) + return riak_obj.data + + def _add_to_timeline(self, msg, msg_type, msg_key): + timeline_key = self._generate_key_from_msg(msg, msg_type) + riak_obj = self.client.bucket(self.BUCKET).get(timeline_key) + + if riak_obj.exists: + riak_obj = self._add_to_existing_timeline(riak_obj, + msg_key) + else: + riak_obj = self._create_new_timeline(riak_obj, + msg, msg_type, + msg_key) + + return riak_obj.store() + + def _create_new_timeline(self, riak_obj, msg, msg_type, msg_key): + owner = self._get_owner(msg, msg_type) + new_timeline = {'owner': owner, + 'msg_type': msg_type, + 'msgs': [msg_key]} + + riak_obj.data = new_timeline + return riak_obj + + def _add_to_existing_timeline(self, riak_obj, msg_key): + riak_obj.data['msgs'].append(msg_key) + return riak_obj + + def _get_owner(self, msg, msg_type): + if msg_type == self.INBOX: + return msg['recipient'] + else: + return msg['sender'] + + def _generate_key_from_msg(self, msg, msg_type): + owner = self._get_owner(msg, msg_type) + return self._generate_key(owner, msg_type, msg['created']) + + def _generate_key(self, owner, msg_type, datetimestr): + dateString = string.split(datetimestr, 'T', 1)[0] + return owner + '_' + msg_type + '_' + dateString + +``` + +Finally, let's test them: + +```python +# Setup our repositories +client = riak.RiakClient(pb_port=10017, protocol='pbc') +userRepo = UserRepository(client) +msgsRepo = MsgRepository(client) +timelineRepo = TimelineRepository(client) + +# Save users +userRepo.save(marleen) +userRepo.save(joe) + +# Post msg to timelines +timelineRepo.post_message(msg) + +# Get Joe's inbox for today, get first message +joes_inbox_today = timelineRepo.get_timeline( + joe['user_name'], + TimelineRepository.INBOX, + datetime.utcnow().isoformat()) + +joes_first_message = msgsRepo.get(joes_inbox_today['msgs'][0]) + +print 'From: {0}\nMsg : {1}\n\n'.format( + joes_first_message['sender'], + joes_first_message['text']) + +``` + +As you can see, the repository pattern helps us with a few things: + +* It helps us to see if an object exists before creating a new one +* It keeps our buckets and key names consistent +* It provides us with a consistent interface to work with. + +While this set of repositories solves many of our problems, it is very +minimal and doesn't cover all the edge cases. For instance, what happens +if two different people try to create a user with the same username? + +We can also easily "compute" key names now, but how do we quickly look +up the last 10 messages a user sent? Many of these answers will be +application dependent. If your application shows the last 10 messages in +reverse order, for example, you may want to store that set of data in +another collection object to make lookup faster. There are drawbacks to +every solution, but we recommend seeking out the key/value-based +solution first, as it will likely be the quickest. + +So to recap, in this chapter we learned: + +* How to choose bucket names +* How to choose natural keys based on how we want to partition our data. + diff --git a/content/riak/kv/2.2.6/developing/getting-started/python/querying.md b/content/riak/kv/2.2.6/developing/getting-started/python/querying.md new file mode 100644 index 0000000000..7051b06333 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/getting-started/python/querying.md @@ -0,0 +1,236 @@ +--- +title_supertext: "Getting Started:" +title: "Querying with Python" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Querying" + identifier: "getting_started_python_query" + weight: 101 + parent: "getting_started_python" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/taste-of-riak/querying-python + - /riak-docs/riak/kv/2.2.6/dev/taste-of-riak/querying-python +--- + +#### A Quick Note on Querying and Schemas +_Schemas_? Yes we said that correctly, S-C-H-E-M-A-S. It's not a dirty word. +Even with a Key/Value store, you will still have a logical database schema of how all the data relates to one another. This can be as simple as using the same key across multiple buckets for different types of data, to having fields in your data that are related by name. These querying methods will introduce you to some ways of laying out your data in Riak, along with how to query it back. + +### Denormalization + +If you're coming from a relational database, the easiest way to get your application's feet wet with NoSQL is to denormalize your data into related chunks. For example with a customer database, you might have separate tables for Customers, Addresses, Preferences, etc. In Riak, you can denormalize all that associated data into a single object and store it into a `Customer` bucket. You can keep pulling in associated data until you hit one of the big denormalization walls: + +* Size Limits (objects greater than 1MB) +* Shared/Referential Data (data that the object doesn't "own") +* Differences in Access Patterns (objects that get read/written once vs. often) + +At one of these points we will have to split the model. + +### Same Keys - Different Buckets + +The simplest way to split up data would be to use the same identity key across different buckets. A good example of this would be a `Customer` object, an `Order` object, and an `OrderSummaries` object that keeps rolled up info about orders such as Total, etc. Let's put some data into Riak so we can play with it. + +```python +import riak + +# Creating Data + +customer = { + 'customer_id': 1, + 'name': "John Smith", + 'address': "123 Main Street", + 'city': "Columbus", + 'state': "Ohio", + 'zip': "43210", + 'phone': "+1-614-555-5555", + 'created_date': "2013-10-01 14:30:26" +} + +orders = [ + { + 'order_id': 1, + 'customer_id': 1, + 'salesperson_id': 9000, + 'items': [ + { + 'item_id': "TCV37GIT4NJ", + 'title': "USB 3.0 Coffee Warmer", + 'price': 15.99 + }, + { + 'item_id': "PEG10BBF2PP", + 'title': "eTablet Pro, 24GB, Grey", + 'price': 399.99 + } + ], + 'total': 415.98, + 'order_date': "2013-10-01 14:42:26" + }, + { + 'order_id': 2, + 'customer_id': 1, + 'salesperson_id': 9001, + 'items': [ + { + 'item_id': "OAX19XWN0QP", + 'title': "GoSlo Digital Camera", + 'price': 359.99 + } + ], + 'total': 359.99, + 'order_date': "2013-10-15 16:43:16" + }, + { + 'order_id': 3, + 'customer_id': 1, + 'salesperson_id': 9000, + 'items': [ + { + 'item_id': "WYK12EPU5EZ", + 'title': "Call of Battle: Goats - Gamesphere 4", + 'price': 69.99 + }, + { + 'item_id': "TJB84HAA8OA", + 'title': "Bricko Building Blocks", + 'price': 4.99 + } + ], + 'total': 74.98, + 'order_date': "2013-11-03 17:45:28" + }] + +order_summary = { + 'customer_id': 1, + 'summaries': [ + { + 'order_id': 1, + 'total': 415.98, + 'order_date': "2013-10-01 14:42:26" + }, + { + 'order_id': 2, + 'total': 359.99, + 'order_date': "2013-10-15 16:43:16" + }, + { + 'order_id': 3, + 'total': 74.98, + 'order_date': "2013-11-03 17:45:28" + } + ] +} + + +# Starting Client +client = riak.RiakClient(pb_port=10017, protocol='pbc') + +# Creating Buckets +customer_bucket = client.bucket('Customers') +order_bucket = client.bucket('Orders') +order_summary_bucket = client.bucket('OrderSummaries') + + +# Storing Data +cr = customer_bucket.new(str(customer['customer_id']), + data=customer) +cr.store() + +for order in orders: + order_riak = order_bucket.new(str(order['order_id']), + data=order) + order_riak.store() + +os = order_summary_bucket.new(str(order_summary['customer_id']), + data=order_summary) +os.store() +``` + + While individual `Customer` and `Order` objects don't change much (or shouldn't change), the `Order Summaries` object will likely change often. It will do double duty by acting as an index for all customer orders, and also holding some relevant data such as the order total, etc. If we showed this information in our application often, it's only one extra request to get all the info. + +```python +customer = customer_bucket.get('1').data +customer['order_summary'] = order_summary_bucket.get('1').data +customer +``` + +Which returns our amalgamated objects: + +```python +{ + u'city': u'Columbus', u'name': u'John Smith', u'zip': u'43210', + u'created_date': u'2013-10-01 14:30:26', + 'order_summary': { + u'customer_id': 1, u'summaries': [ + {u'order_id': 1, u'order_date': u'2013-10-01 14:42:26', u'total': 415.98}, + {u'order_id': 2, u'order_date': u'2013-10-15 16:43:16', u'total': 359.99}, + {u'order_id': 3, u'order_date': u'2013-11-03 17:45:28', u'total': 74.98} + ]}, + u'phone': u'+1-614-555-5555', u'state': u'Ohio', u'address': u'123 Main Street', + u'customer_id': 1 +} +``` + +While this pattern is very easy and extremely fast with respect to queries and complexity, it's up to the application to know about these intrinsic relationships. + + +### Secondary Indexes + +{{% note %}} +Secondary indexes in Riak KV require a sorted backend: [Memory]({{}}riak/kv/2.2.6/setup/planning/backend/memory) or [LevelDB]({{}}riak/kv/2.2.6/setup/planning/backend/leveldb). [Bitcask]({{}}riak/kv/2.2.6/setup/planning/backend/bitcask) does not support secondary indexes. + +See [Using Secondary Indexes (2i)]({{}}riak/kv/2.2.6/developing/usage/secondary-indexes) for more information on developing with secondary indexes. +{{% /note %}} + +If you're coming from a SQL world, Secondary Indexes (2i) are a lot like SQL indexes. They are a way to quickly lookup objects based on a secondary key, without scanning through the whole dataset. This makes it very easy to find groups of related data by values, or even ranges of values. To properly show this off, we will now add some more data to our application, and add some secondary index entries at the same time. + +```python +for i in range(1, 4): + order = order_bucket.get(str(i)) + # Initialize our secondary indices + order.add_index('salesperson_id_int', order.data['salesperson_id']) + order.add_index('order_date_bin', order.data['order_date']) + order.store() +``` + +As you may have noticed, ordinary Key/Value data is opaque to 2i, so we have to add entries to the indexes at the application level. +Now let's find all of Jane Appleseed's processed orders, we'll lookup the orders by searching the `saleperson_id_int` index for Jane's id of `9000`. + +```python +janes_orders = order_bucket.get_index("salesperson_id_int", 9000) +janes_orders.results +``` + +Which returns: + +```text +['1', '3'] +``` + +Jane processed orders 1 and 3. We used an "integer" index to reference Jane's id, next let's use a "binary" index. +Now, let's say that the VP of Sales wants to know how many orders came in during October 2013. In this case, we can exploit 2i's range queries. Let's search the `order_date_bin` index for entries between `2013-10-01` and `2013-10-31`. + +```python +october_orders = order_bucket.get_index("order_date_bin", + "2013-10-01", "2013-10-31") +october_orders.results +``` + +Which returns: + +```text +['1', '2'] +``` + +Boom, easy-peasy. We used 2i's range feature to search for a range of values, and demonstrated binary indexes. + +So to recap: + +* You can use Secondary Indexes to quickly lookup an object based on a secondary id other than the object's key. +* Indexes can have either Integer or Binary(String) keys +* You can search for specific values, or a range of values +* Riak will return a list of keys that match the index query diff --git a/content/riak/kv/2.2.6/developing/getting-started/ruby.md b/content/riak/kv/2.2.6/developing/getting-started/ruby.md new file mode 100644 index 0000000000..400c4482a4 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/getting-started/ruby.md @@ -0,0 +1,64 @@ +--- +title: "Getting Started with Ruby" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Ruby" + identifier: "getting_started_ruby" + weight: 101 + parent: "developing_getting_started" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/taste-of-riak/ruby + - /riak-docs/riak/kv/2.2.6/dev/taste-of-riak/ruby +--- + + + +If you haven't set up a Riak Node and started it, please visit [Running A Cluster]({{}}riak/kv/2.2.6/using/running-a-cluster) first. To try this flavor +of Riak, a working installation of Ruby is required. + +## Client Setup + +First, install the Riak Ruby client via RubyGems. + +```bash +gem install riak-client +``` + +Start IRB, the Ruby REPL, and let’s get set up. Enter the following into +IRB: + +```ruby +require 'riak' +``` + +If you are using a single local Riak node, use the following to create a +new client instance, assuming that the node is running on `localhost` +port 8087: + +```ruby +client = Riak::Client.new(:protocol => "pbc", :pb_port => 8087) + +# Since the Ruby Riak client uses the Protocol Buffers API by default, +# you can also just enter this: +client = Riak::Client.new(:pb_port => 8087) +``` + +If you set up a local Riak cluster using the [[five-minute install]] +method, use this code snippet instead: + +```ruby +client = Riak::Client.new(:protocol => "pbc", :pb_port => 10017) + +# For the reasons explain in the snippet above, this will also work: +client = Riak::Client.new(:pb_port => 10017) +``` + +We are now ready to start interacting with Riak. + +## Next Steps + +[CRUD Operations]({{}}riak/kv/2.2.6/developing/getting-started/ruby/crud-operations) diff --git a/content/riak/kv/2.2.6/developing/getting-started/ruby/crud-operations.md b/content/riak/kv/2.2.6/developing/getting-started/ruby/crud-operations.md new file mode 100644 index 0000000000..ba6e8792ed --- /dev/null +++ b/content/riak/kv/2.2.6/developing/getting-started/ruby/crud-operations.md @@ -0,0 +1,146 @@ +--- +title_supertext: "Getting Started:" +title: "CRUD Operations with Ruby" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "CRUD Operations" + identifier: "getting_started_ruby_crud" + weight: 100 + parent: "getting_started_ruby" +toc: true +--- + +## Creating Objects In Riak + +First, let’s create a few objects and a bucket to keep them in. + +```ruby +my_bucket = client.bucket("test") + +val1 = 1 +obj1 = my_bucket.new('one') +obj1.data = val1 +obj1.store() +``` + +In this first example we have stored the integer 1 with the lookup key +of `one`. Next, let’s store a simple string value of `two` with a +matching key. + +```ruby +val2 = "two" +obj2 = my_bucket.new('two') +obj2.data = val2 +obj2.store() +``` + +That was easy. Finally, let’s store a bit of JSON. You will probably +recognize the pattern by now. + +```ruby +val3 = { myValue: 3 } +obj3 = my_bucket.new('three') +obj3.data = val3 +obj3.store() +``` + +## Reading Objects From Riak + +Now that we have a few objects stored, let’s retrieve them and make sure +they contain the values we expect. + +```ruby +fetched1 = my_bucket.get('one') +fetched2 = my_bucket.get('two') +fetched3 = my_bucket.get('three') + +fetched1.data == val1 +fetched2.data == val2 +fetched3.data.to_json == val3.to_json +``` + +That was easy. we simply request the objects by key. in the last +example, we converted to JSON so we can compare a string key to a symbol +key. + +## Updating Objects In Riak + +While some data may be static, other forms of data may need to be +updated. This is also easy to accomplish. Let’s update the value of +myValue in the 3rd example to 42. + +```ruby +fetched3.data["myValue"] = 42 +fetched3.store() +``` + +## Deleting Objects From Riak + +As a last step, we’ll demonstrate how to delete data. You’ll see that +the delete message can be called either against the bucket or the +object. + +```ruby +my_bucket.delete('one') +obj2.delete() +obj3.delete() +``` + +## Working With Complex Objects + +Since the world is a little more complicated than simple integers and +bits of strings, let’s see how we can work with more complex objects. +Take, for example, this Ruby hash that encapsulates some knowledge about +a book. + +```ruby +book = { + :isbn => '1111979723', + :title => 'Moby Dick', + :author => 'Herman Melville', + :body => 'Call me Ishmael. Some years ago...', + :copies_owned => 3 +} +``` + +All right, so we have some information about our Moby Dick collection +that we want to save. Storing this to Riak should look familiar by now. + +```ruby +books_bucket = client.bucket('books') +new_book = books_bucket.new(book[:isbn]) +new_book.data = book +new_book.store() +``` + +Some of you may be thinking, "But how does the Ruby Riak client +encode/decode my object?" If we fetch our book back and print the raw +data, we shall know: + +```ruby +fetched_book = books_bucket.get(book[:isbn]) +puts fetched_book.raw_data +``` + +Raw Data: + +```json +{"isbn":"1111979723","title":"Moby Dick","author":"Herman Melville", +"body":"Call me Ishmael. Some years ago...","copies_owned":3} +``` + +JSON! The Ruby Riak client will serialize objects to JSON when it comes +across structured data like hashes. For more advanced control over +serialization you can use a library called +[Ripple](https://github.com/basho/ripple), which is a rich Ruby modeling +layer over the basic riak client. Ripple falls outside the scope of +this document but we shall visit it later. + +Now, let’s clean up our mess: + +```ruby +new_book.delete() +``` diff --git a/content/riak/kv/2.2.6/developing/getting-started/ruby/object-modeling.md b/content/riak/kv/2.2.6/developing/getting-started/ruby/object-modeling.md new file mode 100644 index 0000000000..15135c3b16 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/getting-started/ruby/object-modeling.md @@ -0,0 +1,291 @@ +--- +title_supertext: "Getting Started:" +title: "Object Modeling with Ruby" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Object Modeling" + identifier: "getting_started_ruby_object" + weight: 102 + parent: "getting_started_ruby" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/taste-of-riak/object-modeling-ruby + - /riak-docs/riak/kv/2.2.6/dev/taste-of-riak/object-modeling-ruby +--- + +To get started, let's create the models that we'll be using. Since the +[Ruby Riak Client](https://github.com/basho/riak-ruby-client) uses +hashes when converting to and from JSON, we'll use the library +[Hashie](http://rdoc.info/github/intridea/hashie) to help automatically +coerce class properties to and from hashes. You can install this library +with `gem install hashie`. + +```ruby +# Encoding: utf-8 + +require 'riak' +require 'hashie' +require 'time' + +class User < Hashie::Dash + property :user_name + property :full_name + property :email +end + +class Msg < Hashie::Dash + property :from + property :to + property :created + property :text +end + +class Timeline < Hashie::Dash + property :owner + property :type + property :msgs +end +``` + +To use these classes to store data, we will first have to create a user. +Then, when a user creates a message, we will append that message to one +or more timelines. If it's a private message, we'll append it to the +Recipient's `Inbox` timeline and the User's own `Sent` timeline. If it's +a group message, we'll append it to the Group's timeline, as well as to +the User's `Sent` timeline. + +#### Buckets and Keys Revisited + +Now that we've worked out how we will differentiate data in the system, +let's figure out our bucket and key names. + +The bucket names are straightforward. We can use `Users`, `Msgs`, and +`Timelines`. The key names, however, are a little more tricky. In past +examples we've used sequential integers, but this presents a problem: we +would need a secondary service to hand out these IDs. This service could +easily be a future bottleneck in the system, so let's use a natural key. +Natural keys are a great fit for key/value systems because both humans +and computers can easily construct them when needed, and most of the +time they can be made unique enough for a KV store. + +Bucket | Key Pattern | Example Key +:------|:------------|:----------- +`Users` | `` | `joeuser` +`Msgs` | `_` | `joeuser_2014-03-06T02:05:13.223556Z` +`Timelines` | `__` | `joeuser_Sent_2014-03-06Z`
`marketing_group_Inbox_2014-03-06Z` | + +For the `Users` bucket, we can be certain that we will want each +username to be unique, so let's use the `username` as the key. For the +`Msgs` bucket, let's use a combination of the username and the posting +datetime in an [ISO 8601 Long](http://en.wikipedia.org/wiki/ISO_8601) +format. This combination gives us the pattern `_`, +which produces keys like `joeuser_2014-03-05T23:20:28`. + +Now for `Timelines`, we need to differentiate between `Inbox` and `Sent` +timelines, so we can simply add that type into the key name. We will +also want to partition each collection object into some time period, +that way the object doesn't grow too large (see note below). + +For `Timelines`, let's use the pattern `__` for +users, and `_Inbox_` for groups, which will look like +`joeuser_Sent_2014-03-06Z` or `marketing_group_Inbox_2014-03-05Z`, +respectively. + +{{% note title="Note" %}} +Riak performs best with objects under 1-2MB. Objects larger than that can hurt +performance, especially many siblings are being created. We will cover +siblings, sibling resolution, and sibling explosions in the next chapter. +{{% /note %}} + +#### Keeping our story straight with repositories + +Now that we've figured out our object models, let's write some +repositories to help create and work with these objects in Riak: + +```ruby +class UserRepository + BUCKET = 'Users' + + def initialize(client) + @client = client + end + + def save(user) + users = @client.bucket(BUCKET) + key = user.user_name + + riak_obj = users.get_or_new(key) + riak_obj.data = user + riak_obj.content_type = 'application/json' + riak_obj.store + end + + def get(user_name) + riak_obj = @client.bucket(BUCKET)[user_name] + User.new(riak_obj.data) + end +end + +class MsgRepository + BUCKET = 'Msgs' + + def initialize(client) + @client = client + end + + def save(msg) + msgs = @client.bucket(BUCKET) + key = generate_key(msg) + + return msgs.get(key) if msgs.exists?(key) + riak_obj = msgs.new(key) + riak_obj.data = msg + riak_obj.content_type = 'application/json' + riak_obj.prevent_stale_writes = true + riak_obj.store(returnbody: true) + end + + def get(key) + riak_obj = @client.bucket(BUCKET).get(key) + Msg.new(riak_obj.data) + end + + def generate_key(msg) + msg.from + '_' + msg.created.utc.iso8601(6) + end +end + +class TimelineRepository + BUCKET = 'Timelines' + SENT = 'Sent' + INBOX = 'Inbox' + + def initialize(client) + @client = client + @msg_repo = MsgRepository.new(client) + end + + def post_message(msg) + # Save the canonical copy + saved_message = @msg_repo.save(msg) + # Post to sender's Sent timeline + add_to_timeline(msg, SENT, saved_message.key) + # Post to recipient's Inbox timeline + add_to_timeline(msg, INBOX, saved_message.key) + end + + def get_timeline(owner, type, date) + riak_obj = @client.bucket(BUCKET).get(generate_key(owner, type, date)) + Timeline.new(riak_obj.data) + end + + private + + def add_to_timeline(msg, type, msg_key) + timeline_key = generate_key_from_msg(msg, type) + riak_obj = nil + + if @client.bucket(BUCKET).exists?(timeline_key) + riak_obj = add_to_existing_timeline(timeline_key, msg_key) + else + riak_obj = create_new_timeline(timeline_key, msg, type, msg_key) + end + + riak_obj.store + end + + def create_new_timeline(key, msg, type, msg_key) + owner = get_owner(msg, type) + riak_obj = @client.bucket(BUCKET).new(key) + riak_obj.data = Timeline.new(owner: owner, + type: type, + msgs: [msg_key]) + riak_obj.content_type = 'application/json' + riak_obj + end + + def add_to_existing_timeline(key, msg_key) + riak_obj = @client.bucket(BUCKET).get(key) + timeline = Timeline.new(riak_obj.data) + timeline.msgs << msg_key + riak_obj.data = timeline + riak_obj + end + + def get_owner(msg, type) + type == INBOX ? msg.to : msg.from + end + + def generate_key_from_msg(msg, type) + owner = get_owner(msg, type) + generate_key(owner, type, msg.created) + end + + def generate_key(owner, type, date) + owner + '_' + type + '_' + date.utc.strftime('%F') + end +end +``` + +Finally, let's test them: + +```ruby +# Setup our repositories +client = Riak::Client.new(protocol: 'pbc', pb_port: 10017) +user_repo = UserRepository.new(client) +msgs_repo = MsgRepository.new(client) +timeline_repo = TimelineRepository.new(client) + +# Create and save users +marleen = User.new(user_name: 'marleenmgr', + full_name: 'Marleen Manager', + email: 'marleen.manager@basho.com') + +joe = User.new(user_name: 'joeuser', + full_name: 'Joe User', + email: 'joe.user@basho.com') + +user_repo.save(marleen) +user_repo.save(joe) + +# Create new Msg, post to timelines +msg = Msg.new(from: marleen.user_name, + to: joe.user_name, + created: Time.now, + text: 'Welcome to the company!') + +timeline_repo.post_message(msg) + +# Get Joe's inbox for today, get first message +joes_inbox_today = timeline_repo.get_timeline(joe.user_name, 'Inbox', Time.now) +joes_first_message = msgs_repo.get(joes_inbox_today.msgs.first) + +puts "From: #{joes_first_message.from}\nMsg : #{joes_first_message.text}" +``` + +As you can see, the repository pattern helps us with a few things: + +* It helps us to see if an object exists before creating a new one +* It keeps our buckets and key names consistent +* It provides us with a consistent interface to work with. + +While this set of repositories solves many of our problems, it is very +minimal and doesn't cover all the edge cases. For instance, what happens +if two different people try to create a user with the same username? + +We can also easily "compute" key names now, but how do we quickly look +up the last 10 messages a user sent? Many of these answers will be +application dependent. If your application shows the last 10 messages in +reverse order, for example, you may want to store that set of data in +another collection object to make lookup faster. There are drawbacks to +every solution, but we recommend seeking out the key/value-based +solution first, as it will likely be the quickest. + +So to recap, in this chapter we learned: + +* How to choose bucket names +* How to choose natural keys based on how we want to partition our data. + diff --git a/content/riak/kv/2.2.6/developing/getting-started/ruby/querying.md b/content/riak/kv/2.2.6/developing/getting-started/ruby/querying.md new file mode 100644 index 0000000000..3c97ebb95c --- /dev/null +++ b/content/riak/kv/2.2.6/developing/getting-started/ruby/querying.md @@ -0,0 +1,252 @@ +--- +title_supertext: "Getting Started:" +title: "Querying with Ruby" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Querying" + identifier: "getting_started_ruby_query" + weight: 101 + parent: "getting_started_ruby" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/taste-of-riak/querying-ruby + - /riak-docs/riak/kv/2.2.6/dev/taste-of-riak/querying-ruby +--- + +#### A Quick Note on Querying and Schemas +_Schemas_? Yes we said that correctly, S-C-H-E-M-A-S. It's not a dirty word. +Even with a Key/Value store, you will still have a logical database schema of how all the data relates to one another. This can be as simple as using the same key across multiple buckets for different types of data, to having fields in your data that are related by name. These querying methods will introduce you to some ways of laying out your data in Riak, along with how to query it back. + +### Denormalization + +If you're coming from a relational database, the easiest way to get your application's feet wet with NoSQL is to denormalize your data into related chunks. For example with a customer database, you might have separate tables for Customers, Addresses, Preferences, etc. In Riak, you can denormalize all that associated data into a single object and store it into a `Customer` bucket. You can keep pulling in associated data until you hit one of the big denormalization walls: + +* Size Limits (objects greater than 1MB) +* Shared/Referential Data (data that the object doesn't "own") +* Differences in Access Patterns (objects that get read/written once vs. often) + +At one of these points we will have to split the model. + +### Same Keys - Different Buckets + +The simplest way to split up data would be to use the same identity key across different buckets. A good example of this would be a `Customer` object, an `Order` object, and an `OrderSummaries` object that keeps rolled up info about orders such as Total, etc. Let's put some data into Riak so we can play with it. + +```ruby +# Encoding: utf-8 + +require 'riak' +require 'pp' + +# Starting Client +client = Riak::Client.new protocol: 'pbc', pb_port: 10017 + +# Creating Data +customer = { + customer_id: 1, + name: 'John Smith', + address: '123 Main Street', + city: 'Columbus', + state: 'Ohio', + zip: '43210', + phone: '+1-614-555-5555', + created_date: Time.parse('2013-10-1 14:30:26') +} + +orders = [ + { + order_id: 1, + customer_id: 1, + salesperson_id: 9000, + items: [ + { + item_id: 'TCV37GIT4NJ', + title: 'USB 3.0 Coffee Warmer', + price: 15.99 + }, + { + item_id: 'PEG10BBF2PP', + title: 'eTablet Pro, 24GB, Grey', + price: 399.99 + } + ], + total: 415.98, + order_date: Time.parse('2013-10-1 14:42:26') + }, + { + order_id: 2, + customer_id: 1, + salesperson_id: 9001, + items: [ + { + item_id: 'OAX19XWN0QP', + title: 'GoSlo Digital Camera', + price: 359.99 + } + ], + total: 359.99, + order_date: Time.parse('2013-10-15 16:43:16') + }, + { + order_id: 3, + customer_id: 1, + salesperson_id: 9000, + items: [ + { + item_id: 'WYK12EPU5EZ', + title: 'Call of Battle: Goats - Gamesphere 4', + price: 69.99 + }, + { + item_id: 'TJB84HAA8OA', + title: 'Bricko Building Blocks', + price: 4.99 + } + ], + total: 74.98, + order_date: Time.parse('2013-11-3 17:45:28') + }] + +order_summary = { + customer_id: 1, + summaries: [ + { + order_id: 1, + total: 415.98, + order_date: Time.parse('2013-10-1 14:42:26') + }, + { + order_id: 2, + total: 359.99, + order_date: Time.parse('2013-10-15 16:43:16') + }, + { + order_id: 3, + total: 74.98, + order_date: Time.parse('2013-11-3 17:45:28') + } + ] +} + +# Creating Buckets and Storing Data +customer_bucket = client.bucket('Customers') +cr = customer_bucket.new(customer[:customer_id].to_s) +cr.data = customer +cr.store + +order_bucket = client.bucket('Orders') +orders.each do |order| + order_riak = order_bucket.new(order[:order_id].to_s) + order_riak.data = order + order_riak.store +end + +order_summary_bucket = client.bucket('OrderSummaries') +os = order_summary_bucket.new(order_summary[:customer_id].to_s) +os.data = order_summary +os.store +``` + + While individual `Customer` and `Order` objects don't change much (or shouldn't change), the `Order Summaries` object will likely change often. It will do double duty by acting as an index for all a customer's orders, and also holding some relevant data such as the order total, etc. If we showed this information in our application often, it's only one extra request to get all the info. + +```ruby +shared_key = '1' +customer = customer_bucket.get(shared_key).data +customer[:order_summary] = order_summary_bucket.get(shared_key).data +puts "Combined Customer and Order Summary: " +pp customer +``` + +Which returns our amalgamated objects: + +```ruby +# Combined Customer and Order Summary: +{"customer_id"=>1, + "name"=>"John Smith", + "address"=>"123 Main Street", + "city"=>"Columbus", + "state"=>"Ohio", + "zip"=>"43210", + "phone"=>"+1-614-555-5555", + "created_date"=>"2013-10-01 14:30:26 -0400", + :order_summary=> + {"customer_id"=>1, + "summaries"=> + [{"order_id"=>1, + "total"=>415.98, + "order_date"=>"2013-10-01 14:42:26 -0400"}, + {"order_id"=>2, + "total"=>359.99, + "order_date"=>"2013-10-15 16:43:16 -0400"}, + {"order_id"=>3, + "total"=>74.98, + "order_date"=>"2013-11-03 17:45:28 -0500"}]}} +``` + +While this pattern is very easy and extremely fast with respect to queries and complexity, it's up to the application to know about these intrinsic relationships. + + +### Secondary Indexes + +{{% note %}} +Secondary indexes in Riak KV require a sorted backend: [Memory]({{}}riak/kv/2.2.6/setup/planning/backend/memory) or [LevelDB]({{}}riak/kv/2.2.6/setup/planning/backend/leveldb). [Bitcask]({{}}riak/kv/2.2.6/setup/planning/backend/bitcask) does not support secondary indexes. + +See [Using Secondary Indexes (2i)]({{}}riak/kv/2.2.6/developing/usage/secondary-indexes) for more information on developing with secondary indexes. +{{% /note %}} + +If you're coming from a SQL world, Secondary Indexes (2i) are a lot like SQL indexes. They are a way to quickly lookup objects based on a secondary key, without scanning through the whole dataset. This makes it very easy to find groups of related data by values, or even ranges of values. To properly show this off, we will now add some more data to our application, and add some secondary index entries at the same time. + +```ruby +(1..3).each do |i| + order = order_bucket.get(i.to_s) + # Initialize our secondary indices + order.indexes['salesperson_id_int'] = [] + order.indexes['order_date_bin'] = [] + + order.indexes['salesperson_id_int'] << order.data['salesperson_id'] + order.indexes['order_date_bin'] << Time.parse(order.data['order_date']) + .strftime('%Y%m%d') + order.store +end +``` + +As you may have noticed, ordinary Key/Value data is opaque to 2i, so we have to add entries to the indexes at the application level. +Now let's find all of Jane Appleseed's processed orders, we'll lookup the orders by searching the `saleperson_id_int` index for Jane's id of `9000`. + +```ruby +puts "#Jane's Orders: " +pp order_bucket.get_index('salesperson_id_int', 9000) +``` + +Which returns: + +```ruby +# Jane's Orders: +["1", "3"] +``` + +Jane processed orders 1 and 3. We used an "integer" index to reference Jane's id, next let's use a "binary" index. +Now, let's say that the VP of Sales wants to know how many orders came in during October 2013. In this case, we can exploit 2i's range queries. Let's search the `order_date_bin` index for entries between `20131001` and `20131031`. + +```ruby +puts "#October's Orders: " +pp order_bucket.get_index('order_date_bin', '20131001'..'20131031') +``` + +Which returns: + +```ruby +# October's Orders: +["1", "2"] +``` + +Boom, easy-peasy. We used 2i's range feature to search for a range of values, and demonstrated binary indexes. + +So to recap: + +* You can use Secondary Indexes to quickly lookup an object based on a secondary id other than the object's key. +* Indexes can have either Integer or Binary(String) keys +* You can search for specific values, or a range of values +* Riak will return a list of keys that match the index query diff --git a/content/riak/kv/2.2.6/developing/key-value-modeling.md b/content/riak/kv/2.2.6/developing/key-value-modeling.md new file mode 100644 index 0000000000..1ea0c8862e --- /dev/null +++ b/content/riak/kv/2.2.6/developing/key-value-modeling.md @@ -0,0 +1,531 @@ +--- +title: "Riak KV Key/Value Modeling" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Key/Value Modeling" + identifier: "developing_kv_model" + weight: 104 + parent: "developing" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/data-modeling/key-value/ + - /riak-docs/riak/kv/2.2.6/dev/data-modeling/key-value/ +--- + +While Riak enables you to take advantage of a wide variety of features +that can be useful in application development, such as [Search]({{}}riak/kv/2.2.6/developing/usage/search), [secondary indexes (2i)]({{}}riak/kv/2.2.6/developing/usage/secondary-indexes/), and [Riak Data Types]({{}}riak/kv/2.2.6/developing/data-types/), Riak almost always performs best when you +build your application around basic CRUD operations (create, read, +update, and delete) on objects, i.e. when you use Riak as a "pure" +key/value store. + +In this tutorial, we'll suggest some strategies for naming and modeling +for key/value object interactions with Riak. If you'd like to use some +of Riak's other features, we recommend checking out the documentation +for each of them or consulting our guide to [building applications with Riak]({{}}riak/kv/2.2.6/developing/app-guide/) for a better sense of which features you might need. + +## Advantages of Key/Value Operations + +Riak's key/value architecture enables it to be more performant than +relational databases in many scenarios because Riak doesn't need to +perform lock, join, union, or other operations when working with +objects. Instead, it interacts with objects on a one-by-one basis, using +**primary key lookups**. + +Primary key lookups store and fetch objects in Riak on the basis of +three basic locators: + +* The object's [key]({{}}riak/kv/2.2.6/learn/concepts/keys-and-objects#keys), which can be anything you + want as long as it is [Unicode compliant](http://www.unicode.org/) +* The [bucket]({{}}riak/kv/2.2.6/learn/concepts/buckets) which houses the object and its key (bucket + names are also Unicode compliant) +* The [bucket type]({{}}riak/kv/2.2.6/developing/usage/bucket-types) that determines the bucket's + [replication]({{}}riak/kv/2.2.6/developing/app-guide/replication-properties) and other properties + +It may be useful to think of this system as analogous to a nested +key/value [hash](http://en.wikipedia.org/wiki/Hash_function) as you +would find in most programming languages. Below is an example from +[Ruby](http://www.ruby-doc.org/core-2.1.2/Hash.html). The hash +`simpsons` contains keys for all of the available seasons, while each +key houses a hash for each episode of that season: + +```ruby +simpsons = { + 'season 1': { + { 'episode 1': 'Simpsons Roasting on an Open Fire' }, + { 'episode 2': 'Bart the Genius' }, + # ... + }, + 'season 2': { + { 'episode 1': 'Bart Gets an "F"' }, + # ... + }, + # ... +} +``` + +If we want to find out the title of an episode, we can retrieve it based +on hash keys: + +```ruby +simpsons['season 4']['episode 12'] + +# => "Marge vs. the Monorail" +``` + +Storing data in Riak is a lot like this. Let's say that we want to store +JSON objects with a variety of information about every episode of the +Simpsons. We could store each season in its own bucket and each episode +in its own key within that bucket. Here's what the URL structure would +look like (for the [HTTP API]({{}}riak/kv/2.2.6/developing/api/http)): + +``` +GET/PUT/DELETE /bucket//keys/ +``` + +The most important benefit of sorting Riak objects this way is that +these types of lookup operations are extremely fast. Riak doesn't need +to search through columns or tables to find an object. If it knows the +bucket/key "address" of the object, so to speak, it can locate that +object just about as quickly with billions of objects in a cluster as +when the cluster holds only a handful of objects. + +## Overcoming the Limitations of Key/Value Operations + +Using any key/value store can be tricky at first, especially if you're +used to relational databases. The central difficulty is that your +application cannot run arbitrary selection queries like `SELECT * FROM +table`, and so it needs to know where to look for objects in advance. + +One of the best ways to enable applications to discover objects in Riak +more easily is to provide **structured bucket and key names** for +objects. This approach often involves wrapping information about the +object _in the object's location data itself_. + +Here are some example sources for bucket or key names: + +* Timestamps, e.g. `2013-11-05T08:15:30-05:00` +* [UUID](http://en.wikipedia.org/wiki/Universally_unique_identifier)s, + e.g. `9b1899b5-eb8c-47e4-83c9-2c62f0300596` +* Geographical coordinates, e.g. `40.172N-21.273E` + +We could use these markers by themselves or in combination with other +markers. For example, sensor data keys could be prefaced by `sensor_` or +`temp_sensor1_` followed by a timestamp (e.g. +`sensor1_2013-11-05T08:15:30-05:00`), or user data keys could be +prefaced with `user_` followed by a UUID (e.g. +`user_9b1899b5-eb8c-47e4-83c9-2c62f0300596`). + +Any of the above suggestions could apply to bucket names as well as key +names. If you were building Twitter using Riak, for example, you could +store tweets from each user in a different bucket and then construct key +names using a combination of the prefix `tweet_` and then a timestamp. +In that case, all the tweets from the user BashoWhisperer123 could be +housed in a bucket named `BashoWhisperer123`, and keys for tweets would +look like `tweet_`. + +The possibilities are essentially endless and, as always, defined by the +use case at hand. + +## Object Discovery with Riak Sets + +Let's say that we've created a solid bucket/key naming scheme for a user +information store that enables your application to easily fetch user +records, which are all stored in the bucket `users` with each user's +username acting as the key. The problem at this point is this: how can +Riak know which user records actually exist? + +One way to determine this is to [list all keys]({{}}riak/kv/2.2.6/developing/api/protocol-buffers/list-keys) in the +bucket `users`. This approach, however, is _not_ recommended, because +listing all keys in a bucket is a very expensive operation that should +not be used in production. And so another strategy must be employed. + +A better possibility is to use [Riak sets]({{}}riak/kv/2.2.6/developing/data-types/#sets) to +store lists of keys in a bucket. Riak sets are a [Riak Data Type]({{}}riak/kv/2.2.6/developing/data-types) that enable you to store lists of binaries or strings in Riak. +Unlike normal Riak objects, you can interact with Riak sets much like +you interact with sets in most programming languages, i.e. you can add +and remove elements at will. + +Going back to our user data example, instead of simply storing user +records in our `users` bucket, we could set up our application to store +each key in a set when a new record is created. We'll store this set in +the bucket `user_info_sets` (we'll keep it simple) and in the key +`usernames`. The following will also assume that we've [set up a bucket type]({{}}riak/kv/2.2.6/developing/data-types/#setting-up-buckets-to-use-riak-data-types) called +`sets`. + +We can interact with that set on the basis of its location: + +```java +Location userIdSet = new Location(new Namespace("sets", "user_info_sets"), "usernames"); + +// With this Location, we can construct fetch operations like this: +FetchSet fetchUserIdSet = new FetchSet.Builder(userIdSet).build(); +``` + +```ruby +require 'riak' + +set_bucket = client.bucket('user_info_sets') + +# We'll make this set global because we'll use it +# inside of a function later on + +$user_id_set = Riak::Crdt::Set.new(set_bucket, 'usernames', 'sets') +``` + +```php +$command = (new \Basho\Riak\Command\Builder\FetchSet($riak)) + ->buildLocation('usernames', 'user_info_sets', 'sets') + ->build(); +``` + +```python +from riak.datatypes import Set + +bucket = client.bucket_type('sets').bucket('user_info_sets') +user_id_set = Set(bucket, 'usernames') +``` + +> **Getting started with Riak clients** +> +> If you are connecting to Riak using one of Basho's official [client libraries]({{}}riak/kv/2.2.6/developing/client-libraries), you can find more information about getting started with your client in [Developing with Riak KV: Getting Started]({{}}riak/kv/2.2.6/developing/getting-started). + +Then, we can create a function that stores a user record's key in that +set every time a record is created: + +```java +// A User class for constructing user records +class User { + public String username; + public String info; + + public User(String username, String info) { + this.username = username; + this.info = info; + } +} + +// A function for storing a user record that has been created +public void storeUserRecord(User user) throws Exception { + // User records themselves will be stored in the bucket "users" + Location userObjectLocation = + new Location(new Namespace("users"), user.username); + RiakObject userObject = new RiakObject() + // We'll keep it simple and store User object data as plain text + .setContentType("text/plain") + .setValue(user.info); + StoreValue store = new StoreValue.Builder(userObjectLocation, userObject) + .build(); + client.execute(store); + + Location userIdSet = + new Location(new Namespace("sets", "user_info_sets"), "usernames"); + SetUpdate su = new SetUpdate() + .add(BinaryValue.create(user.username)); + UpdateSet update = new UpdateSet.Builder(su, update) + .build(); + client.execute(update); +} +``` + +```ruby +class User + attr_accessor :username, :info +end + +def store_record(user) + # First we create an empty object and specify its bucket and key + obj = Riak::RObject.new(client.bucket('users'), user.username) + + # We'll keep it simple by storing plain text for each user's info + obj.content_type = 'text/plain' + obj.raw_data = user.info + obj.store + + # Finally, we'll add the user's username to the set + user_id_set.add(user.username) +end +``` + +```php +class User +{ + public $user_name; + public $info; + + public function __construct($user_name, $info) + { + $this->user_name = $user_name; + $this->info = $info; + } +} + +function store_user(User $user) +{ + (new \Basho\Riak\Command\Builder\StoreObject) + ->buildLocation($user->user_name, 'users') + ->buildJsonObject($user) + ->build() + ->execute(); + + (new \Basho\Riak\Command\Builder\UpdateSet) + ->buildLocation('usernames', 'user_info_sets', 'sets') + ->add($user->user_name) + ->build() + ->execute(); +} +``` + +```python +class User: + def __init__(self, username, info): + this.username = username + this.info = info + +# Using the "user_id_set" object from above +def store_record(user): + # First we create an empty object and specify its bucket and key + obj = RiakObject(client, 'users', user.username) + + # We'll keep it simple by storing plain text for each user's info + obj.content_type = 'text/plain' + obj.data = user.info + obj.store() + + # Finally, we'll add the user's username to the set + user_id_set.add(username) + user_id_set.store() +``` + +Now, let's say that we want to be able to pull up all user records in +the bucket at once. We could do so by iterating through the usernames +stored in our set and then fetching the object corresponding to each +username: + +```java +public Set fetchAllUserRecords() { + // Empty builder sets for usernames and User objects + Set userIdSet = new HashSet(); + Set userSet = new HashSet(); + + // Turn the Riak username set into a set of Strings + Location userIdSet = + new Location(new Namespace("sets", "sets"), "usernames"); + FetchSet fetchUserIdSet = new FetchSet.Builder(userIdSet).build(); + RiakSet set = client.execute(fetchUserIdSet).getDatatype(); + set.viewAsSet().forEach((BinaryValue username) -> { + userIdSet.add(username.toString()); + }); + + // Fetch User objects for each of the usernames stored in the set + userIdSet.forEach((String username) -> { + Location userLocation = new Location(new Namespace("users"), username); + FetchValue fetch = new FetchValue.Builder(userLocation).build(); + User user = client.execute(fetch).getValue(User.class); + userSet.add(user); + }); + return userSet; +} +``` + +```ruby +# Using the "user_id_set" set from above + +def fetch_all_user_records + users_bucket = $client.bucket('users') + user_records = Array.new + $user_id_set.members.each do |user_id| + user_record = users_bucket.get(user_id).data + user_records.push(user_record) + end + user_records +end +``` + +```php +function fetch_users() +{ + $users = []; + + $response = (new \Basho\Riak\Command\Builder\UpdateSet) + ->buildLocation('usernames', 'user_info_sets', 'sets') + ->build() + ->execute(); + + $user_names = $response->getSet()->getData(); + foreach($user_names as $user_name) { + $response = (new \Basho\Riak\Command\Builder\FetchObject) + ->buildLocation($user_name, 'users') + ->build() + ->execute(); + + $users[$user_name] = $response->getObject()->getData(); + } + + return $users; +} +``` + +```python +# We'll create a generator object that will yield a list of Riak objects +def fetch_all_user_records(): + users_bucket = client.bucket('users') + user_id_list = list(user_id_set.reload().value) + for user_id in user_id_list: + yield users_bucket.get(user_id) + +# We can retrieve that list of Riak objects later on +list(fetch_all_user_records()) +``` + +## Naming and Object Verification + +Another advantage of structured naming is that you can prevent queries +for objects that don't exist or that don't conform to how your +application has named them. For example, you could store all user data +in the bucket `users` with keys beginning with the fragment `user_` +followed by a username, e.g. `user_coderoshi` or `user_macintux`. If an +object with an inappropriate key is stored in that bucket, it won't even +be seen by your application because it will only ever query keys that +begin with `user_`: + +```java +// Assuming that we've created a class User: + +public User getUserByUsername(String username) { + String usernameKey = String.format("user_%s", username) + Location loc = new Location("users") + .setKey(usernameKey); + FetchValue fetchUser = new FetchValue.Builder(loc).build(); + FetchValue.Response res = client.execute(fetchUser); + User userObject = res.getValue(User.class); + return userObject; +} +``` + +```ruby +def get_user_by_username(username) + bucket = client.bucket('users') + obj = bucket.get('user_#{username}') + return obj.raw_data +end +``` + +```php +function fetchUser($user_name) +{ + $response = (new \Basho\Riak\Command\Builder\FetchObject) + ->buildLocation($user_name, 'users') + ->build() + ->execute(); + + return $response->getObject()->getData(); +} +``` + +```python +def get_user_by_username(username): + bucket = client.bucket('users') + obj = bucket.get('user_{}'.format(username)) + return obj.data +``` + +## Bucket Types as Additional Namespaces + +Riak [bucket types]({{}}riak/kv/2.2.6/developing/usage/bucket-types) have two essential functions: +they enable you to manage [bucket configurations]({{}}riak/kv/2.2.6/learn/concepts/buckets) in an +efficient and streamlined way and, more importantly for our purposes +here, they act as a third namespace in Riak in addition to buckets and +keys. Thus, in Riak versions 2.0 and later you have access to a third +layer of information for locating objects if you wish. + +While bucket types are typically used to assign different bucket +properties to groups of buckets, you can also create named bucket types +that simply extend Riak's [defaults]({{}}riak/kv/2.2.6/developing/usage/bucket-types/#bucket-types-as-namespaces) or multiple bucket types that have +the same configuration but have different names. + +Here's an example of creating four bucket types that only extend Riak's +defaults: + +```bash +riak-admin bucket-type create john +riak-admin bucket-type create robert +riak-admin bucket-type create jimmy +riak-admin bucket-type create john-paul +``` + +Or you can create five different bucket types that all set `n_val` to 2 +but have different names: + +```bash +riak-admin bucket-type create earth '{"props":{"n_val":2}}' +riak-admin bucket-type create fire '{"props":{"n_val":2}}' +riak-admin bucket-type create wind '{"props":{"n_val":2}}' +riak-admin bucket-type create water '{"props":{"n_val":2}}' +riak-admin bucket-type create heart '{"props":{"n_val":2}}' +``` + +### Bucket Types Example + +To extend our Simpsons example from above, imagine that we become +dissatisfied with our storage scheme because we want to separate the +seasons into good seasons and bad seasons (we'll leave it up to you to +make that determination). + +One way to improve our scheme might be to change our bucket naming +system and preface each bucket name with `good` or `bad`, but a more +elegant way would be to use bucket types instead. So instead of this URL +structure... + +``` +GET/PUT/DELETE /bucket//keys/ +``` + +...we can use this structure: + +``` +GET/PUT/DELETE /types//buckets//keys/ +``` + +That adds an additional layer of namespacing and enables us to think +about our data in terms of a deeper hash than in the example above: + +```ruby +simpsons = { + 'good': { + 'season X': { + { 'episode 1': '' }, + # ... + } + }, + 'bad': { + 'season Y': { + { 'episode 1': '<title>' }, + # ... + } + } +} +``` + +We can fetch the title of season 8, episode 6: + +```ruby +# For the sake of example, we'll classify season 8 as good: + +simpsons['good']['season 8']['episode 6'] + +# => "A Milhouse Divided" +``` + +If your data is best modeled as a three-layered hash, you may want to +consider using bucket types in the way shown above. + +## Resources + +More on key/value modeling in Riak can be found in [this +presentation](http://www.youtube.com/watch?v=-_3Us7Ystyg#aid=P-4heI_bFwo) +by Basho evangelist [Hector Castro](https://github.com/hectcastro), with +the presentation slides available [on Speaker +Deck](https://speakerdeck.com/hectcastro/throw-some-keys-on-it-data-modeling-for-key-value-data-stores-by-example). diff --git a/content/riak/kv/2.2.6/developing/usage.md b/content/riak/kv/2.2.6/developing/usage.md new file mode 100644 index 0000000000..7d83c88a70 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/usage.md @@ -0,0 +1,133 @@ +--- +title: "Usage Overview" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Usage" + identifier: "developing_usage" + weight: 101 + parent: "developing" +toc: true +--- + +## In This Section + +#### [Creating Objects](./creating-objects) + +Creating and storing objects in Riak KV. + +[Learn More >>](./creating-objects) + + +#### [Reading Objects](./reading-objects) + +Reading and fetching objects in Riak KV. + +[Learn More >>](./reading-objects) + + +#### [Updating Objects](./updating-objects) + +Updating objects in Riak KV. + +[Learn More >>](./updating-objects) + + +#### [Deleting Objects](./deleting-objects) + +Deleting objects in Riak KV. + +[Learn More >>](./deleting-objects) + + +#### [Content Types](./content-types) + +Overview of content types and their usage. + +[Learn More >>](./content-types) + + +#### [Using Search](./search) + +Tutorial on using search. + +[Learn More >>](./search) + + +#### [Using MapReduce](./mapreduce) + +Guide to using MapReduce in applications. + +[Learn More >>](./mapreduce) + + +#### [Using Secondary Indexes](./secondary-indexes) + +Overview and usage details of Secondary Indexes (2i). + +[Learn More >>](./secondary-indexes) + + +#### [Bucket Types](./bucket-types) + +Describes how to use bucket properties. + +[Learn More >>](./bucket-types) + + +#### [Using Commit Hooks](./commit-hooks) + +Tutorial on pre-commit and post-commit hook functions. + +[Learn More >>](./commit-hooks) + + +#### [Creating Search Schemas](./search-schemas) + +Step-by-step guide on creating and using custom search schemas. + +[Learn More >>](./search-schemas) + + +#### [Searching with Data Types](./searching-data-types) + +Guide on using search with Data Types. + +[Learn More >>](./searching-data-types) + + +#### [Implementing a Document Store](./document-store) + +Tutorial on using Riak KV as a document store. + +[Learn More >>](./document-store) + + +#### [Custom Extractors](./custom-extractors) + +Details on creating and registering custom extractors with Riak Search. + +[Learn More >>](./custom-extractors) + + +#### [Client-side Security](./security) + +Overview of client-side security. + +[Learn More >>](./security) + + +#### [Replication](./replication) + +Documentation on replication properties and their underlying implementation. + +[Learn More >>](./replication) + + +#### [Conflict Resolution](./conflict-resolution) + +Guide to conflict resolution during object updates. + +[Learn More >>](./conflict-resolution) diff --git a/content/riak/kv/2.2.6/developing/usage/bucket-types.md b/content/riak/kv/2.2.6/developing/usage/bucket-types.md new file mode 100644 index 0000000000..43fc2b8cb4 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/usage/bucket-types.md @@ -0,0 +1,98 @@ +--- +title: "Bucket Types" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Bucket Types" + identifier: "usage_bucket_types" + weight: 108 + parent: "developing_usage" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/advanced/bucket-types + - /riak-docs/riak/kv/2.2.6/dev/advanced/bucket-types +--- + +If you ever need to turn off indexing for a bucket, set the +`search_index` property to the `_dont_index_` sentinel value. + +## Bucket Properties + +Although we recommend that you use all new buckets under a bucket type, +if you have existing data with a type-free bucket (i.e. under the +`default` bucket type) you can set the `search_index` property for a +specific bucket. + +```java +Namespace catsBucket = new Namespace("cats"); +StoreBucketPropsOperation storePropsOp = new StoreBucketPropsOperation.Builder(catsBucket) + .withSearchIndex("famous") + .build(); +client.execute(storePropsOp); +``` + +```ruby +bucket = client.bucket('cats') +bucket.properties = {'search_index' => 'famous'} +``` + +```php +(new \Basho\Riak\Command\Builder\Search\AssociateIndex($riak)) + ->withName('famous') + ->buildBucket('cats') + ->build() + ->execute(); +``` + +```python +bucket = client.bucket('cats') +bucket.set_properties({'search_index': 'famous'}) +``` + +```csharp +var properties = new RiakBucketProperties(); +properties.SetSearchIndex("famous"); +var rslt = client.SetBucketProperties("cats", properties); +``` + +```javascript +var bucketProps_cb = function (err, rslt) { + if (err) { + throw new Error(err); + } + // success +}; + +var store = new Riak.Commands.KV.StoreBucketProps.Builder() + .withBucket("cats") + .withSearchIndex("famous") + .withCallback(bucketProps_cb) + .build(); + +client.execute(store); +``` + +```erlang +riakc_pb_socket:set_search_index(Pid, <<"cats">>, <<"famous">>). +``` + +```golang +cmd, err := riak.NewStoreBucketPropsCommandBuilder(). + WithBucketType("animals"). + WithBucket("cats"). + WithSearchIndex("famous"). + Build() +if err != nil { + return err +} + +err = cluster.Execute(cmd) +``` + +```curl +curl -XPUT $RIAK_HOST/buckets/cats/props \ + -H'content-type:application/json' \ + -d'{"props":{"search_index":"famous"}}' +``` diff --git a/content/riak/kv/2.2.6/developing/usage/commit-hooks.md b/content/riak/kv/2.2.6/developing/usage/commit-hooks.md new file mode 100644 index 0000000000..d5f10679e0 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/usage/commit-hooks.md @@ -0,0 +1,239 @@ +--- +title: "Using Commit Hooks" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Using Commit Hooks" + identifier: "usage_commit_hooks" + weight: 109 + parent: "developing_usage" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/using/commit-hooks + - /riak-docs/riak/kv/2.2.6/dev/using/commit-hooks +--- + +[usage bucket types]: {{<baseurl>}}riak/kv/2.2.6/developing/usage/bucket-types + +Pre- and post-commit hooks are functions that are invoked before or +after an object has been written to Riak. To provide a few examples, +commit hooks can: + +- allow a write to occur with an unmodified object +- modify an object +- fail an update and prevent any modifications to the object + +Post-commit hooks are notified _after the fact_ and should not modify +the object directly. Updating Riak objects while post-commit hooks are +invoked can cause nasty feedback loops which will wedge the hook into an +infinite cycle unless the hook functions are carefully written to detect +and short-circuit such cycles. + +Pre- and post-commit hooks are applied at the [bucket]({{<baseurl>}}riak/kv/2.2.6/learn/concepts/buckets) level, +[using bucket types][usage bucket types]. They are run once per successful response to the +client. + +Both pre- and post-commit hooks are named [Erlang](http://learnyousomeerlang.com/) +functions. + +## Setting Commit Hooks Using Bucket Types + +Because hooks are defined at the bucket level, you can create [bucket types]({{<baseurl>}}riak/kv/2.2.6/developing/usage/bucket-types) +that associate one or more hooks with any bucket that bears that type. +Let's create a bucket type called `with_post_commit` that adds a +post-commit hook to operations on any bucket that bears the +`with_post_commit` type. + +The format for specifying commit hooks is to identify the module (`mod`) +and then the name of the function (`fun`) as a JavaScript object. The +following specifies a commit hook called `my_custom_hook` in the module +`commit_hooks_module`: + +```json +{ + "mod": "commit_hooks_module", + "fun": "my_custom_hook" +} +``` + +When we create our `with_post_commit` bucket type, we add that object +to either the `precommit` or `postcommit` list in the bucket type's +properties. Pre- and post-commit hooks are stored in lists named +`precommit` and `postcommit`, respectively. Let's add the hook we +specified above to the `postcommit` property when we create our bucket +type: + +```bash +riak-admin bucket-type create with_post_commit \ + '{"props":{"postcommit":["my_post_commit_hook"]}' +``` + +Once our bucket type has been created, we must activate it so that it +will be usable through our Riak cluster: + +```bash +riak-admin bucket-type activate with_post_commit +``` + +If the response is `with_post_commit has been activated`, then the +bucket type is ready for use. + +## Pre-Commit Hooks + +Pre-commit hook Erlang functions should take a single argument, the +Riak object being modified. Remember that deletes are also considered +"writes," and so pre-commit hooks will be fired when a delete occurs in +the bucket as well. This means that hook functions will need to inspect +the object for the `X-Riak-Deleted` metadata entry (more on this in our +documentation on [object deletion]({{<baseurl>}}riak/kv/2.2.6/using/reference/object-deletion)) to determine whether a delete is +occurring. + +Erlang pre-commit functions are allowed three possible return values: + +- A Riak object --- This can either be the same object passed to the function or an updated version of the object. This allows hooks to modify the object before they are written. +- `fail` --- The atom `fail` will cause Riak to fail the write and send a 403 Forbidden error response (in the [HTTP API]({{<baseurl>}}riak/kv/2.2.6/developing/api/http)) along with a generic error message about why the write was blocked. +- `{fail, Reason}` --- The tuple `{fail, Reason}` will cause the same behavior as in the case above, but with the addition of `Reason` used as the error text. + +Errors that occur when processing Erlang pre-commit hooks will be +reported in the `sasl-error.log` file with lines that start with +`problem invoking hook`. + +#### Object Size Example + +This Erlang pre-commit hook will limit object values to 5 MB or smaller: + +```erlang +precommit_limit_size(Object) -> + case erlang:byte_size(riak_object:get_value(Object)) of + Size when Size > 5242880 -> {fail, "Object is larger than 5MB."}; + _ -> Object + end. +``` + +The Erlang function `precommit_limit_size` takes the Riak object +(`Object`) as its input and runs a pattern-matching operation on the +object. If the [`erlang:byte_size`](http://www.erlang.org/doc/man/erlang.html#byte_size-1) +function determines that the object's size (determined by the `riak_object:get_value` +function) is greater than 5,242,880 (5 MB in bytes), then the commit +will return failure and the message `Object size is larger than 5 MB`. +This will stop the write. If the object is not larger than 5 MB, Riak +will return the object and allow the write to proceed. + +### Chaining + +The default value of the bucket type's `precommit` property is an empty +list, meaning that no pre-commit hooks are specified by default. Adding +one or more pre-commit hook functions to this list, as documented above, +will cause Riak to start evaluating those hook functions when bucket +entries are created, updated, or deleted. Riak stops evaluating +pre-commit hooks when a hook function fails the commit. + +#### JSON Validation Example + +Pre-commit hooks can be used in many ways in Riak. One such way to use +pre-commmit hooks is to validate data before it is written to Riak. +Below is an example that uses Javascript to validate a JSON object +before it is written to Riak. + +Below is a sample JSON object that will be evaluated by the hook: + +```json +{ + "user_info": { + "name": "Mark Phillips", + "age": "25" + }, + "session_info": { + "id": 3254425, + "items": [29, 37, 34] + } +} +``` + +The following hook will validate the JSON object: + +```erlang +validate(Object) -> + try + mochijson2:decode(riak_object:get_value(Object)), + Object + catch + throw:invalid_utf8 -> + {fail, "Invalid JSON: Illegal UTF-8 character"}; + error:Error -> + {fail, lists:flatten(io_lib:format("Invalid JSON: ~p",[Error]))} + end. +``` + +**Note**: All pre-commit hook functions are executed for each create and update operation. + +## Post-Commit Hooks + +Post-commit hooks are run after a write has completed successfully. More +specifically, the hook function is called immediately before the calling +process is notified of the successful write. + +Hook functions must accept a single argument: the object instance just +written. The return value of the function is ignored. As with pre-commit +hooks, deletes are considered writes, so post-commit hook functions will +need to inspect the object's metadata for the presence of `X-Riak-Deleted` +to determine whether a delete has occurred. As with pre-commit hooks, +errors that occur when processing post-commit hooks will be reported in +the `sasl-error.log` file with lines that start with `problem invoking hook`. + +#### Example + +The following post-commit hook creates a secondary index on the `email` +field of a JSON object: + +```erlang +postcommit_index_on_email(Object) -> + %% Determine the target bucket name + Bucket = erlang:iolist_to_binary([riak_object:bucket(Object),"_by_email"]), + + %% Decode the JSON body of the object + {struct, Properties} = mochijson2:decode(riak_object:get_value(Object)), + + %% Extract the email field + {<<"email">>,Key} = lists:keyfind(<<"email">>,1,Properties), + + %% Create a new object for the target bucket + %% NOTE: This doesn't handle the case where the + %% index object already exists! + IndexObj = riak_object:new( + Bucket, Key, <<>>, %% no object contents + dict:from_list( + [ + {<<"content-type">>, "text/plain"}, + {<<"Links">>, + [ + { + {riak_object:bucket(Object), riak_object:key(Object)}, + <<"indexed">> + }]} + ] + ) + ), + + %% Get a riak client + {ok, C} = riak:local_client(), + + %% Store the object + C:put(IndexObj). +``` + + +### Chaining + +The default value of the bucket `postcommit` property is an empty list, +meaning that no post-commit hooks are specified by default. Adding one +or more post-commit hook functions to the list, as documented above, +will cause Riak to start evaluating those hook functions immediately +after data has been created, updated, or deleted. Each post-commit hook +function runs in a separate process so it's possible for several hook +functions, triggered by the same update, to execute in parallel. + +**Note**: All post-commit hook functions are executed for each create, +update, or delete. diff --git a/content/riak/kv/2.2.6/developing/usage/conflict-resolution.md b/content/riak/kv/2.2.6/developing/usage/conflict-resolution.md new file mode 100644 index 0000000000..2b980dbb44 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/usage/conflict-resolution.md @@ -0,0 +1,677 @@ +--- +title: "Conflict Resolution" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Conflict Resolution" + identifier: "usage_conflict_resolution" + weight: 116 + parent: "developing_usage" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/using/conflict-resolution + - /riak-docs/riak/kv/2.2.6/dev/using/conflict-resolution +--- + +[usage bucket types]: {{<baseurl>}}riak/kv/2.2.6/developing/usage/bucket-types +[use ref strong consistency]: {{<baseurl>}}riak/kv/2.2.6/using/reference/strong-consistency + +One of Riak's [central goals](../../../learn/why-riak-kv) is high availability. It was built as a [clustered]({{<baseurl>}}riak/kv/2.2.6/learn/concepts/clusters) system in which any [node]({{<baseurl>}}riak/kv/2.2.6/learn/glossary/#node) is capable of receiving requests without requiring that +every node participate in each request. + +If you are using Riak in an [eventually consistent]({{<baseurl>}}riak/kv/2.2.6/learn/concepts/eventual-consistency) way, conflicts between object values on different nodes is +unavoidable. Often, Riak can resolve these conflicts on its own +internally if you use causal context, i.e. [vector clocks]({{<baseurl>}}riak/kv/2.2.6/learn/concepts/causal-context#vector-clocks) or [dotted version vectors]({{<baseurl>}}riak/kv/2.2.6/learn/concepts/causal-context#dotted-version-vectors), when updating objects. Instructions on this can be found in the section [below](#siblings). + +{{% note title="Important note on terminology" %}} +In versions of Riak prior to 2.0, vector clocks were the only causal context +mechanism available in Riak, which changed with the introduction of dotted +version vectors in 2.0. Please note that you may frequent find terminology in +client library APIs, internal Basho documentation, and more that uses the term +"vector clock" interchangeably with causal context in general. Riak's HTTP API +still uses a `X-Riak-Vclock` header, for example, even if you are using dotted +version vectors. +{{% /note %}} + +But even when you use causal context, Riak cannot always decide which +value is most causally recent, especially in cases involving concurrent +updates to an object. So how does Riak behave when it can't decide on a +single most-up-to-date value? **That is your choice**. A full listing of +available options can be found in the [section below](#client-and-server-side-conflict-resolution). For now, +though, please bear in mind that we strongly recommend one of the +following two options: + +1. If your data can be modeled as one of the currently available [Riak + Data Types]({{<baseurl>}}riak/kv/2.2.6/developing/data-types), we recommend using one of these types, + because all of them have conflict resolution _built in_, completely + relieving applications of the need to engage in conflict resolution. +2. If your data cannot be modeled as one of the available Data Types, + we recommend allowing Riak to generate [siblings](#siblings) and to design your application to resolve + conflicts in a way that fits your use case. Developing your own + **conflict resolution strategy** can be tricky, but it has clear + advantages over other approaches. + +Because Riak allows for a mixed approach when storing and managing data, +you can apply multiple conflict resolution strategies within a cluster. + +> **Note on strong consistency** +> +> In versions of Riak 2.0 and later, you have the option of using Riak in +a strongly consistent fashion. This document pertains to usage of Riak +as an _eventually_ consistent system. If you'd like to use Riak's +strong consistency feature, please refer to the following documents: +> +> * [Using Strong Consistency]({{<baseurl>}}riak/kv/2.2.6/developing/app-guide/strong-consistency) --- A guide for developers +> * [Managing Strong Consistency]({{<baseurl>}}riak/kv/2.2.6/configuring/strong-consistency) --- A guide for operators +> * [strong consistency][use ref strong consistency] --- A more theoretical explication of strong + consistency + +## Client- and Server-side Conflict Resolution + +Riak's eventual consistency model is powerful because Riak is +fundamentally non-opinionated about how data resolution takes place. +While Riak _does_ have a set of [defaults]({{<baseurl>}}riak/kv/2.2.6/developing/app-guide/replication-properties#available-parameters), there are a variety of general +approaches to conflict resolution that are available. In Riak, you can +mix and match conflict resolution strategies at the bucket level, +[using bucket types][usage bucket types]. The most important [bucket properties]({{<baseurl>}}riak/kv/2.2.6/learn/concepts/buckets) +to consider when reasoning about conflict resolution are the +`allow_mult` and `last_write_wins` properties. + +These properties provide you with the following basic options: + +### Timestamp-based Resolution + +If the [`allow_mult`](#siblings) parameter is set to +`false`, Riak resolves all object replica conflicts internally and does +not return siblings to the client. How Riak resolves those conflicts +depends on the value that you set for a different bucket property, +[`last_write_wins`]({{<baseurl>}}riak/kv/2.2.6/learn/concepts/buckets). If `last_write_wins` is set to `false`, +Riak will resolve all conflicts on the basis of +[timestamps](http://en.wikipedia.org/wiki/Timestamp), which are +attached to all Riak objects as metadata. + +The problem with timestamps is that they are not a reliable resolution +mechanism in distributed systems, and they always bear the risk of data +loss. A better yet still-problematic option is to adopt a +last-write-wins strategy, described directly below. + +### Last-write-wins + +Another way to manage conflicts is to set `allow_mult` to `false`, as +with timestamp-based resolution, while also setting the +`last_write_wins` parameter to +`true`. This produces a so-called last-write-wins (LWW) strategy whereby +Riak foregoes the use of all internal conflict resolution strategies +when making writes, effectively disregarding all previous writes. + +The problem with LWW is that it will necessarily drop some writes in the +case of concurrent updates in the name of preventing sibling creation. +If your use case requires that your application be able to reason about +differing values produced in the case of concurrent updates, then we +advise against LWW as a general conflict resolution strategy. + +However, LWW can be useful---and safe---if you are certain that there +will be no concurrent updates. If you are storing immutable data in +which each object is guaranteed to have its own key or engaging in +operations related to bulk loading, you should consider LWW. + +{{% note title="Undefined behavior warning" %}} +Setting both `allow_mult` and `last_write_wins` to `true` necessarily leads to +unpredictable behavior and should always be avoided. +{{% /note %}} + +### Resolve Conflicts on the Application Side + +While setting `allow_mult` to `false` unburdens applications from having +to reason about siblings, delegating that responsibility to Riak itself, +it bears all of the drawbacks explained above. On the other hand, +setting `allow_mult` to `true` has the following benefits: + +* Riak will retain writes even in the case of concurrent updates to a + key, which enables you to capture the benefits of high availability + with a far lower risk of data loss +* If your application encounters siblings, it can apply its own + use-case-specific conflict resolution logic + +Conflict resolution in Riak can be a complex business, but the presence +of this variety of options means that requests to Riak can always be +made in accordance with your data model(s), business needs, and use +cases. For examples of client-side sibling resolution, see the following +client-library-specific docs: + +* [Java]({{<baseurl>}}riak/kv/2.2.6/developing/usage/conflict-resolution/java) +* [Ruby]({{<baseurl>}}riak/kv/2.2.6/developing/usage/conflict-resolution/ruby) +* [Python]({{<baseurl>}}riak/kv/2.2.6/developing/usage/conflict-resolution/python) +* [C#]({{<baseurl>}}riak/kv/2.2.6/developing/usage/conflict-resolution/csharp) +* [Node.js]({{<baseurl>}}riak/kv/2.2.6/developing/usage/conflict-resolution/nodejs) + +In Riak versions 2.0 and later, `allow_mult` is set to `true` by default +for any [bucket types]({{<baseurl>}}riak/kv/2.2.6/developing/usage/bucket-types) that you create. This means +that if you wish to avoid client-side sibling resolution, you have a few +options: + +* Explicitly create and activate [bucket types]({{<baseurl>}}riak/kv/2.2.6/developing/usage/bucket-types) + that set `allow_mult` to `false` +* Use Riak's [Configuration Files]({{<baseurl>}}riak/kv/2.2.6/configuring/reference) to change the [default bucket properties]({{<baseurl>}}riak/kv/2.2.6/configuring/reference#default-bucket-properties) for your + cluster. If you set the `buckets.default.allow_mult` parameter to + `false`, all bucket types that you create will have `allow_mult` set + to `false` by default. + +## Causal Context + +When a value is stored in Riak, it is tagged with a piece of metadata +called a **causal context** which establishes the object's initial +version. Causal context comes in one of two possible forms, depending +on what value you set for `dvv_enabled`. If set to `true`, [dotted version vectors]({{<baseurl>}}riak/kv/2.2.6/learn/concepts/causal-context#dotted-version-vectors) will be used; if set to `false` (the default), [vector clocks]({{<baseurl>}}riak/kv/2.2.6/learn/concepts/causal-context#vector-clocks) will be used. + +Causal context essentially enables Riak to compare the different values +of objects stored in Riak and to determine a number of important things +about those values: + + * Whether one value is a direct descendant of the other + * Whether the values are direct descendants of a common parent + * Whether the values are unrelated in recent heritage + +Using the information provided by causal context, Riak is frequently, +though not always, able to resolve conflicts between values without +producing siblings. + +Both vector clocks and dotted version vectors are non human readable and +look something like this: + +``` +a85hYGBgzGDKBVIcR4M2cgczH7HPYEpkzGNlsP/VfYYvCwA= +``` + +If `allow_mult` is set to `true`, you should _always_ use causal context +when updating objects, _unless you are certain that no object exists +under that key_. Failing to use causal context with mutable data, +especially for objects that are frequently updated, can lead to +[sibling explosion]({{<baseurl>}}riak/kv/2.2.6/using/performance/latency-reduction#siblings), which can +produce a variety of problems in your cluster. Fortunately, much of the +work involved with using causal context is handled automatically by +Basho's official [client libraries]({{<baseurl>}}riak/kv/2.2.6/developing/client-libraries). Examples can be found for each +client library in the [Object Updates]({{<baseurl>}}riak/kv/2.2.6/developing/usage/updating-objects) document. + +## Siblings + +A **sibling** is created when Riak is unable to resolve the canonical +version of an object being stored, i.e. when Riak is presented with +multiple possible values for an object and can't figure out which one is +most causally recent. The following scenarios can create sibling values +inside of a single object: + +1. **Concurrent writes** --- If two writes occur simultaneously from +clients, Riak may not be able to choose a single value to store, in +which case the object will be given a sibling. These writes could happen +on the same node or on different nodes. +2. **Stale causal context** --- Writes from any client using a stale +[causal context]({{<baseurl>}}riak/kv/2.2.6/learn/concepts/causal-context). This is a less likely scenario if a client updates +the object by reading the object first, fetching the causal context +currently attached to the object, and then returning that causal context +to Riak when performing the update (fortunately, our client libraries +handle much of this automatically). However, even if a client follows +this protocol when performing updates, a situation may occur in which an +update happens from a different client while the read/write cycle is +taking place. This may cause the first client to issue the write with an +old causal context value and for a sibling to be created. A client is +"misbehaved" if it habitually updates objects with a stale or no context +object. +3. **Missing causal context** --- If an object is updated with no causal +context attached, siblings are very likely to be created. This is an +unlikely scenario if you're using a Basho client library, but it _can_ +happen if you are manipulating objects using a client like `curl` and +forgetting to set the `X-Riak-Vclock` header. + +## Siblings in Action + +Let's have a more concrete look at how siblings work in Riak. First, +we'll create a bucket type called `siblings_allowed` with `allow_mult` +set to `true`: + +```bash +riak-admin bucket-type create siblings_allowed '{"props":{"allow_mult":true}}' +riak-admin bucket-type activate siblings_allowed +riak-admin bucket-type status siblings_allowed +``` + +If the type has been activated, running the `status` command should +return `siblings_allowed is active`. Now, we'll create two objects and +write both of them to the same key without first fetching the object +(which obtains the causal context): + +```java +Location bestCharacterKey = + new Location(new Namespace("siblings_allowed", "nickolodeon"), "best_character"); + +RiakObject obj1 = new RiakObject() + .withContentType("text/plain") + .withValue(BinaryValue.create("Ren")); +RiakObject obj2 = new RiakObject() + .withContentType("text/plain") + .withValue(BinaryValue.create("Stimpy")); +StoreValue store1 = new StoreValue.Builder(obj1) + .withLocation(bestCharacterKey) + .build(); +StoreValue store2 = new StoreValue.Builder(obj2) + .withLocation(bestCharacterKey) + .build(); +client.execute(store1); +client.execute(store2); +``` + +```ruby +bucket = client.bucket_type('siblings_allowed').bucket('nickolodeon') +obj1 = Riak::RObject.new(bucket, 'best_character') +obj1.content_type = 'text/plain' +obj1.raw_data = 'Ren' +obj1.store + +obj2 = Riak::RObject.new(bucket, 'best_character') +obj2.content_type = 'text/plain' +obj2.raw_data = 'Stimpy' +obj2.store +``` + +```python +bucket = client.bucket_type('siblings_allowed').bucket('nickolodeon') +obj1 = RiakObject(client, bucket, 'best_character') +obj1.content_type = 'text/plain' +obj1.data = 'Ren' +obj1.store() + +obj2 = RiakObject(client, bucket, 'best_character') +obj2.content_type = 'text/plain' +obj2.data = 'Stimpy' +obj2.store() +``` + +```csharp +var id = new RiakObjectId("siblings_allowed", "nickolodeon", "best_character"); + +var renObj = new RiakObject(id, "Ren", RiakConstants.ContentTypes.TextPlain); +var stimpyObj = new RiakObject(id, "Stimpy", RiakConstants.ContentTypes.TextPlain); + +var renResult = client.Put(renObj); +var stimpyResult = client.Put(stimpyObj); +``` + +```javascript +var obj1 = new Riak.Commands.KV.RiakObject(); +obj1.setContentType('text/plain'); +obj1.setBucketType('siblings_allowed'); +obj1.setBucket('nickolodeon'); +obj1.setKey('best_character'); +obj1.setValue('Ren'); + +var obj2 = new Riak.Commands.KV.RiakObject(); +obj2.setContentType('text/plain'); +obj2.setBucketType('siblings_allowed'); +obj2.setBucket('nickolodeon'); +obj2.setKey('best_character'); +obj2.setValue('Ren'); + +var storeFuncs = []; +[obj1, obj2].forEach(function (obj) { + storeFuncs.push( + function (async_cb) { + client.storeValue({ value: obj }, function (err, rslt) { + async_cb(err, rslt); + }); + } + ); +}); + +async.parallel(storeFuncs, function (err, rslts) { + if (err) { + throw new Error(err); + } +}); +``` + +```erlang +Obj1 = riakc_obj:new({<<"siblings_allowed">>, <<"nickolodeon">>}, + <<"best_character">>, + <<"Ren">>, + <<"text/plain">>), +Obj2 = riakc_obj:new({<<"siblings_allowed">>, <<"nickolodeon">>}, + <<"best_character">>, + <<"Stimpy">>, + <<"text/plain">>), +riakc_pb_socket:put(Pid, Obj1), +riakc_pb_socket:put(Pid, Obj2). +``` + +```curl +curl -XPUT http://localhost:8098/types/siblings_allowed/nickolodeon/whatever/keys/best_character \ + -H "Content-Type: text/plain" \ + -d "Ren" + +curl -XPUT http://localhost:8098/types/siblings_allowed/nickolodeon/whatever/keys/best_character \ + -H "Content-Type: text/plain" \ + -d "Stimpy" +``` + +> **Getting started with Riak KV clients** +> +> If you are connecting to Riak using one of Basho's official +[client libraries]({{<baseurl>}}riak/kv/2.2.6/developing/client-libraries), you can find more information about getting started with your client in [Developing with Riak KV: Getting Started]({{<baseurl>}}riak/kv/2.2.6/developing/getting-started) section. + +At this point, multiple objects have been stored in the same key without +passing any causal context to Riak. Let's see what happens if we try to +read contents of the object: + +```java +Location bestCharacterKey = + new Location(new Namespace("siblings_allowed", "nickolodeon"), "best_character"); + +FetchValue fetch = new FetchValue.Builder(bestCharacterKey).build(); +FetchValue.Response response = client.execute(fetch); +RiakObject obj = response.getValue(RiakObject.class); +System.out.println(obj.getValue().toString()); +``` + +```ruby +bucket = client.bucket_type('siblings_allowed').bucket('nickolodeon') +obj = bucket.get('best_character') +obj +``` + +```python +bucket = client.bucket_type('siblings_allowed').bucket('nickolodeon') +obj = bucket.get('best_character') +obj.siblings +``` + +```csharp +var id = new RiakObjectId("siblings_allowed", "nickolodeon", "best_character"); +var getResult = client.Get(id); +RiakObject obj = getResult.Value; +Debug.WriteLine(format: "Sibling count: {0}", args: obj.Siblings.Count); +foreach (var sibling in obj.Siblings) +{ + Debug.WriteLine( + format: " VTag: {0}", + args: sibling.VTag); +} +``` + +```javascript +client.fetchValue({ + bucketType: 'siblings_allowed', bucket: + 'nickolodeon', key: 'best_character' +}, function (err, rslt) { + if (err) { + throw new Error(err); + } + logger.info("nickolodeon/best_character has '%d' siblings", + rslt.values.length); +}); +``` + +```curl +curl http://localhost:8098/types/siblings_allowed/buckets/nickolodeon/keys/best_character +``` + +Uh-oh! Siblings have been found. We should get this response: + +```java +com.basho.riak.client.cap.UnresolvedConflictException: Siblings found +``` + +```ruby +<Riak::RObject {nickolodeon,best_character} [#<Riak::RContent [text/plain]:"Ren">, #<Riak::RContent [text/plain]:"Stimpy">]> +``` + +```python +[<riak.content.RiakContent object at 0x10a00eb90>, <riak.content.RiakContent object at 0x10a00ebd0>] +``` + +```csharp +Sibling count: 2 + VTag: 1DSVo7VED8AC6llS8IcDE6 + VTag: 7EiwrlFAJI5VMLK87vU4tE +``` + +```javascript +info: nickolodeon/best_character has '2' siblings +``` + +```curl +Siblings: +175xDv0I3UFCfGRC7K7U9z +6zY2mUCFPEoL834vYCDmPe +``` + +As you can see, reading an object with sibling values will result in +some form of "multiple choices" response (e.g. `300 Multiple Choices` in +HTTP). If you're using the HTTP interface and want to view all sibling +values, you can attach an `Accept: multipart/mixed` header to your +request: + +```curl +curl -H "Accept: multipart/mixed" \ + http://localhost:8098/types/siblings_allowed/buckets/nickolodeon/keys/best_character +``` + +Response (without headers): + +``` +ren +--WUnzXITIPJFwucNwfdaofMkEG7H + +stimpy +--WUnzXITIPJFwucNwfdaofMkEG7H-- +``` + +If you select the first of the two siblings and retrieve its value, you +should see `Ren` and not `Stimpy`. + +### Using Causal Context + +Once you are presented with multiple options for a single value, you +must determine the correct value. In an application, this can be done +either in an automatic fashion, using a use case-specific resolver, or +by presenting the conflicting objects to the end user. For more +information on application-side conflict resolution, see our +client-library-specific documentation for the following languages: + +* [Java]({{<baseurl>}}riak/kv/2.2.6/developing/usage/conflict-resolution/java) +* [Ruby]({{<baseurl>}}riak/kv/2.2.6/developing/usage/conflict-resolution/ruby) +* [Python]({{<baseurl>}}riak/kv/2.2.6/developing/usage/conflict-resolution/python) +* [C#]({{<baseurl>}}riak/kv/2.2.6/developing/usage/conflict-resolution/csharp) +* [Node.js]({{<baseurl>}}riak/kv/2.2.6/developing/usage/conflict-resolution/nodejs) + +We won't deal with conflict resolution in this section. Instead, we'll +focus on how to use causal context. + +After having written several objects to Riak in the section above, we +have values in our object: `Ren` and `Stimpy`. But let's say that we +decide that `Stimpy` is the correct value based on our application's use +case. In order to resolve the conflict, we need to do three things: + +1. Fetch the current object (which will return both siblings) +2. Modify the value of the object, i.e. make the value `Stimpy` +3. Write the object back to the `best_character` key + +What happens when we fetch the object first, prior to the update, is +that the object handled by the client has a causal context attached. At +that point, we can modify the object's value, and when we write the +object back to Riak, _the causal context will automatically be attached +to it_. Let's see what that looks like in practice: + +```java +// First, we fetch the object +Location bestCharacterKey = + new Location(new Namespace("siblings_allowed", "nickolodeon"), "best_character"); +FetchValue fetch = new FetchValue.Builder(bestCharacterKey).build(); +FetchValue.Response res = client.execute(fetch); +RiakObject obj = res.getValue(RiakObject.class); + + +// Then we modify the object's value +obj.setValue(BinaryValue.create("Stimpy")); + +// Then we store the object, which has the vector clock already attached +StoreValue store = new StoreValue.Builder(obj) + .withLocation(bestCharacterKey); +client.execute(store); +``` + +```ruby +# First, we fetch the object +bucket = client.bucket('nickolodeon') +obj = bucket.get('best_character', type: 'siblings_allowed') + +# Then we modify the object's value +obj.raw_data = 'Stimpy' + +# Then we store the object, which has the vector clock already attached +obj.store +``` + +```python +# First, we fetch the object +bucket = client.bucket_type('siblings_allowed').bucket('nickolodeon') +obj = bucket.get('best_character') + +# Then we modify the object's value +new_obj.data = 'Stimpy' + +# Then we store the object, which has the vector clock already attached +new_obj.store(vclock=vclock) +``` + +```csharp +// First, fetch the object +var getResult = client.Get(id); + +// Then, modify the object's value +RiakObject obj = getResult.Value; +obj.SetObject<string>("Stimpy", RiakConstants.ContentTypes.TextPlain); + +// Then, store the object which has vector clock attached +var putRslt = client.Put(obj); +CheckResult(putRslt); + +obj = putRslt.Value; +// Voila, no more siblings! +Debug.Assert(obj.Siblings.Count == 0); +``` + +```javascript +client.fetchValue({ + bucketType: 'siblings_allowed', + bucket: 'nickolodeon', + key: 'best_character' + }, function (err, rslt) { + if (err) { + throw new Error(err); + } + + var riakObj = rslt.values.shift(); + riakObj.setValue('Stimpy'); + client.storeValue({ value: riakObj, returnBody: true }, + function (err, rslt) { + if (err) { + throw new Error(err); + } + + assert(rslt.values.length === 1); + } + ); + } +); +``` + +```curl +curl -i http://localhost:8098/types/siblings_allowed/buckets/nickolodeon/keys/best_character + +# In the HTTP interface, the causal context can be found in the +# "X-Riak-Vclock" header. That will look something like this: + +X-Riak-Vclock: a85hYGBgzGDKBVIcR4M2cgczH7HPYEpkzGNlsP/VfYYvCwA= + +# When performing a write to the same key, that same header needs to +# accompany the write for Riak to be able to use the vector clock +``` + +{{% note title="Concurrent conflict resolution" %}} +It should be noted that it is possible to have two clients that are +simultaneously engaging in conflict resolution. To avoid a pathological +divergence, you should be sure to limit the number of reconciliations and fail +once that limit has been exceeded. +{{% /note %}} + +### Sibling Explosion + +Sibling explosion occurs when an object rapidly collects siblings +without being reconciled. This can lead to myriad issues. Having an +enormous object in your node can cause reads of that object to crash +the entire node. Other issues include [increased cluster latency]({{<baseurl>}}riak/kv/2.2.6/using/performance/latency-reduction) as the object is replicated and out-of-memory errors. + +### Vector Clock Explosion + +Besides sibling explosion, the vector clock itself can grow extremely +large when a significant volume of updates are performed on a single +object in a small period of time. While updating a single object +_extremely_ frequently is not recommended, you can tune Riak's vector +clock pruning to prevent vector clocks from growing too large too +quickly. More on pruning in the [section below](#vector-clock-pruning). + +### How does `last_write_wins` affect resolution? + +On the surface, it seems like setting `allow_mult` to `false` +(the default) and `last_write_wins` to `true` would result in the same +behavior, but there is a subtle distinction. + +Even though both settings return only one value to the client, setting +`allow_mult` to `false` still uses vector clocks for resolution, whereas +if `last_write_wins` is `true`, Riak reads the timestamp to determine +the latest version. Deeper in the system, if `allow_mult` is `false`, +Riak will still allow siblings to exist when they are created (via +concurrent writes or network partitions), whereas setting +`last_write_wins` to `true` means that Riak will overwrite the value +with the one that has the later timestamp. + +When you don't care about sibling creation, setting `allow_mult` to +`false` has the least surprising behavior: you get the latest value, +but network partitions are handled gracefully. However, for cases in +which keys are rewritten often (and quickly) and the new value isn't +necessarily dependent on the old value, `last_write_wins` will provide +better performance. Some use cases where you might want to use +`last_write_wins` include caching, session storage, and insert-only +(no updates). + +{{% note title="Note on combining `allow_mult` and `last_write_wins`" %}} +The combination of setting both the `allow_mult` and `last_write_wins` +properties to `true` leads to undefined behavior and should not be used. +{{% /note %}} + +## Vector Clock Pruning + +Riak regularly prunes vector clocks to prevent overgrowth based on four +parameters which can be set for any bucket type that you create: + +Parameter | Default value | Description +:---------|:--------------|:----------- +`small_vclock` | `50` | If the length of the vector clock list is smaller than this value, the list's entries will not be pruned +`big_vclock` | `50` | If the length of the vector clock list is larger than this value, the list will be pruned +`young_vclock` | `20` | If a vector clock entry is younger than this value (in milliseconds), it will not be pruned +`old_vclock` | `86400` (one day) | If a vector clock entry is older than this value (in milliseconds), it will be pruned + +This diagram shows how the values of these parameters dictate the vector +clock pruning process: + +![Vclock Pruning]({{<baseurl>}}images/vclock-pruning.png) + +## More Information + +Additional background information on vector clocks: + +* [Vector Clocks on Wikipedia](http://en.wikipedia.org/wiki/Vector_clock) +* [Why Vector Clocks are Easy](http://basho.com/why-vector-clocks-are-easy/) +* [Why Vector Clocks are Hard](http://basho.com/why-vector-clocks-are-hard/) +* The vector clocks used in Riak are based on the [work of Leslie Lamport](http://portal.acm.org/citation.cfm?id=359563) diff --git a/content/riak/kv/2.2.6/developing/usage/conflict-resolution/csharp.md b/content/riak/kv/2.2.6/developing/usage/conflict-resolution/csharp.md new file mode 100644 index 0000000000..ff377a5d16 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/usage/conflict-resolution/csharp.md @@ -0,0 +1,119 @@ +--- +title_supertext: "Conflict Resolution:" +title: "C Sharp" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "C Sharp" + identifier: "usage_conflict_resolution_csharp" + weight: 103 + parent: "usage_conflict_resolution" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/using/conflict-resolution/csharp + - /riak-docs/riak/kv/2.2.6/dev/using/conflict-resolution/csharp +--- + +For reasons explained in the [Introduction to conflict resolution]({{<baseurl>}}riak/kv/2.2.6/developing/usage/conflict-resolution), we strongly recommend adopting a conflict resolution strategy that requires applications to resolve siblings according to use-case-specific +criteria. Here, we'll provide a brief guide to conflict resolution using the +official [Riak .NET client][riak_dotnet_client]. + +## How the .NET Client Handles Conflict Resolution + +In the Riak .NET client, every Riak object has a `siblings` property that +provides access to a list of that object's sibling values. If there are no +siblings, that property will return an empty list. + +Here's an example of an object with siblings: + +```csharp +var id = new RiakObjectId("siblings_allowed", "nickolodeon", "best_character"); + +var renObj = new RiakObject(id, "Ren", RiakConstants.ContentTypes.TextPlain); +var stimpyObj = new RiakObject(id, "Stimpy", RiakConstants.ContentTypes.TextPlain); + +var renResult = client.Put(renObj); +var stimpyResult = client.Put(stimpyObj); + +var getResult = client.Get(id); +RiakObject obj = getResult.Value; +Debug.WriteLine(format: "Sibling count: {0}", args: obj.Siblings.Count); +foreach (var sibling in obj.Siblings) +{ + Debug.WriteLine( + format: " VTag: {0}", + args: sibling.VTag); +} +``` + +So what happens if the count of `obj.Siblings` is greater than 0, as in the case +above? + +In order to resolve siblings, you need to either fetch, update and store a +canonical value, or choose a sibling from the `Siblings` list and store that as +the canonical value. + +## Basic Conflict Resolution Example + +In this example, you will ignore the contents of the `Siblings` list and will +fetch, update and store the definitive value. + +```csharp +var id = new RiakObjectId("siblings_allowed", "nickolodeon", "best_character"); + +var renObj = new RiakObject(id, "Ren", RiakConstants.ContentTypes.TextPlain); +var stimpyObj = new RiakObject(id, "Stimpy", RiakConstants.ContentTypes.TextPlain); + +var renResult = client.Put(renObj); +var stimpyResult = client.Put(stimpyObj); + +var getResult = client.Get(id); +RiakObject obj = getResult.Value; +Debug.Assert(obj.Siblings.Count == 2); + +// Now, modify the object's value +obj.SetObject<string>("Stimpy", RiakConstants.ContentTypes.TextPlain); + +// Then, store the object which has vector clock attached +var putRslt = client.Put(obj); +CheckResult(putRslt); + +obj = putRslt.Value; +// Voila, no more siblings! +Debug.Assert(obj.Siblings.Count == 0); +``` + +### Choosing a value from `Siblings` + +This example shows a basic sibling resolution strategy in which the first +sibling is chosen as the canonical value. + +```csharp +var id = new RiakObjectId("siblings_allowed", "nickolodeon", "best_character"); + +var renObj = new RiakObject(id, "Ren", RiakConstants.ContentTypes.TextPlain); +var stimpyObj = new RiakObject(id, "Stimpy", RiakConstants.ContentTypes.TextPlain); + +var renResult = client.Put(renObj); +var stimpyResult = client.Put(stimpyObj); + +var getResult = client.Get(id); +RiakObject obj = getResult.Value; +Debug.Assert(obj.Siblings.Count == 2); + +// Pick the first sibling +RiakObject chosenSibling = getResult.Value.Siblings.First(); + +// Then, store the chosen object +var putRslt = client.Put(chosenSibling); +CheckResult(putRslt); + +RiakObject updatedObject = putRslt.Value; +// Voila, no more siblings! +Debug.Assert(updatedObject.Siblings.Count == 0); +``` + + +[riak_dotnet_client]: https://github.com/basho/riak-dotnet-client diff --git a/content/riak/kv/2.2.6/developing/usage/conflict-resolution/golang.md b/content/riak/kv/2.2.6/developing/usage/conflict-resolution/golang.md new file mode 100644 index 0000000000..ee086e9308 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/usage/conflict-resolution/golang.md @@ -0,0 +1,58 @@ +--- +title_supertext: "Conflict Resolution:" +title: "Go" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Go" + identifier: "usage_conflict_resolution_golang" + weight: 106 + parent: "usage_conflict_resolution" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/using/conflict-resolution/golang + - /riak-docs/riak/kv/2.2.6/dev/using/conflict-resolution/golang +--- + +For reasons explained in the [Introduction to conflict resolution]({{<baseurl>}}riak/kv/2.2.6/developing/usage/conflict-resolution), we strongly recommend adopting a conflict resolution strategy that +requires applications to resolve siblings according to usecase-specific +criteria. Here, we'll provide a brief guide to conflict resolution using the +official [Riak Go client](https://github.com/basho/riak-go-client). + +## How the Go Client Handles Conflict Resolution + +In the Riak Go client, it is possible that the result of a fetch will return an array +of sibling objects. If there are no siblings, that property will return an +array with one value in it. + +[*Example:* creating object with siblings](https://github.com/basho/riak-go-client/blob/master/examples/dev/using/conflict-resolution/main.go#L68-L70) + +So what happens if the length of `Values` is greater than 1, as in the case +above? + +In order to resolve siblings, you need to either: fetch, update, and store a +canonical value; or choose a sibling from the `Values` slice and store that as +the canonical value. + +## Basic Conflict Resolution Example + +In this example, you will ignore the contents of the `Values` slice and will +fetch, update and store the definitive value. + +[*Example:* resolving siblings via store](https://github.com/basho/riak-nodejs-client-examples/blob/master/dev/using/conflict-resolution.js#L125-L146) + +### Choosing a value from `Values` + +This example shows a basic sibling resolution strategy in which the first +sibling is chosen as the canonical value. + +[*Example:* resolving siblings using the first value](https://github.com/basho/riak-go-client/blob/master/examples/dev/using/conflict-resolution/main.go#L148-L167) + +### Using `ConflictResolver` + +This example shows a basic sibling resolution strategy in which the first +sibling is chosen as the canonical value via a conflict resolution type. + +[*Example:* resolving siblings via `ConflictResolver`](https://github.com/basho/riak-go-client/blob/master/examples/dev/using/conflict-resolution/main.go#L169-L210) diff --git a/content/riak/kv/2.2.6/developing/usage/conflict-resolution/java.md b/content/riak/kv/2.2.6/developing/usage/conflict-resolution/java.md new file mode 100644 index 0000000000..82a9d02cf7 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/usage/conflict-resolution/java.md @@ -0,0 +1,272 @@ +--- +title_supertext: "Conflict Resolution:" +title: "Java" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Java" + identifier: "usage_conflict_resolution_java" + weight: 100 + parent: "usage_conflict_resolution" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/using/conflict-resolution/java + - /riak-docs/riak/kv/2.2.6/dev/using/conflict-resolution/java +--- + +For reasons explained in the [Introduction to conflict resolution]({{<baseurl>}}riak/kv/2.2.6/developing/usage/conflict-resolution), we strongly recommend adopting a +conflict resolution strategy that requires applications to resolve +siblings according to use-case-specific criteria. Here, we'll provide a +brief guide to conflict resolution using the official [Riak Java +client](https://github.com/basho/riak-java-client). + +## How the Java Client Handles Conflict Resolution + +The official Riak Java client provides a `ConflictResolver` interface +for handling sibling resolution. This interface requires that you +implement a `resolve` method that takes a Java `List` of objects of a +specific type that are stored in Riak and produces a single object of +that type, i.e. converts a `List<T>` to a single `T`. Once that +interface has been implemented, it can be registered as a singleton and +thereby applied to all read operations on a specific data type. Below is +an example resolver for the class `Foo`: + +```java +import com.basho.riak.client.api.cap.ConflictResolver; + +public class FooResolver implements ConflictResolver<Foo> { + @Override + public Foo resolve(List<Foo> siblings) { + // Insert your sibling resolution logic here + } +} +``` + +What happens within the `resolve` method is up to you and will always +depend on the use case at hand. You can implement a resolver that +selects a random `Foo` from the list, chooses the `Foo` with the most +recent timestamp (if you've set up the class `Foo` to have timestamps), +etc. In this tutorial we'll provide a simple example to get you started. + +## Basic Conflict Resolution Example + +Let's say that we're building a social network application and storing +lists of usernames representing each user's "friends" in the network. +Each user will bear the class `User`, which we'll create below. All of +the data for our application will be stored in buckets that bear the +[bucket type]({{<baseurl>}}riak/kv/2.2.6/developing/usage/bucket-types) `siblings`, and for this bucket type +`allow_mult` is set to `true`, which means that Riak will generate +siblings in certain cases---siblings that our application will need to +be equipped to resolve when they arise. + +The question that we need to ask ourselves now is this: if a given user +has sibling values, i.e. if there are multiple `friends` lists and Riak +can't decide which one is most causally recent, which list should be +deemed "correct" from the standpoint of the application? What criteria +should be applied in making that decision? Should the lists be merged? +Should we pick a `User` object at random? + +This decision will always be yours to make. Here, though, we'll keep it +simple and say that the following criterion will hold: if conflicting +lists exist, _the longer list will be the one that our application deems +correct_. So if the user `user1234` has a sibling conflict where one +possible value has `friends` lists with 100, 75, and 10 friends, +respectively, the list of 100 friends will win out. While this might +not make sense in real-world applications, it's a good jumping-off +point. We'll explore the drawbacks of this approach, as well as a better +alternative, in this document as well. + +### Creating Our Data Class + +We'll start by creating a `User` class for each user's data. Each `User` +object will consist of a `username` as well as a `friends` property that +lists the usernames, as strings, of the user's friends. We'll use a +`Set` for the `friends` property to avoid duplicates. + +```java +public class User { + public String username; + public Set<String> friends; + + public User(String username, Set<String> friends) { + this.username = username; + this.friends = friends; + } +} +``` + +Here's an example of instantiating a new `User` object: + +```java +Set<String> friends = new HashSet<String>(); +friends.add("fred"); +friends.add("barney"); +User bashobunny = new User("bashobunny", friends); +``` + +### Implementing a Conflict Resolution Interface + +So what happens if siblings are present and the user `bashobunny` has +different friend lists in different object replicas? For that we can +implement the `ConflictResolver` class described [above](#how-the-java-client-handles-conflict-resolution). We +need to implement that interface in a way that is specific to the need +at hand, i.e. taking a list of `User` objects and returning the `User` +object that has the longest `friends` list: + +```java +import com.basho.riak.client.api.cap.ConflictResolver; + +public class UserResolver implements ConflictResolver<User> { + @Override + public User resolve(List<User> siblings) { + // If there are no objects present, return null + if (siblings.size == 0) { + return null; + // If there is only one User object present, return that object + } else if (siblings.size == 1) { + return siblings.get(0); + // And if there are multiple User objects, return the object + // with the longest list + } else { + int longestList = 0; + User userWithLongestList; + + // Iterate through the User objects to check for the longest + // list + for (User user : siblings) { + if (user.friends.size() > longestList) { + userWithLongestList = user; + longestList = user.friends.size(); + } + } + // If all sibling User objects have a friends list with a length + // of 0, it doesn't matter which sibling is selected, so we'll + // simply select the first one in the list: + return userWithLongestList == null ? siblings.get(0) : userWithLongestList; + } + } +} +``` + +### Registering a Conflict Resolver Class + +To use a conflict resolver, we must register it: + +```java +ConflictResolverFactory factory = ConflictResolverFactory.getInstance(); +factory.registerConflictResolver(User.class, new UserResolver()); +``` + +With the resolver registered, the resolution logic that we have created +will resolve siblings automatically upon read. Registering a custom +conflict resolver can occur at any point in the application's lifecycle +and will be applied on all reads that involve that object type. + +## Conflict Resolution and Writes + +In the above example, we created a conflict resolver that resolves a +list of discrepant `User` objects and returns a single `User`. It's +important to note, however, that this resolver will only provide the +application with a single "correct" value; it will _not_ write that +value back to Riak. That requires a separate step. When this step should +be undertaken depends on your application. In general, though, we +recommend writing objects to Riak only when the application is ready to +commit them, i.e. when all of the changes that need to be made to the +object have been made and the application is ready to persist the state +of the object in Riak. + +Correspondingly, we recommend that updates to objects in Riak follow +these steps: + +1. **Read** the object from Riak +2. **Resolving sibling conflicts** if they exist, allowing the +application to reason about one "correct" value for the object (this +step is the subject of this tutorial) +3. **Modify** the object +4. **Write** the object to Riak once the necessary changes have been +made + +You can find more on writing objects to Riak, including examples from +the official Java client library, in the [Developing with Riak KV: Usage]({{<baseurl>}}riak/kv/2.2.6/developing/usage) section. + +## More Advanced Example + +Resolving sibling `User` values on the basis of which user has the +longest `friends` list has the benefit of being simple but it's probably +not a good resolution strategy for our social networking application +because it means that unwanted data loss is inevitable. If one friends +list contains `A`, `B`, and `C` and the other contains `D` and `E`, the +list containing `A`, `B`, and `C` will be chosen. So what about friends +`D` and `E`? Those usernames are essentially lost. In the sections +below, we'll implement some other conflict resolution strategies as +examples. + +### Merging the Lists + +To avoid losing data like this, a better strategy may be to merge the +lists. We can modify our original `resolve` function in our +`UserResolver` to accomplish precisely that: + +```java +public class UserResolver implements ConflictResolver<User> { + @Override + public User resolve(List<User> siblings) { + // We apply the same logic as before, returning null if the + // key is empty and returning the one sibling if there is only + // one User in the siblings list + if (siblings.size == 0) { + return null; + } else if (siblings.size == 1) { + return siblings.get(0); + } else { + // We begin with an empty Set + Set<String> setBuilder = new HashSet<String>(); + + // We know that all User objects in the List will have the + // same username, since we used the username for the key, so + // we can fetch the username of any User in the list: + String username = siblings.get(0).username; + + // Now for each User object in the list we add the friends + // list to our empty Set + for (User user : siblings) { + setBuilder.addAll(user.friends); + } + + // Then we return a new User object that takes the Set we + // built as the friends list + return new User(username, setBuilder); + } + } +} +``` + +Since the `friends` list is a Java `Set`, we don't need to worry about +duplicate usernames. + +The drawback to this approach is the following: with a conflict +resolution strategy like this, it's more or less inevitable that a user +will remove a friend from their friends list, and that that friend will +end up back on the list during a conflict resolution operation. While +that's certainly not desirable, that is likely better than the +alternative proposed in the first example, which entails usernames being +simply dropped from friends lists. Sibling resolution strategies almost +always carry potential drawbacks of this sort. + +## Riak Data Types + +An important thing to always bear in mind when working with conflict +resolution is that Riak offers a variety of [Data Types]({{<baseurl>}}riak/kv/2.2.6/developing/data-types/) that have +specific conflict resolution mechanics built in. If you have data that +can be modeled as a [counter]({{<baseurl>}}riak/kv/2.2.6/developing/data-types/#counters), [set]({{<baseurl>}}riak/kv/2.2.6/developing/data-types/#sets), or [map]({{<baseurl>}}riak/kv/2.2.6/developing/data-types/#maps), then you should seriously +consider using those Data Types instead of creating your own +application-side resolution logic. + +In the example above, we were dealing with conflict resolution within a +set, in particular the `friends` list associated with each `User` +object. The merge operation that we built to handle conflict resolution +is analogous to the resolution logic that is built into Riak sets. For +more information on how you could potentially replace the client-side +resolution that we implemented above, see our [tutorial on Riak sets]({{<baseurl>}}riak/kv/2.2.6/developing/data-types/#sets). diff --git a/content/riak/kv/2.2.6/developing/usage/conflict-resolution/nodejs.md b/content/riak/kv/2.2.6/developing/usage/conflict-resolution/nodejs.md new file mode 100644 index 0000000000..0f72a6b9a7 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/usage/conflict-resolution/nodejs.md @@ -0,0 +1,58 @@ +--- +title_supertext: "Conflict Resolution:" +title: "NodeJS" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "NodeJS" + identifier: "usage_conflict_resolution_nodejs" + weight: 104 + parent: "usage_conflict_resolution" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/using/conflict-resolution/nodejs + - /riak-docs/riak/kv/2.2.6/dev/using/conflict-resolution/nodejs +--- + +For reasons explained in the [Introduction to conflict resolution]({{<baseurl>}}riak/kv/2.2.6/developing/usage/conflict-resolution), we strongly recommend adopting a conflict resolution strategy that +requires applications to resolve siblings according to use-case-specific +criteria. Here, we'll provide a brief guide to conflict resolution using the +official [Riak Node.js client](https://github.com/basho/riak-nodejs-client). + +## How the Node.js Client Handles Conflict Resolution + +In the Riak Node.js client, the result of a fetch can possibly return an array +of sibling objects. If there are no siblings, that property will return an +array with one value in it. + +[*Example:* creating object with siblings](https://github.com/basho/riak-nodejs-client-examples/blob/master/dev/using/conflict-resolution.js#L21-L68) + +So what happens if the length of `rslt.values` is greater than 1, as in the case +above? + +In order to resolve siblings, you need to either fetch, update and store a +canonical value, or choose a sibling from the `values` array and store that as +the canonical value. + +## Basic Conflict Resolution Example + +In this example, you will ignore the contents of the `values` array and will +fetch, update and store the definitive value. + +[*Example:* resolving siblings via store](https://github.com/basho/riak-nodejs-client-examples/blob/master/dev/using/conflict-resolution.js#L91-L111) + +### Choosing a value from `rslt.values` + +This example shows a basic sibling resolution strategy in which the first +sibling is chosen as the canonical value. + +[*Example:* resolving siblings via first](https://github.com/basho/riak-nodejs-client-examples/blob/master/dev/using/conflict-resolution.js#L113-L133) + +### Using `conflictResolver` + +This example shows a basic sibling resolution strategy in which the first +sibling is chosen as the canonical value via a conflict resolution function. + +[*Example:* resolving siblings via `conflictResolver](https://github.com/basho/riak-nodejs-client-examples/blob/master/dev/using/conflict-resolution.js#L135-L170) diff --git a/content/riak/kv/2.2.6/developing/usage/conflict-resolution/php.md b/content/riak/kv/2.2.6/developing/usage/conflict-resolution/php.md new file mode 100644 index 0000000000..7779b4c559 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/usage/conflict-resolution/php.md @@ -0,0 +1,240 @@ +--- +title_supertext: "Conflict Resolution:" +title: "PHP" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "PHP" + identifier: "usage_conflict_resolution_php" + weight: 105 + parent: "usage_conflict_resolution" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/using/conflict-resolution/php + - /riak-docs/riak/kv/2.2.6/dev/using/conflict-resolution/php +--- + +For reasons explained in the [Introduction to conflict resolution]({{<baseurl>}}riak/kv/2.2.6/developing/usage/conflict-resolution), we strongly recommend adopting a +conflict resolution strategy that requires applications to resolve +siblings according to use-case-specific criteria. Here, we'll provide a +brief guide to conflict resolution using the official [Riak PHP +client](https://github.com/basho/riak-php-client). + +## How the PHP Client Handles Conflict Resolution + +Every `\Basho\Riak\Object` command returns a `\Basho\Riak\Command\Object\Response` +object, which provides what is needed to handle object conflicts. If siblings exist +and have been returned from the server within the response body, they will be +available within the response object. See below: + +```php +$response = (new \Basho\Riak\Command\Builder\FetchObject($riak)) + ->buildLocation('conflicted_key', 'bucket_name', 'bucket_type') + ->build() + ->execute(); + +echo $response->getStatusCode(); // 300 +echo $response->hasSiblings(); // 1 +echo $response->getSiblings(); // \Basho\Riak\Object[] +``` + +## Basic Conflict Resolution Example + +Let's say that we're building a social network application and storing +lists of usernames representing each user's "friends" in the network. +Each user will bear the class `User`, which we'll create below. All of +the data for our application will be stored in buckets that bear the +[bucket type]({{<baseurl>}}riak/kv/2.2.6/developing/usage/bucket-types) `siblings`, and for this bucket type +`allow_mult` is set to `true`, which means that Riak will generate +siblings in certain cases---siblings that our application will need to +be equipped to resolve when they arise. + +The question that we need to ask ourselves now is this: if a given user +has sibling values, i.e. if there are multiple `friends` lists and Riak +can't decide which one is most causally recent, which list should be +deemed "correct" from the standpoint of the application? What criteria +should be applied in making that decision? Should the lists be merged? +Should we pick a `User` object at random? + +This decision will always be yours to make. Here, though, we'll keep it +simple and say that the following criterion will hold: if conflicting +lists exist, _the longer list will be the one that our application deems +correct_. So if the user `user1234` has a sibling conflict where one +possible value has `friends` lists with 100, 75, and 10 friends, +respectively, the list of 100 friends will win out. While this might +not make sense in real-world applications, it's a good jumping-off +point. We'll explore the drawbacks of this approach, as well as a better +alternative, in this document as well. + +### Creating Our Data Class + +We'll start by creating a `User` class for each user's data. Each `User` +object will consist of a `username` as well as a `friends` property that +lists the usernames, as strings, of the user's friends. We'll use a +`Set` for the `friends` property to avoid duplicates. + +```php +class User { + public $username; + public $friends; + + public function __construct($username, array $friends = []) + { + $this->username = $username; + $this->friends = $friends; + } + + public function __toString() + { + return json_encode([ + 'username' => $this->username, + 'friends' => $this->friends, + 'friends_count' => count($this->friends) + ]); + } +} +``` + +Here's an example of instantiating a new `User` object: + +```php +$bashobunny = new User('bashobunny', ['fred', 'barney']); +``` + +### Implementing a Conflict Resolution Function + +Let's say that we've stored a bunch of `User` objects in Riak and that a +few concurrent writes have led to siblings. How is our application going +to deal with that? First, let's say that there's a `User` object stored +in the bucket `users` (which is of the bucket type `siblings`, as +explained above) under the key `bashobunny`. We can fetch the object +that is stored there and see if it has siblings: + +```php +$response = (new \Basho\Riak\Command\Builder\FetchObject($riak)) + ->buildLocation('bashobunny', 'users', 'siblings') + ->build() + ->execute(); + +echo $response->hasSiblings(); // 1 +``` + +If we get `true`, then there are siblings. So what do we do in that +case? At this point, we need to write a function that resolves the list +of siblings, i.e. reduces the `$response->getSiblings()` array down to one member. +In our case, we need a function that takes a Riak response object as its argument, +applies some logic to the list of values contained in the `siblings` property +of the object, and returns a single value. For our example use case here, we'll +return the sibling with the longest `friends` list: + +```php +use \Basho\Riak; +use \Basho\Riak\Command; + +function longest_friends_list_resolver(Command\Object\Response $response) +{ + if ($response->hasSiblings()) { + $siblings = $response->getSiblings(); + $max_key = 0; + foreach ($siblings as $key => $sibling) { + if ($sibling->getData()['friends_count'] > $siblings[$max_key]->getData()['friends_count']) { + $max_key = $key; + } + } + } + + return $siblings[$max_key]; +} +``` + +We can then embed this function into a more general function for fetching +objects from the users bucket: + +```php +function fetch_user_by_username($username, Riak $riak) +{ + $response = (new Command\Builder\FetchObject($riak)) + ->buildLocation($username, 'users', 'siblings') + ->build() + ->execute(); + + return longest_friends_list_resolver($response); +} + +bashobunny = fetch_user_by_username('bashobunny', $riak); +``` + +Now, when a `User` object is fetched (assuming that the username acts as +a key for the object), a single value is returned for the `friends` +list. This means that our application can now use a "correct" value +instead of having to deal with multiple values. + +## Conflict Resolution and Writes + +In the above example, we created a conflict resolver that resolves a +list of discrepant `User` objects and returns a single `User`. It's +important to note, however, that this resolver will only provide the +application with a single "correct" value; it will _not_ write that +value back to Riak. That requires a separate step. When this step should +be undertaken depends on your application. In general, though, we +recommend writing objects to Riak only when the application is ready to +commit them, i.e. when all of the changes that need to be made to the +object have been made and the application is ready to persist the state +of the object in Riak. + +Correspondingly, we recommend that updates to objects in Riak follow +these steps: + +1. **Read** the object from Riak +2. **Resolving sibling conflicts** if they exist, allowing the +application to reason about one "correct" value for the object (this +step is the subject of this tutorial) +3. **Modify** the object +4. **Write** the object to Riak once the necessary changes have been +made + +You can find more on writing objects to Riak, including examples from +the official PHP client library, in the [Developing with Riak KV: Usage]({{<baseurl>}}riak/kv/2.2.6/developing/usage) section. + +## More Advanced Example + +Resolving sibling `User` values on the basis of which user has the longest +friends list has the benefit of being simple but it's probably not a +good resolution strategy for our social networking application because +it means that unwanted data loss is inevitable. If one friend list +contains `A`, `B`, and `C` and the other contains `D` and `E`, the list +containing `A`, `B`, and `C` will be chosen. So what about friends `D` +and `E`? Those usernames are essentially lost. In the sections below, +we'll implement an alternative strategy as an example. + +### Merging the Lists + +To avoid losing data like this, a better strategy would be to merge the +lists. We can modify our original resolver function to accomplish +precisely that and will also store the resulting `User` object. + +The drawback to this approach is that it's more or less inevitable that a user +will remove a friend from their friends list, and then that friend will +end up back on the list during a conflict resolution operation. While +that's certainly not desirable, that is likely better than the +alternative proposed in the first example, which entails usernames being +simply dropped from friends lists. Sibling resolution strategies almost +always carry potential drawbacks of this sort. + +## Riak Data Types + +An important thing to always bear in mind when working with conflict +resolution is that Riak offers a variety of [Data Types]({{<baseurl>}}riak/kv/2.2.6/developing/data-types/) that have +specific conflict resolution mechanics built in. If you have data that +can be modeled as a [counter]({{<baseurl>}}riak/kv/2.2.6/developing/data-types/#counters), [set]({{<baseurl>}}riak/kv/2.2.6/developing/data-types/#sets), or [map]({{<baseurl>}}riak/kv/2.2.6/developing/data-types/#maps), then you should seriously +consider using those Data Types instead of creating your own +application-side resolution logic. + +In the example above, we were dealing with conflict resolution within a +set, in particular the `friends` list associated with each `User` +object. The merge operation that we built to handle conflict resolution +is analogous to the resolution logic that is built into Riak sets. For +more information on how you could potentially replace the client-side +resolution that we implemented above, see our [tutorial on Riak sets]({{<baseurl>}}riak/kv/2.2.6/developing/data-types/#sets). diff --git a/content/riak/kv/2.2.6/developing/usage/conflict-resolution/python.md b/content/riak/kv/2.2.6/developing/usage/conflict-resolution/python.md new file mode 100644 index 0000000000..ee0323881b --- /dev/null +++ b/content/riak/kv/2.2.6/developing/usage/conflict-resolution/python.md @@ -0,0 +1,254 @@ +--- +title_supertext: "Conflict Resolution:" +title: "Python" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Python" + identifier: "usage_conflict_resolution_python" + weight: 102 + parent: "usage_conflict_resolution" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/using/conflict-resolution/python + - /riak-docs/riak/kv/2.2.6/dev/using/conflict-resolution/python +--- + +For reasons explained in the [Introduction to conflict resolution]({{<baseurl>}}riak/kv/2.2.6/developing/usage/conflict-resolution), we strongly recommend adopting a +conflict resolution strategy that requires applications to resolve +siblings according to use-case-specific criteria. Here, we'll provide a +brief guide to conflict resolution using the official [Riak Python +client](https://github.com/basho/riak-python-client). + +## How the Python Client Handles Conflict Resolution + +In the official Python client, every object of the `RiakObject` class +has a `siblings` property that provides access to a list of an object's +sibling values. If there are no siblings, that property will return a +list with only one item. Here's an example of an object with siblings: + +```python +bucket = client.bucket('seahawks') +obj = bucket.get('coach') +obj.siblings + +# The output: +[<riak.content.RiakContent object at 0x106cc51d0>, <riak.content.RiakContent object at 0x108x1da62c1>] +``` + +So what happens if the length of `obj.siblings` is greater than 1, as in +the case above? The easiest way to resolve siblings automatically with +the Python client is to create a conflict-resolving function that takes +a list of sibling values and returns a single value. Such resolution +functions can be registered either at the object level or the bucket +level. A more complete explanation can be found in the section directly +below. + +## Basic Conflict Resolution Example + +Let's say that we're building a social network application and storing +lists of usernames representing each user's "friends." Each user will +be of the class `User`, which we'll create below. All of the data for our +application will be stored in buckets that bear the [bucket type]({{<baseurl>}}riak/kv/2.2.6/developing/usage/bucket-types) `siblings`, and for this bucket type `allow_mult` is set +to `true`, which means that Riak will generate siblings in certain +cases---siblings that our application will need to be equipped to +resolve when necessary. + +The question that we need to ask ourselves at this point is the +following: if a given user has conflicting lists, which list should be +deemed more "correct?" What criteria should be applied? Should the lists +be merged? Should we pick a list at random and deem that list correct? +We'll keep it simple here and say that the following criterion will +hold: if multiple conflict lists exist, _the longer list will be the one +that our application deems correct_. While this might not make sense in +real-world applications, it's a good jumping-off point. + +### Creating Our Data Class + +We'll start by creating a `User` class for each user's data. Each `User` +object will consist of a `friends` property that lists the usernames, as +strings, of the user's friends. We will also create a `to_json` method, +as we'll be storing each `User` object as JSON: + +```python +class User(object): + def __init__(self, username, friends): + self.username = username + self.friends = friends + + def to_json(self): + return vars(self) +``` + +Now, we can create `User` objects and see what they look like as JSON: + +```python +new_user = User('riakuser127', ['captheorem', 'siblingsrule572']) + +new_user.to_json() +# {'username': 'riakuser127', 'friends': ['captheorem238', 'siblingsrule572']} +``` + +### Implementing and Registering a Conflict Resolution Function + +Let's say that we've stored a bunch of `User` objects in Riak and that a +few concurrent writes have led to siblings. How is our application going +to deal with that? First, let's say that there's a `User` object stored +in the bucket `users` (which is of the bucket type `siblings`, as +explained above) under the key `bashobunny`. We can fetch the object +that is stored there and see if it has siblings: + +```python +bucket = client.bucket_type('siblings').bucket('users') +obj = bucket.get('bashobunny') + +print len(obj.siblings) > 1 +``` + +If we get `True`, then there are siblings. So what do we do in that +case? The Python client allows us to write a conflict resolution hook +function that will be triggered any time siblings are found, i.e. any +time `len(obj.siblings) > 1`. A hook function like this needs to take a +single `RiakObject` object as its argument, apply some sort of logic to +the list of values contained in the `siblings` property, and ultimately +return a list with a single "correct" value. For our example case, we'll +return the value with the longest `friends` list: + +```python +def longest_friends_list_resolver(riak_object): + # We'll specify a lambda function that operates on the length of + # each sibling's "friends" list: + lm = lambda sibling: len(sibling.data['friends']) + # Then we'll return a list that contains only the object with the + # maximum value for the length of the "friends" list: + riak_object.siblings = [max(riak_object.siblings, key=lm), ] +``` + +### Registering a Conflict Resolver Function + +In the Python client, resolver functions can be registered at the object +level, as in this example: + +```python +bucket = client.bucket_type('siblings').bucket('users') +obj = RiakObject(client, bucket, 'bashobunny') +obj.resolver = longest_friends_list_resolver + +# Now, when the object is loaded from Riak, it will resolve to a single +# value instead of multiple values when both commands are executed: +obj.reload() +obj.store() +``` + +Alternatively, resolvers can be registered at the bucket level, so that +the resolution is applied to all objects in the bucket: + +```python +bucket = client.bucket_type('siblings').bucket('users') +bucket.resolver = longest_friends_list_resolver + +obj = RiakObject(client, bucket, 'bashobunny') +obj.reload() +obj.store() + +# The resolver will also be applied if you perform operations using the +# bucket object: + +bucket.get('bashobunny') +bucket.get('some_other_user') +``` + +## Conflict Resolution and Writes + +In the above example, we created a conflict resolver that resolves a +list of discrepant `User` object values and returns a single value. It's +important to note, however, that this resolver will only provide the +application with a single "correct" value; it will _not_ write that +value back to Riak. That requires a separate step. When this step should +be undertaken depends on your application. In general, though, we +recommend writing objects to Riak only when the application is ready to +commit them, i.e. when all of the changes that need to be made to the +object have been made and the application is ready to persist the state +of the object in Riak. + +Correspondingly, we recommend that updates to objects in Riak follow +these steps: + +1. **Read** the object from Riak +2. **Resolving sibling conflicts** if they exist, allowing the +application to reason about one "correct" value for the object (this +step is the subject of this tutorial) +3. **Modify** the object +4. **Write** the object to Riak once the necessary changes have been +made + +You can find more on writing objects to Riak, including code examples +from the official Python client library, in the [Developing with Riak KV: Usage]({{<baseurl>}}riak/kv/2.2.6/developing/usage) section. + +## More Advanced Example + +Resolving sibling `User` values on the basis of which user has the +longest `friends` list has the benefit of being simple but it's probably +not a good resolution strategy for our social networking application +because it means that unwanted data loss is inevitable. If one friend +list contains `A`, `B`, and `C` and the other contains `D` and `E`, the +list containing `A`, `B`, and `C` will be chosen. So what about friends +`D` and `E`? Those usernames are essentially lost. In the sections +below, we'll implement an alternative strategy as an example. + +### Merging the Lists + +To avoid losing data like this, a better strategy would be to merge the +lists. We can modify our original resolver function to accomplish +precisely that and will also store the resulting `User` object: + +```python +from riak.content import RiakContent + +def longest_friends_list_resolver(riak_object): + # We start with an empty set + friends_list = set() + + # Then we add all the friends from all siblings to the set + for user in riak_object.siblings: + friends_list.update(user.data['friends']) + + # Then we make a new User object. First, we fetch the username from + # any one of the siblings, then we pass in our new friends list. + username = riak_object.siblings[0].data['username'] + new_user = User(username, list(friends_list)) + + # Now we reuse the first sibling as a container for the merged data + riak_object.siblings[0].data = new_user.to_json() + + # And finally we set the siblings property to include just the + # single, resolved sibling + riak_object.siblings = [riak_object.siblings[0]] +``` + +The drawback to this approach is the following: with a conflict +resolution strategy like this, it's more or less inevitable that a user +will remove a friend from their friends list, and that that friend will +end up back on the list during a conflict resolution operation. While +that's certainly not desirable, that is likely better than the +alternative proposed in the first example, which entails usernames being +simply dropped from friends lists. Sibling resolution strategies almost +always carry potential drawbacks of this sort. + +## Riak Data Types + +An important thing to always bear in mind when working with conflict +resolution is that Riak offers a variety of [Data Types]({{<baseurl>}}riak/kv/2.2.6/developing/data-types/) that have +specific conflict resolution mechanics built in. If you have data that +can be modeled as a [counter]({{<baseurl>}}riak/kv/2.2.6/developing/data-types/#counters), [set]({{<baseurl>}}riak/kv/2.2.6/developing/data-types/#sets), or [map]({{<baseurl>}}riak/kv/2.2.6/developing/data-types/#maps), then you should seriously +consider using those Data Types instead of creating your own +application-side resolution logic. + +In the example above, we were dealing with conflict resolution within a +set, in particular the `friends` list associated with each `User` +object. The merge operation that we built to handle conflict resolution +is analogous to the resolution logic that is built into Riak sets. For +more information on how you could potentially replace the client-side +resolution that we implemented above, see our [tutorial on Riak sets]({{<baseurl>}}riak/kv/2.2.6/developing/data-types/#sets). diff --git a/content/riak/kv/2.2.6/developing/usage/conflict-resolution/ruby.md b/content/riak/kv/2.2.6/developing/usage/conflict-resolution/ruby.md new file mode 100644 index 0000000000..9d4d98a7f3 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/usage/conflict-resolution/ruby.md @@ -0,0 +1,250 @@ +--- +title_supertext: "Conflict Resolution:" +title: "Ruby" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Ruby" + identifier: "usage_conflict_resolution_ruby" + weight: 101 + parent: "usage_conflict_resolution" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/using/conflict-resolution/ruby + - /riak-docs/riak/kv/2.2.6/dev/using/conflict-resolution/ruby +--- + +For reasons explained in the [Introduction to conflict resolution]({{<baseurl>}}riak/kv/2.2.6/developing/usage/conflict-resolution), we strongly recommend adopting a +conflict resolution strategy that requires applications to resolve +siblings according to use-case-specific criteria. Here, we'll provide a +brief guide to conflict resolution using the official [Riak Ruby +client](https://github.com/basho/riak-ruby-client). + +## How the Ruby Client Handles Conflict Resolution + +In the official Ruby client, every Riak object has a `siblings` property +that provides access to a list of that object's sibling values. If there +are no siblings, that property will return an array with only one item. +Here's an example of an object with siblings: + +```ruby +bucket = client.bucket('seahawks') +obj = bucket.get('coach') +obj.siblings + +# The output: +[#<Riak::RContent [content/type]: "Jim Mora">, #<Riak::RContent [content/type]: "Pete Carroll">] +``` + +So what happens if the length of `obj.siblings` is greater than 1, as in +the case above? In order to resolve siblings, you need to create a +resolution function that takes a Riak object and reduces the `siblings` +array down to a single value. An example is provided in the section +below. + +## Basic Conflict Resolution Example + +Let's say that we're building a social network application and storing +lists of usernames representing each user's "friends." Each user will be +of the class `User`, which we'll create below. All of the data for our +application will be stored in buckets that bear the [bucket type]({{<baseurl>}}riak/kv/2.2.6/developing/usage/bucket-types) `siblings`, and for this bucket type `allow_mult` is set +to `true`, which means that Riak will generate siblings in certain +cases---siblings that our application will need to be equipped to +resolve when necessary. + +The question that we need to ask ourselves at this point is the +following: if a given user has conflicting lists, which list should be +deemed more "correct?" What criteria should be applied? Should the lists +be merged? Should we pick a list at random and deem that list correct? +We'll keep it simple here and say that the following criterion will +hold: if multiple conflict lists exist, _the longer list will be the one +that our application deems correct_. While this might not make sense in +real-world applications, it's a good jumping-off point. + +### Creating Our Data Class + +We'll start by creating a `User` class for each user's data. Each `User` +object will consist of a `username` and a `friends` property that lists +the usernames, as strings, of the user's friends. We will also create a +`to_json` method, as we'll be storing each `User` object as JSON: + +```ruby +class User + def initialize(username, friends) + @username = username + @friends = friends + end + + def to_json + { :username => @username, :friends => @friends } + end +end +``` + +Now, we can create `User` objects and see what they look like as JSON: + +```ruby +new_user = User.new('riakuser127', ['captheorem238', 'siblingsrule572']) + +new_user.to_json +# {'username': 'riakuser127', 'friends': ['captheorem238', 'siblingsrule572']} +``` + +### Implementing a Conflict Resolution Function + +Let's say that we've stored a bunch of `User` objects in Riak and that a +few concurrent writes have led to siblings. How is our application going +to deal with that? First, let's say that there's a `User` object stored +in the bucket `users` (which is of the bucket type `siblings`, as +explained above) under the key `bashobunny`. We can fetch the object +that is stored there and see if it has siblings: + +```ruby +bucket = client.bucket('users') +obj = bucket.get('bashobunny', type: 'siblings') +p obj.siblings.length > 1 +``` + +If we get `true`, then there are siblings. So what do we do in that +case? At this point, we need to write a function that resolves the list +of siblings, i.e. reduces the `obj.siblings` array down to one member. +In our case, we need a function that takes a single Riak object (or +`RObject` in the Ruby client) as its argument, applies some logic to the +list of values contained in the `siblings` property of the object, and +returns a single value. For our example use case here, we'll return the +sibling with the longest `friends` list: + +```ruby +def longest_friends_list_resolver(riak_object) + # The "conflict?" method is built into the Ruby client + if riak_object.conflict? + # The "max_by" method enables us to select the sibling with the + # longest "friends" list + riak_object.siblings.max_by{ |user| user.data['friends'].length } + else + # If there are no siblings, we can simply return the object's + # "content" as is + riak_object.content + end +end +``` + +We can then embed this function into a more general function for +fetching objects from the `users` bucket: + +```ruby +def fetch_user_by_username(username) + bucket = client.bucket('users') + user_object = bucket.get(username) + longest_friends_list_resolve(user_object) + user_object +end + +bashobunny = fetch_user_by_username('bashobunny') +``` + +Now, when a `User` object is fetched (assuming that the username acts as +a key for the object), a single value is returned for the `friends` +list. This means that our application can now use a "correct" value +instead of having to deal with multiple values. + +## Conflict Resolution and Writes + +In the above example, we created a conflict resolver that resolves a +list of discrepant `User` objects and returns a single `User`. It's +important to note, however, that this resolver will only provide the +application with a single "correct" value; it will _not_ write that +value back to Riak. That requires a separate step. When this step should +be undertaken depends on your application. In general, though, we +recommend writing objects to Riak only when the application is ready to +commit them, i.e. when all of the changes that need to be made to the +object have been made and the application is ready to persist the state +of the object in Riak. + +Correspondingly, we recommend that updates to objects in Riak follow +these steps: + +1. **Read** the object from Riak +2. **Resolving sibling conflicts** if they exist, allowing the +application to reason about one "correct" value for the object (this +step is the subject of this tutorial) +3. **Modify** the object +4. **Write** the object to Riak once the necessary changes have been +made + +You can find more on writing objects to Riak, including examples from +the official Ruby client library, in the [Developing with Riak KV: Usage]({{<baseurl>}}riak/kv/2.2.6/developing/usage) section. + +## More Advanced Example + +Resolving sibling User values on the basis of which user has the longest +friends list has the benefit of being simple but it's probably not a +good resolution strategy for our social networking application because +it means that unwanted data loss is inevitable. If one friend list +contains `A`, `B`, and `C` and the other contains `D` and `E`, the list +containing `A`, `B`, and `C` will be chosen. So what about friends `D` +and `E`? Those usernames are essentially lost. In the sections below, +we'll implement an alternative strategy as an example. + +### Merging the Lists + +To avoid losing data like this, a better strategy would be to merge the +lists. We can modify our original resolver function to accomplish +precisely that and will also store the resulting `User` object: + +```ruby +def longest_friends_list_resolver(riak_object) + # An empty array for use later on + friends_list = [] + if riak_object.conflict? + # The "friends" arrays for all siblings will be merged into one + # array + riak_object.siblings.each do |sibling| + friends_list.push(sibling.data['friends']) + end + + # Then we make a new User object. First, we fetch the username from + # any one of the siblings, then we pass in our new friends list, + # calling the "uniq" method to eliminate duplicate usernames. + username = riak_object.siblings[0].data['username'] + new_user = User.new(username, friends_list.uniq) + + # Now we reuse the first sibling as a container for the merged data + riak_object.siblings[0].data = new_user.to_json + + # And finally we set the siblings property to include just the + # single, resolved sibling + riak_object.siblings = [riak_object.siblings[0]] + else + riak_object.content + end +end +``` + +The drawback to this approach is the following: with a conflict +resolution strategy like this, it's more or less inevitable that a user +will remove a friend from their friends list, and that that friend will +end up back on the list during a conflict resolution operation. While +that's certainly not desirable, that is likely better than the +alternative proposed in the first example, which entails usernames being +simply dropped from friends lists. Sibling resolution strategies almost +always carry potential drawbacks of this sort. + +## Riak Data Types + +An important thing to always bear in mind when working with conflict +resolution is that Riak offers a variety of [Data Types]({{<baseurl>}}riak/kv/2.2.6/developing/data-types/) that have +specific conflict resolution mechanics built in. If you have data that +can be modeled as a [counter]({{<baseurl>}}riak/kv/2.2.6/developing/data-types/#counters), [set]({{<baseurl>}}riak/kv/2.2.6/developing/data-types/#sets), or [map]({{<baseurl>}}riak/kv/2.2.6/developing/data-types/#maps), then you should seriously +consider using those Data Types instead of creating your own +application-side resolution logic. + +In the example above, we were dealing with conflict resolution within a +set, in particular the `friends` list associated with each `User` + +object. The merge operation that we built to handle conflict resolution +is analogous to the resolution logic that is built into Riak sets. For +more information on how you could potentially replace the client-side +resolution that we implemented above, see our [tutorial on Riak sets]({{<baseurl>}}riak/kv/2.2.6/developing/data-types/#sets). diff --git a/content/riak/kv/2.2.6/developing/usage/content-types.md b/content/riak/kv/2.2.6/developing/usage/content-types.md new file mode 100644 index 0000000000..0b6ececb70 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/usage/content-types.md @@ -0,0 +1,187 @@ +--- +title: "Content Types" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Content Types" + identifier: "usage_content_types" + weight: 104 + parent: "developing_usage" +toc: true +--- + +Riak KV is a fundamentally content-agnostic database. You can use it to +store anything you want, from JSON to XML to HTML to binaries to images +and beyond. It's important to note that _all_ objects stored in +Riak need a specified content type. If you don't specify a +content type, the reaction will vary based on your client library: + +```java +// In the Java client, the response when storing an object without +// specifying a content type will depend on what is being stored. If you +// store a Java Map, for example, the client will automatically specify +// that the object is "application/json"; if you store a String, the +// client will specify "application/x-www-form-urlencoded"; POJOs are +// stored as JSON by default, and so on. +``` + +```ruby +# In the Ruby client, you must always specify a content type. If you +# you don't, you'll see the following error: +ArgumentError: content_type is not defined! +``` + +```php +# PHP will default to cURLs default content-type for POST & PUT requests: +# application/x-www-form-urlencoded + +# If you use the StoreObject::buildJsonObject() method when building your command, +# it will store the item with application/json as the content-type +``` + +```python +# In the Python client, the default content type is "application/json". +# Because of this, you should always make sure to specify the content +# type when storing other types of data. +``` + +```csharp +// Using the Riak .NET Client, the response when storing an object without +// specifying a content type will depend on what is being stored. +// If you store a Dictionary, for example, the client will +// automatically specify that the object is "application/json"; +// POCOs are stored as JSON by default, and so on. +``` + +```javascript +// In the Node.js client, the default content type is "application/json". +// Because of this, you should always make sure to specify the content +// type when storing other types of data. +``` + +```erlang +%% In the Erlang client, the response when storing an object without +%% specify8ing a content type will depend on what is being stored. If +%% you store a simple binary, for example, the client will automatically +%% specify that the object is "application/octet-stream"; if you store a +%% string, the client will specify "application/x-erlang-binary"; and so +%% on. +``` + +```golang +// In the Go client, you must always specify a content type. +``` + +Because content type negotiation varies so widely from client to client, +we recommend consulting the documentation for your preferred client for +more information. + +## Specifying Content Type + +For all writes to Riak, you will need to specify a content type, for +example `text/plain` or `application/json`. + +```java +Location wildeGeniusQuote = new Location(new Namespace("quotes", "oscar_wilde"), "genius"); +BinaryValue text = BinaryValue.create("I have nothing to declare but my genius"); +RiakObject obj = new RiakObject() + .setContentType("text/plain") + .setValue(text); +StoreValue store = new StoreValue.Builder(myKey, obj) + .build(); +client.execute(store); +``` + +```ruby +bucket = client.bucket_type('quotes').bucket('oscar_wilde') +obj = Riak::RObject.new(bucket, 'genius') +obj.content_type = 'text/plain' +obj.raw_data = 'I have nothing to declare but my genius' +obj.store +``` + +```php +$response = (new \Basho\Riak\Command\Builder\StoreObject($riak)) + ->buildLocation('genius', 'oscar_wilde', 'quotes') + ->buildObject('I have nothing to declare but my genius!', 'text/plain') + ->build() + ->execute(); +``` + +```python +bucket = client.bucket_type('quotes').bucket('oscar_wilde') +obj = RiakObject(client, bucket, 'genius') +obj.content_type = 'text/plain' +obj.data = 'I have nothing to declare but my genius' +obj.store() +``` + +```csharp +var id = new RiakObjectId("quotes", "oscar_wilde", "genius"); +var obj = new RiakObject(id, "I have nothing to declare but my genius", + RiakConstants.ContentTypes.TextPlain); +var rslt = client.Put(obj); +``` + +```javascript +var riakObj = new Riak.Commands.KV.RiakObject(); +riakObj.setContentType('text/plain'); +riakObj.setValue('I have nothing to declare but my genius'); +client.storeValue({ + bucketType: 'quotes', bucket: 'oscar_wilde', key: 'genius', + value: riakObj +}, function (err, rslt) { + if (err) { + throw new Error(err); + } +}); +``` + +```erlang +Object = riakc_obj:new({<<"quotes">>, <<"oscar_wilde">>}, + <<"genius">>, + <<"I have nothing to declare but my genius">>, + <<"text/plain">>). +riakc_pb_socket:put(Pid, Object). +``` + +```golang +obj := &riak.Object{ + ContentType: "text/plain", + Charset: "utf-8", + ContentEncoding: "utf-8", + Value: []byte("I have nothing to declare but my genius"), +} + +cmd, err := riak.NewStoreValueCommandBuilder(). + WithBucketType("quotes"). + WithBucket("oscar_wilde"). + WithKey("genius"). + WithContent(obj). + Build() + +if err != nil { + fmt.Println(err.Error()) + return +} + +if err := cluster.Execute(cmd); err != nil { + fmt.Println(err.Error()) + return +} + +svc := cmd.(*riak.StoreValueCommand) +rsp := svc.Response +``` + +```curl +curl -XPUT \ + -H "Content-Type: text/plain" \ + -d "I have nothing to declare but my genius" \ + http://localhost:8098/types/quotes/buckets/oscar_wilde/keys/genius + +# Please note that POST is also a valid method for writes, for the sake +# of compatibility +``` diff --git a/content/riak/kv/2.2.6/developing/usage/creating-objects.md b/content/riak/kv/2.2.6/developing/usage/creating-objects.md new file mode 100644 index 0000000000..9aa55b1f67 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/usage/creating-objects.md @@ -0,0 +1,550 @@ +--- +title: "Creating Objects in Riak KV" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Creating Objects" + identifier: "usage_creating_objects" + weight: 100 + parent: "developing_usage" +toc: true +--- + +[usage content types]: {{<baseurl>}}riak/kv/2.2.6/developing/usage/content-types + +Writes in Riak KV (storing or modifying objects) are like HTTP `PUT` +requests. Here is the basic form of writes: + +``` +PUT /types/<type>/buckets/<bucket>/keys/<key> + +# If you're using HTTP to interact with Riak, you can also use POST +``` + +As an example, let's store an object containing information about a dog named Rufus. We'll store that object in the key `rufus` in the bucket `dogs`, which bears the `animals` [bucket type]({{<baseurl>}}riak/kv/2.2.6/using/cluster-operations/bucket-types). + +The object we're storing will be very simple, just a basic text snippet +of something that Rufus might say. Let's build the object and then store +it. + +``` java +String quote = "WOOF!"; +Namespace bucket = new Namespace("animals", "dogs"); +Location rufusLocation = new Location(bucket, "rufus"); +RiakObject rufusObject = new RiakObject() + .setContentType("text/plain") + .setValue(BinaryValue.create(quote)); +StoreValue storeOp = new StoreValue.Builder(rufusObject) + .withLocation(rufusLocation) + .build(); +client.execute(storeOp); +``` + +``` ruby +bucket = client.bucket_type('animals').bucket('dogs') +obj = Riak::RObject.new(bucket, 'rufus') +obj.content_type = 'text/plain' +obj.data = 'WOOF!' +obj.store +``` + +``` php +$response = (new \Basho\Riak\Command\Builder\StoreObject($riak)) + ->buildLocation('rufus', 'users', 'animals') + ->buildObject('WOOF!', 'text/plain') + ->build() + ->execute(); +``` + +``` python +bucket = client.bucket_type('animals').bucket('dogs') +obj = RiakObject(client, bucket, 'rufus') +obj.content_type = 'text/plain' +obj.data = 'WOOF!' +obj.store() +``` + +``` csharp +var id = new RiakObjectId("animals", "dogs", "rufus") +var obj = new RiakObject(id, "WOOF!", "text/plain"); +var result = client.Put(obj); +``` + +``` javascript +var riakObj = new Riak.Commands.KV.RiakObject(); +riakObj.setContentType('text/plain'); +riakObj.setValue('WOOF!'); +client.storeValue({ + bucketType: 'animals', bucket: 'dogs', key: 'rufus', + value: riakObj +}, function (err, rslt) { + if (err) { + throw new Error(err); + } +}); +``` + +``` golang +obj := &riak.Object{ + ContentType: "text/plain", + Charset: "utf-8", + ContentEncoding: "utf-8", + Value: []byte("WOOF!"), +} + +cmd, err := riak.NewStoreValueCommandBuilder(). + WithBucketType("animals"). + WithBucket("dogs"). + WithKey("rufus"). + WithContent(obj). + Build() + +if err != nil { + fmt.Println(err.Error()) + return +} + +if err := cluster.Execute(cmd); err != nil { + fmt.Println(err.Error()) + return +} + +svc := cmd.(*riak.StoreValueCommand) +rsp := svc.Response +``` + +Notice that we specified both a value for the object, i.e. `WOOF!`, and +a content type, `text/plain`. See [content types][usage content types] for more information. + +Now, you run the same read operation as in [Reading Objects]({{<baseurl>}}riak/kv/2.2.6/developing/usage/reading-objects). If the write operation was successful, you should be able to successfully read the object. Please note that the operation will fail if you don't first create the bucket-type `animals` as per the page on [bucket types]({{<baseurl>}}riak/kv/2.2.6/using/cluster-operations/bucket-types). + +### Store an Object + +Your application will often have its own method of generating the keys +for its data, e.g. on the basis of timestamps. If so, storing that data +is easy. The basic request looks like this. + +``` +PUT /types/TYPE/buckets/BUCKET/keys/KEY + +# If you're using HTTP, POST can be used instead of PUT. The only +# difference between POST and PUT is that you should POST in cases where +# you want Riak to auto-generate a key. More on this can be found in the +# examples below. +``` + +There is no need to intentionally create buckets in Riak. They pop into +existence when keys are added to them, and disappear when all keys have +been removed from them. If you don't specify a bucket's type, the type +[`default`]({{<baseurl>}}riak/kv/2.2.6/developing/usage/bucket-types) will be applied. + +#### Write Parameters + +Write requests support the following parameters: + +Parameter | Default | Description +:---------|:--------|:----------- +`w` | `quorum` | How many replicas to write to before returning a successful response +`pw` | `0` | How many primary vnodes must respond for a write to be deemed successful +`dw` | `quorum` | How many replicas to commit to durable storage before returning a successful response +`returnbody` | `false` | Whether to return the contents of the stored object + +Here is an example of storing an object (another brief text snippet) +under the key `viper` in the bucket `dodge`, which bears the type +`cars`, with `w` set to `3`: + +```java +Location viperKey = new Location(new Namespace("cars", "dodge"), "viper"); +BinaryValue text = BinaryValue.create("vroom"); +RiakObject obj = new RiakObject() + .setContentType("text/plain") + .setValue(text); +StoreValue store = new StoreValue.Builder(myKey, obj) + .withOption(StoreOption.W, new Quorum(3)) + .build(); +client.execute(store); +``` + +```ruby +bucket = client.bucket_type('cars').bucket('dodge') +obj = Riak::RObject.new(bucket, 'viper') +obj.content_type = 'text/plain' +obj.raw_data = 'vroom' +obj.store(w: 3) +``` + +```php +(new \Basho\Riak\Command\Builder\StoreObject($riak)) + ->buildLocation('viper', 'dodge', 'cars') + ->buildObject('vroom', 'text/plain') + ->withParameter('w', 3) + ->build() + ->execute(); +``` + +```python +bucket = client.bucket_type('cars').bucket('dodge') +obj = RiakObject(client, bucket, 'viper') +obj.content_type = 'text/plain' +obj.data = 'vroom' +obj.store(w=3) +``` + +```csharp +var id = new RiakObjectId("cars", "dodge", "viper"); +var obj = new RiakObject(id, "vroom", "text/plain"); +var options = new RiakPutOptions(); +options.SetW(new Quorum(3)); +var result = client.Put(obj, options); +``` + +```javascript +var riakObj = new Riak.Commands.KV.RiakObject(); +riakObj.setContentType('text/plain'); +riakObj.setValue('vroom'); + +var options = { + bucketType: 'cars', bucket: 'dodge', key: 'viper', + w: 3, value: riakObj +}; +client.storeValue(options, function (err, rslt) { + if (err) { + throw new Error(err); + } +}); +``` + +```erlang +Object = riakc_obj:new({<<"cars">>, <<"dodge">>}, + <<"viper">>, + <<"vroom">>, + <<"text/plain">>, + [{w, 3}]). +riakc_pb_socket:put(Pid, Object). +``` + +```golang +obj := &riak.Object{ + ContentType: "text/plain", + Charset: "utf-8", + ContentEncoding: "utf-8", + Value: []byte("vroom"), +} + +cmd, err := riak.NewStoreValueCommandBuilder(). + WithBucketType("cars"). + WithBucket("dodge"). + WithKey("viper"). + WithW(3). + WithContent(obj). + Build() + +if err != nil { + fmt.Println(err.Error()) + return +} + +if err := cluster.Execute(cmd); err != nil { + fmt.Println(err.Error()) + return +} +``` + +```curl +curl -XPUT \ + -H "Content-Type: text/plain" \ + -d "vroom" \ + http://localhost:8098/types/cars/buckets/dodge/keys/viper?w=3 +``` + +Again, the above will only work if the `cars` bucket type has been created and activated. + +#### Return Body + +If `returnbody` is set to `true`, any of the response headers expected +from a read request may be present. Like a `GET` request, `300 Multiple +Choices` may be returned if siblings existed or were created as part of +the operation, and the response can be dealt with similarly. + +Normal HTTP status codes (responses will vary for client libraries): + +* `200 OK` +* `204 No Content` +* `300 Multiple Choices` + +For example, using the same object from above: + +```java +Location viperKey = new Location(new Namespace("cars", "dodge"), "viper"); +BinaryValue text = BinaryValue.create("vroom"); +RiakObject obj = new RiakObject() + .setContentType("text/plain") + .setValue(text); +StoreValue store = new StoreValue.Builder(myKey, obj) + .withOption(StoreOption.W, new Quorum(3)) + .withOption(StoreOption.RETURN_BODY, true) + .build(); +client.execute(store); +``` + +```ruby +bucket = client.bucket_type('cars').bucket('dodge') +obj = Riak::RObject.new(bucket, 'viper') +obj.content_type = 'text/plain' +obj.raw_data = 'vroom' +obj.store(w: 3, returnbody: true) +``` + +```php +(new \Basho\Riak\Command\Builder\StoreObject($riak)) + ->buildLocation('viper', 'dodge', 'cars') + ->buildObject('vroom', 'text/plain') + ->withParameter('w', 3) + ->withParameter('returnbody', 'true') + ->build() + ->execute(); +``` + +```python +bucket = client.bucket_type('cars').bucket('dodge') +obj = RiakObject(client, bucket, 'viper') +obj.content_type = 'text/plain' +obj.data = 'vroom' +obj.store(w=3, return_body=True) +``` + +```csharp +var id = new RiakObjectId("cars", "dodge", "viper"); +var obj = new RiakObject(id, "vroom", "text/plain"); +var options = new RiakPutOptions(); +options.SetW(new Quorum(3)); +options.SetReturnBody(true); +var result = client.Put(obj, options); +``` + +```javascript +var riakObj = new Riak.Commands.KV.RiakObject(); +riakObj.setContentType('text/plain'); +riakObj.setValue('vroom'); + +var options = { + bucketType: 'cars', bucket: 'dodge', key: 'viper', + w: 3, returnBody: true, value: riakObj +}; +client.storeValue(options, function (err, rslt) { + if (err) { + throw new Error(err); + } + var riakObj = rslt.values.shift(); + var viper = riakObj.value; + logger.info("dodge viper: %s", viper.toString('utf8')); +}); +``` + +```erlang +Object = riakc_obj:new({<<"cars">>, <<"dodge">>}, + <<"viper">>, + <<"vroom">>, + <<"text/plain">>). +riakc_pb_socket:put(Pid, Object, [return_body]). +``` + +```golang +obj := &riak.Object{ + ContentType: "text/plain", + Charset: "utf-8", + ContentEncoding: "utf-8", + Value: []byte("vroom"), +} + +cmd, err := riak.NewStoreValueCommandBuilder(). + WithBucketType("cars"). + WithBucket("dodge"). + WithKey("viper"). + WithW(3). + WithContent(obj). + WithReturnBody(true). + Build() + +if err != nil { + fmt.Println(err.Error()) + return +} + +if err := cluster.Execute(cmd); err != nil { + fmt.Println(err.Error()) + return +} +``` + +```curl +curl -XPUT \ + -H "Content-Type: text/plain" \ + -d "vroom" \ + http://localhost:8098/types/cars/buckets/dodge/keys/viper?w=3&returnbody=true +``` + +### Store a New Object and Assign a Random Key + +If your application would rather leave key-generation up to Riak, issue +a `POST` request to the bucket URL instead of a PUT to a bucket/key +pair: + +``` +POST /types/TYPE/buckets/BUCKET/keys +``` + +If you don't pass Riak a `key` name after the bucket, it will know to +create one for you. + +Supported headers are the same as for bucket/key write requests, though +`X-Riak-Vclock` will never be relevant for these POST requests. +Supported query parameters are also the same as for bucket/key PUT +requests. + +Normal status codes: + +* `201 Created` + +This command will store an object in the bucket `random_user_keys`, +which bears the bucket type `users`. + +```java +Namespace locationWithoutKey = new Namespace("users", "random_user_keys"); +BinaryValue text = BinaryValue.create("{'user':'data'}"); +RiakObject obj = new RiakObject() + .setContentType("application/json") + .setValue(text); +StoreValue store = new StoreValue.Builder(locationWithoutKey, obj) + .build(); +String key = client.execute(store).getLocation().getKeyAsString(); + +// The Java client will assign a random key along the following lines: +"ZPFF18PUqGW9efVou7EHhfE6h8a" +``` + +```ruby +bucket = client.bucket_type('users').bucket('random_user_keys') +obj = Riak::RObject.new(bucket) +obj.content_type = 'application/json' +obj.raw_data = '{"user":"data"}' + +obj.store + +# The client will assign a key like the following: +obj.key +"GB8fW6DDZtXogK19OLmaJf247DN" +``` + +```php +$response = (new \Basho\Riak\Command\Builder\StoreObject($riak)) + ->buildBucket('random_user_keys', 'users') + ->buildJsonObject(['user'=>'data']) + ->build() + ->execute(); + +echo $response->getLocation()->getKey(); // GB8fW6DDZtXogK19OLmaJf247DN +``` + +```python +bucket = client.bucket_type('users').bucket('random_user_keys') +obj = RiakObject(client, bucket) +obj.content_type = 'application/json' +obj.data = '{"user":"data"}' +obj.store() + +obj.key + +# The Python client will assign a random key along the following lines: +'ZPFF18PUqGW9efVou7EHhfE6h8a' +``` + +```csharp +var id = new RiakObjectId("users", "random_user_keys", null); +var obj = new RiakObject(id, @"{'user':'data'}", + RiakConstants.ContentTypes.ApplicationJson); +var rslt = client.Put(obj); +Debug.WriteLine(format: "Generated key: {0}", args: rslt.Value.Key); + +// The .NET client will output a random key similar to this: +// Generated key: DWDsnpYSqOU363c0Bqe8hCwAM7Q +``` + +```javascript +var user = { + user: 'data' +}; +var options = { + bucketType: 'users', bucket: 'random_user_keys', + returnBody: true, value: user +}; +client.storeValue(options, function (err, rslt) { + if (err) { + throw new Error(err); + } + var riakObj = rslt.values.shift(); + var generatedKey = riakObj.getKey(); + logger.info("Generated key: %s", generatedKey); +}); + +// The Node.js client will output a random key similar to this: +// info: Generated key: VBAMoX0OOucymVCxeQEYzLzzAh2 +``` + +```erlang +Object = riakc_obj:new({<<"users">>, <<"random_user_keys">>}, undefined, <<"{'user':'data'}">>, <<"application/json">>). +riakc_pb_socket:put(Pid, Object). + +%% The key can be retrieved from the output of the above call. +%% It will look something like this: + +{ok,{riakc_obj,{<<"users">>,<<"random_user_keys">>}, + <<"EZ7pp4bpdfpZw0fPUdTUafveQjO">>,undefined,[],undefined, + undefined}} +``` + +```golang +obj := &riak.Object{ + ContentType: "application/json", + Charset: "utf-8", + ContentEncoding: "utf-8", + Value: []byte("{'user':'data'}"), +} + +cmd, err := riak.NewStoreValueCommandBuilder(). + WithBucketType("users"). + WithBucket("random_user_keys"). + WithContent(obj). + Build() + +if err != nil { + fmt.Println(err.Error()) + return +} + +if err := cluster.Execute(cmd); err != nil { + fmt.Println(err.Error()) + return +} + +svc := cmd.(*riak.StoreValueCommand) +rsp := svc.Response +fmt.Printf("Generated key: %v\n", rsp.GeneratedKey) + +// Output: +// Generated key: QSHkZjFdWwfrxtKl3wtUhL2gz7N +``` + +```curl +curl -i -XPOST \ + -H "Content-Type: text/plain" \ + -d "this is a test" \ + http://localhost:8098/types/users/buckets/random_user_keys/keys + +# In the output, you should see a Location header that will give you the +# location of the object in Riak, with the key at the end: + +Location: /buckets/test/keys/G7FYUXtTsEdru4NP32eijMIRK3o +``` diff --git a/content/riak/kv/2.2.6/developing/usage/custom-extractors.md b/content/riak/kv/2.2.6/developing/usage/custom-extractors.md new file mode 100644 index 0000000000..c6baedcded --- /dev/null +++ b/content/riak/kv/2.2.6/developing/usage/custom-extractors.md @@ -0,0 +1,420 @@ +--- +title: "Custom Extractors" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Custom Extractors" + identifier: "usage_custom_extractors" + weight: 113 + parent: "developing_usage" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/search/custom-extractors + - /riak-docs/riak/kv/2.2.6/dev/search/custom-extractors +--- + +Solr, and by extension Riak Search, has default extractors for a wide +variety of data types, including JSON, XML, and plaintext. Riak Search +ships with the following extractors: + +Content Type | Erlang Module +:------------|:------------- +`application/json` | `yz_json_extractor` +`application/xml` | `yz_xml_extractor` +`text/plain` | `yz_text_extractor` +`text/xml` | `yz_xml_extractor` +No specified type | `yz_noop_extractor` + +There are also built-in extractors for [Riak Data Types]({{<baseurl>}}riak/kv/2.2.6/developing/usage/searching-data-types). + +If you're working with a data format that does not have a default Solr +extractor, you can create your own and register it with Riak Search. +We'll show you how to do so by way of example. + +## The Extractor Interface + +Creating a custom extract involves creating an Erlang interface that +implements two functions: + +* `extract/1` --- Takes the contents of the object and calls `extract/2` + with the same contents and an empty list +* `extract/2` --- Takes the contents of the object and returns an Erlang + [proplist](http://www.erlang.org/doc/man/proplists.html) with a + single field name and a single value associated with that name + +The following extractor shows how a pure text extractor implements those +two functions: + +```erlang +-module(search_test_extractor). +-include("yokozuna.hrl"). +-compile(export_all). + +extract(Value) -> + extract(Value, []). + +extract(Value, Opts) -> + FieldName = field_name(Opts), + [{FieldName, Value}]. + +-spec field_name(proplist()) -> any(). +field_name(Opts) -> + proplists:get_value(field_name, Opts, text). +``` + +This extractor takes the contents of a `Value` and returns a proplist +with a single field name (in this case `text`) and the single value. +This function can be run in the Erlang shell. Let's run it providing the +text `hello`: + +```erlang +> c(search_test_extractor). +%% {ok, search_test_extractor} + +> search_test_extractor:extract("hello"). + +%% Console output: +[{text, "hello"}] +``` + +Upon running this command, the value `hello` would be indexed in Solr +under the fieldname `text`. If you wanted to find all objects with a +`text` field that begins with `Fourscore`, you could use the +Solr query `text:Fourscore*`, to give just one example. + +## An Example Custom Extractor + +Let's say that we're storing HTTP header packet data in Riak. Here's an +example of such a packet: + +``` +GET http://www.google.com HTTP/1.1 +``` + +We want to register the following information in Solr: + +Field name | Value | Extracted value in this example +:----------|:------|:------------------------------- +`method` | The HTTP method | `GET` +`host` | The URL's host | `www.google.com` +`uri` | The URI, i.e. what comes after the host | `/` + +The example extractor below would provide the three desired +fields/values. It relies on the +[`decode_packet`](http://www.erlang.org/doc/man/erlang.html#decode_packet-3) +function from Erlang's standard library. + +```erlang +-module(yz_httpheader_extractor). +-compile(export_all). + +extract(Value) -> + extract(Value, []). + +%% In this example, we can ignore the Opts variable from the example +%% above, hence the underscore: +extract(Value, _Opts) -> + {ok, + {http_request, + Method, + {absoluteURI, http, Host, undefined, Uri}, + _Version}, + _Rest} = erlang:decode_packet(http, Value, []), + [{method, Method}, {host, list_to_binary(Host)}, {uri, list_to_binary(Uri)}]. +``` + +This file will be stored in a `yz_httpheader_extractor.erl` file (as +Erlang filenames must match the module name). Now that our extractor has +been written, it must be compiled and registered in Riak before it can +be used. + +## Registering Custom Extractors + +In order to use a custom extractor, you must create a compiled `.beam` +file out of your `.erl` extractor file and then tell Riak where that +file is located. Let's say that we have created a +`search_test_extractor.erl` file in the directory `/opt/beams`. First, +we need to compile that file: + +```bash +erlc search_test_extractor.erl +``` + +To instruct Riak where to find the resulting +`search_test_extractor.beam` file, we'll need to add a line to an +`advanced.config` file in the node's `/etc` directory (more information +can be found in our documentation on [advanced]({{<baseurl>}}riak/kv/2.2.6/configuring/reference/#advanced-configuration)). Here's an +example: + +```advancedconfig +[ + %% Other configs + {vm_args, [ + {"-pa /opt/beams", ""} + ]}, + %% Other configs +] +``` + +This will instruct the Erlang VM on which Riak runs to look for compiled +`.beam` files in the proper directory. You should re-start the node at +this point. Once the node has been re-started, you can use the node's +Erlang shell to register the `yz_httpheader_extractor`. First, attach to +the shell: + +```bash +riak attach +``` + +At this point, we need to choose a MIME type for our extractor. Let's +call it `application/httpheader`. Once you're in the shell: + +```erlang +> yz_extractor:register("application/httpheader", yz_httpheader_extractor). +``` + +If successful, this command will return a list of currently registered +extractors. It should look like this: + +```erlang +[{default,yz_noop_extractor}, + {"application/httpheader",yz_httpheader_extractor}, + {"application/json",yz_json_extractor}, + {"application/riak_counter",yz_dt_extractor}, + {"application/riak_map",yz_dt_extractor}, + {"application/riak_set",yz_dt_extractor}, + {"application/xml",yz_xml_extractor}, + {"text/plain",yz_text_extractor}, + {"text/xml",yz_xml_extractor}] +``` + +If the `application/httpheader` extractor is part of that list, then the +extractor has been successfully registered. + +## Verifying Our Custom Extractor + +Now that Riak Search knows how to decode and extract HTTP header packet +data, let's store some in Riak and then query it. We'll put the example +packet data from above in a `google_packet.bin` file. Then, we'll `PUT` +that binary to Riak's `/search/extract` endpoint: + +```curl +curl -XPUT $RIAK_HOST/search/extract \ + -H 'Content-Type: application/httpheader' \ # Note that we used our custom MIME type + --data-binary @google_packet.bin +``` + +That should return the following JSON: + +```json +{ + "method": "GET", + "host": "www.google.com", + "uri": "/" +} +``` + +We can also verify this in the Erlang shell (whether in a Riak node's +Erlang shell or otherwise): + +```erlang +yz_extractor:run(<<"GET http://www.google.com HTTP/1.1\n">>, yz_httpheader_extractor). + +%% Console output: +[{method,'GET'},{host,<<"www.google.com">>},{uri,<<"/">>}] +``` + +## Indexing and Searching HTTP Header Packet Data + +Now that Solr knows how to extract HTTP header packet data, we need to +create a schema that extends the [default schema]({{<baseurl>}}riak/kv/2.2.6/developing/usage/search-schemas/#creating-a-custom-schema). The following fields should be added +to `<fields>` in the schema, which we'll name `http_header_schema` and +store in a `http_header_schema.xml` file: + +```xml +<?xml version="1.0" encoding="UTF-8" ?> +<schema name="http_header_schema" version="1.5"> +<fields> + <!-- other required fields here --> + + <field name="method" type="string" indexed="true" stored="true" multiValued="false"/> + <field name="host" type="string" indexed="true" stored="true" multiValued="false"/> + <field name="uri" type="string" indexed="true" stored="true" multiValued="false"/> +</fields> +``` + +Now, we can store the schema: + +```java +import org.apache.commons.io.FileUtils + +File xml = new File("http_header_schema.xml"); +String xmlString = FileUtils.readFileToString(xml); +YokozunaSchema schema = new YokozunaSchema("http_header_schema", xmlString); +StoreSchema storeSchemaOp = new StoreSchema.Builder(schema).build(); +client.execute(storeSchemaOp); +``` + +```ruby +schema_xml = File.read('http_header_schema.xml') +client.create_search_schema('http_header_schema', schema_xml) +``` + +```php +$schema_string = file_get_contents('http_header_schema.xml'); +(new \Basho\Riak\Command\Builder\StoreSchema($riak)) + ->withName('http_header_schema') + ->withSchemaString($schema_string) + ->build() + ->execute(); +``` + +```python +import io + +schema_xml = open('http_header_schema.xml').read() +client.create_search_schema('http_header_schema', schema_xml) +``` + +```curl +curl -XPUT $RIAK_HOST/search/schema/http_header_schema \ + -H 'Content-Type: application/xml' \ + --data-binary @http_header_schema.xml +``` + +Riak now has our schema stored and ready for use. Let's create a search +index called `header_data` that's associated with our new schema: + +```java +YokozunaIndex headerDataIndex = new YokozunaIndex("header_data", "http_header_schema"); +StoreSearchIndex storeIndex = new StoreSearchIndex.Builder(headerDataIndex) + .build(); +client.execute(storeIndex); +``` + +```ruby +client.create_search_index('header_data', 'http_header_schema') +``` + +```php +(new \Basho\Riak\Command\Builder\StoreIndex($riak)) + ->withName('header_data') + ->usingSchema('http_header_schema') + ->build() + ->execute(); +``` + +```python +client.create_search_index('header_data', 'http_header_schema') +``` + +```curl +curl -XPUT $RIAK_HOST/search/index/header_data \ + -H 'Content-Type: application/json' \ + -d '{"schema":"http_header_schema"}' +``` + +Now, we can create and activate a [bucket type]({{<baseurl>}}riak/kv/2.2.6/developing/usage/bucket-types) +for all of the HTTP header data that we plan to store. Any bucket that +bears this type will be associated with our `header_data` search index. +We'll call our bucket type `http_data_store`. + +```bash +riak-admin bucket-type create http_data_store '{"props":{"search_index":"header_data"}}' +riak-admin bucket-type activate http_data_store +``` + +Let's use the same `google_packet.bin` file that we used previously and +store it in a bucket with the `http_data_store` bucket type, making sure +to use our custom `application/httpheader` MIME type: + +```java +Location key = new Location(new Namespace("http_data_store", "packets"), "google"); +File packetData = new File("google_packet.bin"); +byte[] packetBinary = FileUtils.readFileToByteArray(packetData); + +RiakObject packetObject = new RiakObject() + .setContentType("application/httpheader") + .setValue(BinaryValue.create(packetBinary)); + +StoreValue storeOp = new StoreValue.Builder(packetObject) + .setLocation(key) + .build(); +client.execute(storeOp); +``` + +```ruby +packet_data = File.read('google_packet.bin') +bucket = client.bucket_type('http_data_store').bucket('packets') +obj = Riak::Robject.new(bucket, 'google') +obj.content_type = 'application/httpheader' +obj.raw_data = packetData +obj.store +``` + +```php +$object = new Object(file_get_contents("google_packet.bin"), ['Content-Type' => 'application/httpheader']); + +(new \Basho\Riak\Command\Builder\StoreObject($riak)) + ->buildLocation('google', 'packets', 'http_data_store') + ->withObject($object) + ->build() + ->execute(); +``` + +```python +packet_data = open('google_packet.bin').read() +bucket = client.bucket_type('http_data_store').bucket('packets') +obj = RiakObject(client, bucket, 'google') +obj.content_type = 'application/httpheader' +obj.data = packet_data +obj.store() +``` + +```curl +curl -XPUT $RIAK_HOST/types/http_data_store/buckets/packets/keys/google \ + -H 'Content-Type: application/httpheader' \ + --data-binary @google_packet.bin +``` + +Now that we have some header packet data stored, we can query our +`header_data` index on whatever basis we'd like. First, let's verify +that we'll get one result if we query for objects that have the HTTP +method `GET`: + +```java +// Using the same method from above: +String query = "method:GET"; + +// Again using the same method from above: +int numberFound = results.numResults(); // 1 +``` + +```ruby +results = client.search('http_header_schema', 'method:GET') +results['num_found'] # 1 +``` + +```php +$response = (\Basho\Riak\Command\Search\FetchObjects($riak)) + ->withQuery('method:GET') + ->withIndexName('header_data') + ->build() + ->execute(); + +$response->getNumFound(); +``` + +```python +results = client.fulltext_search('http_header_schema', 'method:GET') +results['num_found'] # 1 +``` + +```curl +curl "$RIAK_HOST/search/query/header_data?wt=json&q=method:GET" + +# This should return a fairly large JSON object with a "num_found" field +# The value of that field should be 1 +``` diff --git a/content/riak/kv/2.2.6/developing/usage/deleting-objects.md b/content/riak/kv/2.2.6/developing/usage/deleting-objects.md new file mode 100644 index 0000000000..c0fc5a64e2 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/usage/deleting-objects.md @@ -0,0 +1,152 @@ +--- +title: "Deleting Objects" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Deleting Objects" + identifier: "usage_deleting_objects" + weight: 103 + parent: "developing_usage" +toc: true +--- + +The delete command follows a predictable pattern and looks like this: + +``` +DELETE /types/TYPE/buckets/BUCKET/keys/KEY +``` + +The normal HTTP response codes for `DELETE` operations are `204 No +Content` and `404 Not Found`. 404 responses are *normal*, in the sense +that `DELETE` operations are idempotent and not finding the resource has +the same effect as deleting it. + +Let's try to delete the `genius` key from the `oscar_wilde` bucket +(which bears the type `quotes`): + +```java +Location geniusQuote = new Location(new Namespace("quotes", "oscar_wilde"), "genius"); +DeleteValue delete = new DeleteValue.Builder(geniusQuote).build(); +client.execute(delete); +``` + +```ruby +bucket = client.bucket_type('quotes').bucket('oscar_wilde') +bucket.delete('genius') +``` + +```php +(new \Basho\Riak\Command\Builder\DeleteObject($riak)) + ->buildBucket('oscar_wilde', 'quotes') + ->build() + ->execute(); +``` + +```python +bucket = client.bucket_type('quotes').bucket('oscar_wilde') +bucket.delete('genius') +``` + +```csharp +var id = new RiakObjectId("users", "random_user_keys", null); +var obj = new RiakObject(id, @"{'user':'data'}", + RiakConstants.ContentTypes.ApplicationJson); +var rslt = client.Put(obj); +string key = rslt.Value.Key; +id = new RiakObjectId("users", "random_user_keys", key); +var del_rslt = client.Delete(id); +``` + +```javascript +// continuing from above example +options = { + bucketType: 'users', bucket: 'random_user_keys', + key: generatedKey +}; +client.deleteValue(options, function (err, rslt) { + if (err) { + throw new Error(err); + } +}); +``` + +```erlang +riakc_pb_socket:delete(Pid, {<<"quotes">>, <<"oscar_wilde">>}, <<"genius">>) +``` + +```golang +// Continuing from above example +cmd, err = riak.NewDeleteValueCommandBuilder(). + WithBucketType("users"). + WithBucket("random_user_keys"). + WithKey(rsp.GeneratedKey). + Build() + +if err != nil { + fmt.Println(err.Error()) + return +} + +if err := cluster.Execute(cmd); err != nil { + fmt.Println(err.Error()) + return +} +``` + +```curl +curl -XDELETE http://localhost:8098/types/quotes/buckets/oscar_wilde/keys/genius +``` + +## Client Library Examples + +If you are updating an object that has been deleted---or if an update +might target a deleted object---we recommend that +you first fetch the [causal context]({{<baseurl>}}riak/kv/2.2.6/learn/concepts/causal-context) of the object prior to updating. +This can be done by setting the `deletedvclock` parameter to `true` as +part of the [fetch operation]({{<baseurl>}}riak/kv/2.2.6/developing/api/protocol-buffers/fetch-object). This can also be done +with the official Riak clients for Ruby, Java, and Erlang, as in the +example below: + + +```ruby +object.delete +deleted_object = bucket.get('bucket', 'key', deletedvclock: true) +deleted_object.vclock +``` + +```python +# It is not currently possible to fetch the causal context for a deleted +# key in the Python client. +``` + +```java +Location loc = new Location("<bucket>") + .setBucketType("<bucket_type>") + .setKey("<key>"); +FetchValue fetch = new FetchValue.Builder(loc) + .withOption(Option.DELETED_VCLOCK, true) + .build(); +FetchValue.Response response = client.execute(fetch); +System.out.println(response.getVclock().asString()); +``` + +```erlang +{ok, Obj} = riakc_pb_socket:get(Pid, + {<<"bucket_type">>, <<"bucket">>}, + <<"key">>, + [{deleted_vclock}]). + +%% In the Erlang client, the vector clock is accessible using the Obj +%% object obtained above. +``` + +```php +$response = (new \Basho\Riak\Command\Builder\FetchObject($riak)) + ->buildLocation('deleted_key', 'in_some_bucket', 'of_a_certain_type') + ->build() + ->execute(); + +echo $response->getVclock(); // a85hYGBgzGDKBVI8m9WOeb835ZRhYCg1zGBKZM5jZdhnceAcXxYA +``` diff --git a/content/riak/kv/2.2.6/developing/usage/document-store.md b/content/riak/kv/2.2.6/developing/usage/document-store.md new file mode 100644 index 0000000000..b6fc79ae9f --- /dev/null +++ b/content/riak/kv/2.2.6/developing/usage/document-store.md @@ -0,0 +1,613 @@ +--- +title: "Implementing a Document Store" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Implementing a Document Store" + identifier: "usage_document_store" + weight: 112 + parent: "developing_usage" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/search/document-store + - /riak-docs/riak/kv/2.2.6/dev/search/document-store +--- + +Although Riak wasn't explicitly created as a document store, two +features recently added to Riak---[Riak Search]({{<baseurl>}}riak/kv/2.2.6/developing/usage/search/) and [Riak Data Types]({{<baseurl>}}riak/kv/2.2.6/developing/data-types/)---make it possible to use Riak as a +highly scalable document store with rich querying capabilities. In this +tutorial, we'll build a basic implementation of a document store using +[Riak maps]({{<baseurl>}}riak/kv/2.2.6/developing/data-types/#maps). + +## Basic Approach + +Riak Search enables you to implement a document store in Riak in a +variety of ways. You could, for example, store and query JSON objects or +XML and then retrieve them later via Solr queries. In this tutorial, +however, we will store data in [Riak maps]({{<baseurl>}}riak/kv/2.2.6/developing/data-types/#maps), +index that data using Riak Search, and then run Solr queries against +those stored objects. + +You can think of these Search indexes as **collections**. Each indexed +document will have an ID generated automatically by Search, and because +we're not interested in running normal [key/value queries]({{<baseurl>}}riak/kv/2.2.6/developing/key-value-modeling) on these objects, we'll allow Riak to assign [keys]({{<baseurl>}}riak/kv/2.2.6/learn/concepts/keys-and-objects) automatically. This means that all we have to do is worry about the bucket type and/or bucket when storing objects. + +## Use Case + +Let's say that we're building a WordPress-style CMS and storing blog +posts in Riak. We will be storing the following information about each +post: + +* Title +* Author +* Content (the body of the post) +* Keywords associated with the post +* Date posted +* Whether the post has been published on the site + +For each of those pieces of information, we'll need to decide on (a) +which Riak Data Type most directly corresponds and (b) which Solr type +we want to associate with the info. It's important to bear in mind that +Riak Data Types can be indexed as a wide variety of things, e.g. +registers as Solr text fields, sets as multi-valued datetimes, etc. The +table below shows which Riak Data Type and Solr type we'll be using for +each field in our Riak maps. + +Info | Riak Data Type | Solr type +:----|:---------------|:--------- +Post title | Register | String +Post author | Register | String +Post content | Register | Text +Keywords | Set | Multi-valued string +Date posted | Register | Datetime +Whether the post is currently in draft form | Flag | Boolean + +Before we start actually creating and storing blog posts, let's set up +Riak Search with an appropriate index and schema. + +## Creating a Schema and Index + +In the documentation on [search schemas]({{<baseurl>}}riak/kv/2.2.6/developing/usage/search-schemas), you'll find a +baseline schema to be used for creating custom schemas. We'll use that +baseline schema here and add the following fields to the `<fields>` +list: + +```xml +<field name="title_register" type="string" indexed="true" stored="true" /> +<field name="author_register" type="string" indexed="true" stored="true" /> +<field name="content_register" type="text" indexed="true" stored="true" /> +<field name="keywords_set" type="string" indexed="true" stored="true" multiValued="true" /> +<field name="date_register" type="datetime" indexed="true" stored="true" /> +<field name="published_flag" type="boolean" indexed="true" stored="true" /> +``` + +You can see the full schema [on +GitHub](https://github.com/basho/basho_docs/raw/master/extras/data/blog_post_schema.xml). +Let's store that schema in a file called `blog_post_schema.xml` and +upload that schema to Riak: + +```java +import org.apache.commons.io.FileUtils; + +File xml = new File("blog_post_schema.xml"); +String xmlString = FileUtils.readFileToString(xml); +YokozunaSchema schema = new YokozunaSchema("blog_post_schema", xmlString); +StoreSchema storeSchemaOp = new StoreSchema.Builder(schema).build(); +client.execute(storeSchemaOp); +``` + +```ruby +schema_data = File.read('blog_post_schema.xml') +client.create_search_schema('blog_post_schema', schema_data) +``` + +```php +$schema_string = file_get_contents('blog_post_schema.xml'); +(new \Basho\Riak\Command\Builder\StoreSchema($riak)) + ->withName('blog_post_schema') + ->withSchemaString($schema_string) + ->build() + ->execute(); +``` + +```python +xml_file = open('blog_post_schema.xml', 'r') +schema_data = xml_file.read() +client.create_search_schema('blog_post_schema', schema_data) +xml_file.close() +``` + +```csharp +var schemaXml = File.ReadAllText("blog_post_schema.xml"); +var schema = new SearchSchema("blog_post_schema", schemaXml); +var rslt = client.PutSearchSchema(schema); +``` + +```javascript +/* + * Full example here: + * https://github.com/basho/riak-nodejs-client-examples/blob/master/dev/search/document-store.js + * + */ +var options = { + schemaName: 'blog_post_schema', + schema: schemaXml +}; +client.storeSchema(options, function (err, rslt) { + if (err) { + throw new Error(err); + } +}); +``` + +```erlang +{ok, SchemaData} = file:read_file("blog_post_schema.xml"), +riakc_pb_socket:create_search_schema(Pid, <<"blog_post_schema">>, SchemaData). +``` + +```curl +curl -XPUT $RIAK_HOST/search/schema/blog_post_schema \ + -H 'Content-Type: application/xml' \ + --data-binary @blog_post_schema.xml +``` + +With our schema uploaded, we can create an index called `blog_posts` and +associate that index with our schema: + +```java +YokozunaIndex blogPostIndex = new YokozunaIndex("blog_posts", "blog_post_schema"); +StoreIndex storeIndex = new StoreIndex.Builder(blogPostIndex).build(); +client.execute(storeIndex); +``` + +```ruby +client.create_search_index('blog_posts', 'blog_post_schema') +``` + +```php +(new Command\Builder\Search\StoreIndex($riak)) + ->withName('blog_posts') + ->usingSchema('blog_post_schema') + ->build() + ->execute(); +``` + +```python +client.create_search_index('blog_posts', 'blog_post_schema') +``` + +```csharp +var idx = new SearchIndex("blog_posts", "blog_post_schema"); +var rslt = client.PutSearchIndex(idx); +``` + +```javascript +var options = { + schemaName: 'blog_post_schema', + indexName: 'blog_posts' +}; +client.storeIndex(options, function (err, rslt) { + if (err) { + throw new Error(err); + } +}); +``` + +```erlang +riakc_pb_socket:create_search_index(Pid, <<"blog_posts">>, <<"blog_post_schema">>, []). +``` + +```curl +curl -XPUT $RIAK_HOST/search/index/blog_posts \ + -H 'Content-Type: application/json' \ + -d '{"schema": "blog_post_schema"}' +``` + +## How Collections will Work + +Collections are not a concept that is native to Riak but we can easily +mimic collections by thinking of a bucket type as a collection. When we +associate a bucket type with a Riak Search index, all of the objects +stored in any bucket of that bucket type will be queryable on the basis +of that one index. For this tutorial, we'll create a bucket type called +`cms` and think of that as a collection. We could also restrict our +`blog_posts` index to a single bucket just as easily and think of that +as a queryable collection, but we will not do that in this tutorial. + +The advantage of the bucket-type-based approach is that we could store +blog posts from different blogs in different blog posts and query them +all at once as part of the same index. It depends on the use case at +hand. In this tutorial, we'll only be storing posts from one blog, which +is called "Cat Pics Quarterly" and provides in-depth theoretical +discussions of cat pics with a certain number of Reddit upvotes. All of +the posts in this blog will be stored in the bucket +`cat_pics_quarterly`. + +First, let's create our `cms` bucket type and associate it with the +`blog_posts` index: + +```bash +riak-admin bucket-type create cms \ + '{"props":{"datatype":"map","search_index":"blog_posts"}}' +riak-admin bucket-type activate cms +``` + +Now, any object stored in any bucket of the type `cms` will be indexed +as part of our "collection." + +## Storing Blog Posts as Maps + +Now that we know how each element of a blog post can be translated into +one of the Riak Data Types, we can create an interface in our +application to serve as that translation layer. Using the method +described in [Data Modeling with Riak Data Types]({{<baseurl>}}riak/kv/2.2.6/developing/data-modeling), we can construct a +class that looks like this: + +```java +import java.util.Set; + +public class BlogPost { + private String title; + private String author; + private String content; + private Set<String> keywords; + private DateTime datePosted; + private Boolean published; + private static final String bucketType = "cms"; + + private Location location; + + private RiakClient client; + + public BlogPost(RiakClient client + String bucketName, + String title, + String author, + String content, + Set<String> keywords, + DateTime datePosted, + Boolean published) { + this.client = client; + this.location = new Location(new Namespace(bucketType, bucketName), null); + this.title = title; + this.author = author; + this.content = content; + this.keywords = keywords; + this.datePosted = datePosted; + this.published = published; + } + + public void store() throws Exception { + RegisterUpdate titleUpdate = new RegisterUpdate(title); + RegisterUpdate authorUpdate = new RegisterUpdate(author); + RegisterUpdate contentUpdate = new RegisterUpdate(content); + SetUpdate keywordsUpdate = new SetUpdate(); + for (String keyword : keywords) { + keywordsUpdate.add(keyword); + } + RegisterUpdate dateUpdate = + new RegisterUpdate(datePosted.toString("YYYY-MM-DD HH:MM")); + if (published) { + FlagUpdate published = new FlagUpdate(published); + } + FlagUpdate publishedUpdate = new FlagUpdate(published); + MapUpdate mapUpdate = new MapUpdate() + .update("title", titleUpdate) + .update("author", authorUpdate) + .update("content", contentUpdate) + .update("keywords", keywordsUpdate) + .update("date", dateUpdate) + .update("published", publishedUpdate); + UpdateMap storeBlogPost = new UpdateMap.Builder(location, mapUpdate) + .build(); + client.execute(storeBlogPost); + } +} +``` + +```ruby +class BlogPost + def initialize(bucket_name, title, author, content, keywords, date_posted, published) + bucket = client.bucket_type('cms').bucket(bucket_name) + map = Riak::Crdt::Map.new(bucket, nil) + map.batch do |m| + m.registers['title'] = title + m.registers['author'] = author + m.registers['content'] = content + keywords.each do |k| + m.sets['keywords'].add(k) + end + m.registers['date'] = date_posted + if published + m.flags['published'] = true + end + end +end +``` + +```php +class BlogPost { + private $title = ''; + private $author = ''; + private $content = ''; + private $keywords = []; + private $datePosted = ''; + private $published = false; + private $bucketType = "cms"; + + private $bucket = null; + + private $riak = null; + + public function __construct(\Basho\Riak $riak, $bucket, $title, $author, $content, array $keywords, $date, $published) + { + this->riak = $riak; + this->bucket = new Bucket($bucket, $this->bucketType); + this->title = $title; + this->author = $author; + this->content = $content; + this->keywords = $keywords; + this->datePosted = $date; + this->published = $published; + } + + public function store() + { + $setBuilder = (new \Basho\Riak\Command\Builder\UpdateSet($this->riak)); + + foreach($this->keywords as $keyword) { + $setBuilder->add($keyword); + } + + (new \Basho\Riak\Command\Builder\UpdateMap($this->riak)) + ->updateRegister('title', $this->title) + ->updateRegister('author', $this->author) + ->updateRegister('content', $this->content) + ->updateRegister('date', $this->date) + ->updateFlag('published', $this->published) + ->updateSet('keywords', $setBuilder) + ->withBucket($this->bucket) + ->build() + ->execute(); + } +} +``` + +```python +from riak.datatypes import Map + +class BlogPost: + def __init__(bucket_name, title, author, content, keywords, date_posted, published): + bucket = client.bucket_type('cms').bucket(bucket_name) + map = Map(bucket, None) + self.map.registers['title'].assign(title) + self.map.registers['author'].assign(author) + self.map.registers['content'].assign(content) + for k in keywords: + self.map.sets['keywords'].add(k) + self.map.registers['date'] = date_posted + if published: + self.map.flags['published'].enable() + self.map.store() +``` + +```csharp +/* + * Please see the code in the RiakClientExamples project: + * https://github.com/basho/riak-dotnet-client/tree/develop/src/RiakClientExamples/Dev/Search + */ +``` + +```javascript +/* + * Please see the code in the examples repository: + * https://github.com/basho/riak-nodejs-client-examples/blob/master/dev/search/ + */ +``` + +Now, we can store some blog posts. We'll start with just one: + +```java +Set<String> keywords = new HashSet<String>(); +keywords.add("adorbs"); +keywords.add("cheshire"); + +BlogPost post1 = new BlogPost(client, // client object + "cat_pics_quarterly", // bucket + "This one is so lulz!", // title + "Cat Stevens", // author + "Please check out these cat pics!", // content + keywords, // keywords + new DateTime(), // date posted + true); // published +try { + post1.store(); +} catch (Exception e) { + System.out.println(e); +} +``` + +```ruby +keywords = ['adorbs', 'cheshire'] +date = Time.now.strftime('%Y-%m-%d %H:%M') +blog_post1 = BlogPost.new('cat_pics_quarterly', + 'This one is so lulz!', + 'Cat Stevens', + 'Please check out these cat pics!', + keywords, + date, + true) +``` + +```php +$keywords = ['adorbs', 'cheshire']; +$date = new \DateTime('now'); + +$post1 = new BlogPost( + $riak, // client object + 'cat_pics_quarterly', // bucket + 'This one is so lulz!', // title + 'Cat Stevens', // author + 'Please check out these cat pics!', // content + $keywords, // keywords + $date, // date posted + true // published +); +``` + +```python +import datetime + +keywords = ['adorbs', 'cheshire'] +date = datetime.datetime.now().strftime('%Y-%m-%d %H:%M') +blog_post1 = BlogPost('cat_pics_quarterly', + 'This one is so lulz!', + 'Cat Stevens', + 'Please check out these cat pics!', + keywords, + date, + true) +``` + +```csharp +var keywords = new HashSet<string> { "adorbs", "cheshire" }; + +var post = new BlogPost( + "This one is so lulz!", + "Cat Stevens", + "Please check out these cat pics!", + keywords, + DateTime.Now, + true); + +var repo = new BlogPostRepository(client, "cat_pics_quarterly"); +string id = repo.Save(post); +``` + +```javascript +var post = new BlogPost( + 'This one is so lulz!', + 'Cat Stevens', + 'Please check out these cat pics!', + [ 'adorbs', 'cheshire' ], + new Date(), + true +); + +var repo = new BlogPostRepository(client, 'cat_pics_quarterly'); + +repo.save(post, function (err, rslt) { + logger.info("key: '%s', model: '%s'", rslt.key, JSON.stringify(rslt.model)); +}); +``` + +## Querying + +Now that we have some blog posts stored in our "collection," we can +start querying for whatever we'd like. Let's say that we want to find +all blog posts with the keyword `funny` (after all, some cat pics are +quite serious, and we may not want those). + +```java +String index = "blog_posts"; +String query = "keywords_set:funny"; + +SearchOperation searchOp = new SearchOperation + .Builder(BinaryValue.create(index), query) + .build(); +cluster.execute(searchOp); +List<Map<String, List<String>>> results = searchOp.get().getAllResults(); +``` + +```ruby +results = client.search('blog_posts', 'keywords_set:funny') +``` + +```php +$response = (new \Basho\Riak\Command\Builder\Search\FetchObjects($riak)) + ->withIndexName('blog_posts') + ->withQuery('keywords_set:funny') + ->build() + ->execute(); +``` + +```python +results = client.fulltext_search('blog_posts', 'keywords_set:funny') +``` + +```csharp +var searchRequest = new RiakSearchRequest("blog_posts", "keywords_set:funny"); +var rslt = client.Search(searchRequest); +``` + +```javascript +var searchCmd = new Riak.Commands.YZ.Search.Builder() + .withIndexName('blog_posts') + .withQuery('keywords_set:funny') + .withCallback(search_cb) + .build(); + +client.execute(searchCmd); +``` + +```curl +curl "$RIAK_HOST/search/query/blog_posts?wt=json&q=keywords_set:funny" +``` + +Or we can find posts that contain the word `furry`: + +```java +String index = "blog_posts"; +String query = "content_register:furry"; + +SearchOperation searchOp = new SearchOperation + .Builder(BinaryValue.create(index), query) + .build(); +cluster.execute(searchOp); +List<Map<String, List<String>>> results = searchOp.get().getAllResults(); +``` + +```ruby +results = client.search('blog_posts', 'content_register:furry') +``` + +```php +$response = (new \Basho\Riak\Command\Builder\Search\FetchObjects($riak)) + ->withIndexName('blog_posts') + ->withQuery('content_register:furry') + ->build() + ->execute(); +``` + +```python +results = client.fulltext_search('blog_posts', 'content_register:furry') +``` + +```csharp +var searchRequest = new RiakSearchRequest("blog_posts", "content_register:furry"); +var rslt = client.Search(searchRequest); +``` + +```javascript +var searchCmd = new Riak.Commands.YZ.Search.Builder() + .withIndexName('blog_posts') + .withQuery('content_register:furry') + .withCallback(search_cb) + .build(); + +client.execute(searchCmd); +``` + +```curl +curl "$RIAK_HOST/search/query/blog_posts?wt=json&q=content_register:furry" +``` + +Here are some more possible queries: + +Info | Query +:----|:----- +Unpublished posts | `published_flag:false` +Titles that begin with `Loving*` | `title_register:Loving*` +Post bodies containing the words `furry` and `jumping` | `content_register:[furry AND jumping]` diff --git a/content/riak/kv/2.2.6/developing/usage/mapreduce.md b/content/riak/kv/2.2.6/developing/usage/mapreduce.md new file mode 100644 index 0000000000..5d9dd7cb24 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/usage/mapreduce.md @@ -0,0 +1,242 @@ +--- +title: "Using MapReduce" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Using MapReduce" + identifier: "usage_mapreduce" + weight: 106 + parent: "developing_usage" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/using/mapreduce + - /riak-docs/riak/kv/2.2.6/dev/using/mapreduce +--- + +[usage 2i]: {{<baseurl>}}riak/kv/2.2.6/developing/usage/secondary-indexes +[usage search]: {{<baseurl>}}riak/kv/2.2.6/developing/usage/search +[usage types]: {{<baseurl>}}riak/kv/2.2.6/developing/usage/bucket-types +[api http]: {{<baseurl>}}riak/kv/2.2.6/developing/api/http +[api pb]: {{<baseurl>}}riak/kv/2.2.6/developing/api/protocol-buffers +[glossary vnode]: {{<baseurl>}}riak/kv/2.2.6/learn/glossary/#vnode +[guide mapreduce]: {{<baseurl>}}riak/kv/2.2.6/developing/app-guide/advanced-mapreduce + +{{% note title="Use MapReduce sparingly" %}} +In Riak KV, MapReduce is the primary method for non-primary-key-based +querying. Although useful for tasks such as batch +processing jobs, MapReduce operations can be very computationally +expensive to the extent that they can degrade performance in +production clusters operating under load. Thus, we recommend running +MapReduce operations in a controlled, rate-limited fashion and never for +realtime querying purposes. +{{% /note %}} + +MapReduce (M/R) is a technique for dividing data processing work across +a distributed system. It takes advantage of the parallel processing +power of distributed systems and also reduces network bandwidth, as the +algorithm is passed around to where the data lives rather than +transferring a potentially huge dataset to a client algorithm. + +You can use MapReduce for things like: filtering documents by +tags, counting words in documents, and extracting links to related data. +In Riak KV, MapReduce is one method for querying that is not strictly based +on key querying, alongside [secondary indexes][usage 2i] +and [search][usage search]. MapReduce jobs can be submitted through the +[HTTP API][api http] or the [Protocol Buffers API][api pb], although we +strongly recommend using the Protocol Buffers API for performance +reasons. + +## Features + +* Map phases execute in parallel with data locality. +* Reduce phases execute in parallel on the node where the job was + submitted. +* MapReduce queries written in Erlang. + +## When to Use MapReduce + +* When you know the set of objects over which you want to MapReduce + (i.e. the locations of the objects, as specified by [bucket type][usage types], bucket, and key) +* When you want to return actual objects or pieces of objects and not + just the keys. [Search][usage search] and [secondary indexes][usage 2i] are other means of returning objects based on + non-key-based queries, but they only return lists of keys and not + whole objects. +* When you need the utmost flexibility in querying your data. MapReduce + gives you full access to your object and lets you pick it apart any + way you want. + +## When Not to Use MapReduce + +* When you want to query data over an entire bucket. MapReduce uses a + list of keys, which can place a lot of demand on the cluster. +* When you want latency to be as predictable as possible. + +## How it Works + +The MapReduce framework helps developers divide a query into steps, +divide the dataset into chunks, and then run those step/chunk pairs in +separate physical hosts. + +There are two steps in a MapReduce query: + +* **Map** --- The data collection phase, which breaks up large chunks of + work into smaller ones and then takes action on each chunk. Map + phases consist of a function and a list of objects on which the map + operation will operate. +* **Reduce** --- The data collation or processing phase, which combines + the results from the map step into a single output. The reduce phase + is optional. + +Riak KV MapReduce queries have two components: + +* A list of inputs +* A list of phases + +The elements of the input list are object locations as specified by +[bucket type][usage types], bucket, and key. The elements of the +phases list are chunks of information related to a map, a reduce, or a +link function. + +A MapReduce query begins when a client makes the request to Riak KV. The +node that the client contacts to make the request becomes the +*coordinating node* responsible for the MapReduce job. As described +above, each job consists of a list of phases, where each phase is either +a map or a reduce phase. The coordinating node uses the list of phases +to route the object keys and the function that will operate over the +objects stored in those keys and instruct the proper [vnode][glossary vnode] to +run that function over the right objects. + +After running the map function, the results are sent back to the +coordinating node. This node then concatenates the list and passes that +information over to a reduce phase on the same coordinating node, +assuming that the next phase in the list is a reduce phase. + +The diagram below provides an illustration of how a coordinating vnode +orchestrates a MapReduce job. + +![MapReduce Diagram]({{<baseurl>}}images/MapReduce-diagram.png) + +## Example + +In this example, we'll create four objects with the text "caremad" +repeated a varying number of times and store those objects in the bucket +`training` (which does not bear a [bucket type][usage types]). +An Erlang MapReduce function will be used to count the occurrences of +the word "caremad." + +### Data object input commands + +For the sake of simplicity, we'll use [curl](http://curl.haxx.se/) +in conjunction with Riak KV's [HTTP API][api http] to store the objects: + +```curl +curl -XPUT http://localhost:8098/buckets/training/keys/foo \ + -H 'Content-Type: text/plain' \ + -d 'caremad data goes here' + +curl -XPUT http://localhost:8098/buckets/training/keys/bar \ + -H 'Content-Type: text/plain' \ + -d 'caremad caremad caremad caremad' + +curl -XPUT http://localhost:8098/buckets/training/keys/baz \ + -H 'Content-Type: text/plain' \ + -d 'nothing to see here' + +curl -XPUT http://localhost:8098/buckets/training/keys/bam \ + -H 'Content-Type: text/plain' \ + -d 'caremad caremad caremad' +``` + +### MapReduce invocation + +To invoke a MapReduce function from a compiled Erlang program requires +that the function be compiled and distributed to all nodes. + +For interactive use, however, it's not necessary to do so; instead, we +can invoke the client library from the +[Erlang shell](http://www.erlang.org/doc/man/shell.html) and define +functions to send to Riak KV on the fly. + +First we defined the map function, which specifies that we want to get +the key for each object in the bucket `training` that contains the text +`caremad`. + +We're going to generalize and optimize it a bit by supplying a +compiled regular expression when we invoke MapReduce; our function +will expect that as the third argument. + +```erlang +ReFun = fun(O, _, Re) -> case re:run(riak_object:get_value(O), Re, [global]) of + {match, Matches} -> [{riak_object:key(O), length(Matches)}]; + nomatch -> [{riak_object:key(O), 0}] +end end. +``` + +Next, to call `ReFun` on all keys in the `training` bucket, we can do +the following in the Erlang shell. + +{{% note title="Warning" %}} +Do not use this in a production +environment; listing all keys to identify those in the `training` bucket +is a very expensive process. +{{% /note %}} + +```erlang +{ok, Re} = re:compile("caremad"). +``` + +That will return output along the following lines, verifying that +compilation has completed: + +``` +{ok,{re_pattern,0,0, + <<69,82,67,80,69,0,0,0,0,0,0,0,6,0,0,0,0,0,0,0,99,0,100, + ...>>}} +``` + +Then, we can create a socket link to our cluster: + +```erlang +{ok, Riak} = riakc_pb_socket:start_link("127.0.0.1", 8087). + +%% This should return a process ID: +%% {ok,<0.34.0>} +``` + +Then we can run the compiled MapReduce job on the `training` bucket: + +```erlang +riakc_pb_socket:mapred_bucket(Riak, <<"training">>, + [{map, {qfun, ReFun}, Re, true}]). +``` + +If your bucket is part of a bucket type, you would use the following: + +```erlang +B = {<<"my_bucket_type">>, <<"training">>}, +Args = [{map, {qfun, ReFun}, Re, true}]), +riakc_pb_socket:mapred_bucket(Riak, B, Args). +``` + +That will return a list of tuples. The first element in each tuple is +the key for each object in the bucket, while the second element displays +the number of instances of the word "caremad" in the object: + +``` +{ok,[{0, + [{<<"foo">>,1},{<<"bam">>,3},{<<"baz">>,0},{<<"bar">>,4}]}]} +``` + +### Recap + +In this tutorial, we ran an Erlang MapReduce function against a total of +four object in the `training` bucket. This job took each key/value +object in the bucket and searched the text for the word "caremad," +counting the number of instances of the word. + +## Advanced MapReduce Queries + +For more detailed information on MapReduce queries in Riak KV, we recommend +checking out our [Advanced MapReduce][guide mapreduce] guide. diff --git a/content/riak/kv/2.2.6/developing/usage/reading-objects.md b/content/riak/kv/2.2.6/developing/usage/reading-objects.md new file mode 100644 index 0000000000..35334bd0a9 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/usage/reading-objects.md @@ -0,0 +1,247 @@ +--- +title: "Reading Objects" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Reading Objects" + identifier: "usage_reading_objects" + weight: 101 + parent: "developing_usage" +toc: true +--- + +[glossary vnode]: {{<baseurl>}}riak/kv/2.2.6/learn/glossary/#vnode + +You can think of reads in Riak as analogous to HTTP `GET` requests. You +specify a bucket type, bucket, and key, and Riak either returns the +object that's stored there---including its [siblings]({{<baseurl>}}riak/kv/2.2.6/developing/usage/conflict-resolution/#siblings) \(more on that later)---or it returns `not found` (the +equivalent of an HTTP `404 Object Not Found`). + +Here is the basic command form for retrieving a specific key from a +bucket: + +``` +GET /types/<type>/buckets/<bucket>/keys/<key> +``` + +Here is an example of a read performed on the key `rufus` in the bucket +`dogs`, which bears the bucket type `animals`. Please note that for this example to work, you must have first created the bucket-type `animals` as per the instructions on the [bucket type]({{<baseurl>}}riak/kv/2.2.6/using/cluster-operations/bucket-types) page. + +```java +// In the Java client, it is best to specify a bucket type/bucket/key +// Location object that can be used as a reference for further +// operations, as in the example below: +Location myKey = new Location(new Namespace("animals", "dogs"), "rufus"); +``` + +```ruby +bucket = client.bucket_type('animals').bucket('dogs') +obj = bucket.get('rufus') +``` + +```php +$response = (new \Basho\Riak\Command\Builder\FetchObject($riak)) + ->buildLocation('rufus', 'users', 'animals') + ->build() + ->execute(); +``` + +```python +bucket = client.bucket_type('animals').bucket('dogs') +obj = bucket.get('rufus') +``` + +```csharp +// Using the Riak .NET Client it is best to specify a bucket type/bucket/key +// RiakObjectId object that can be used as a reference for further +// operations +var id = new RiakObjectId("animals", "dogs", "rufus"); +``` + +```javascript +client.fetchValue({ bucketType: 'animals', bucket: 'dogs', key: 'rufus' }, function (err, rslt) { + assert(rslt.isNotFound); +}); +``` + +```erlang +{ok, Obj} = riakc_pb_socket:get(Pid, + {<<"animals">>, <<"dogs">>}, + <<"rufus">>). +``` + +```golang +cmd, err = riak.NewFetchValueCommandBuilder(). + WithBucketType("animals"). + WithBucket("dogs"). + WithKey("rufus"). + Build() +if err != nil { + // error occurred +} +``` + +```curl +curl http://localhost:8098/types/animals/buckets/dogs/keys/rufus +``` + +## Read Parameters + +Parameter | Default | Description +:---------|:--------|:----------- +`r` | `quorum` | How many replicas need to agree when retrieving an existing object before the write +`pr` | `0` | How many [vnodes][glossary vnode] must respond for a read to be deemed successful +`notfound_ok` | `true` | If set to `true`, if the first vnode to respond doesn't have a copy of the object, Riak will deem the failure authoritative and immediately return a `notfound` error to the client + +Riak also accepts many query parameters, including `r` for setting the +R-value for GET requests (R values describe how many replicas need to +agree when retrieving an existing object in order to return a successful +response). + +Here is an example of attempting a read with `r` set to `3`: + +```java +// Using the "myKey" location specified above: +FetchValue fetch = new FetchValue.Builder(myKey) + .withOption(FetchOption.R, new Quorum(3)) + .build(); +FetchValue.Response response = client.execute(fetch); +RiakObject obj = response.getValue(RiakObject.class); +System.out.println(obj.getValue()); +``` + +```ruby +bucket = client.bucket_type('animals').bucket('dogs') +obj = bucket.get('rufus', r: 3) +p obj.data +``` + +```php +$response = (new \Basho\Riak\Command\Builder\FetchObject($riak)) + ->buildLocation('rufus', 'dogs', 'animals') + ->build() + ->execute(); + +var_dump($response->getObject()->getData()); +``` + +```python +bucket = client.bucket_type('animals').bucket('dogs') +obj = bucket.get('rufus', r=3) +print obj.data +``` + +```csharp +var id = new RiakObjectId("animals", "dogs", "rufus"); +var opts = new RiakGetOptions(); +opts.SetR(3); +var rslt = client.Get(id, opts); +Debug.WriteLine(Encoding.UTF8.GetString(rslt.Value.Value)); +``` + +```javascript +var fetchOptions = { + bucketType: 'animals', bucket: 'dogs', key: 'rufus', + r: 3 +}; +client.fetchValue(fetchOptions, function (err, rslt) { + var riakObj = rslt.values.shift(); + var rufusValue = riakObj.value.toString("utf8"); + logger.info("rufus: %s", rufusValue); +}); +``` + +```erlang +{ok, Obj} = riakc_pb_socket:get(Pid, + {<<"animals">>, <<"dogs">>}, + <<"rufus">>, + [{r, 3}]). +``` + +```golang +cmd, err := riak.NewFetchValueCommandBuilder(). + WithBucketType("animals"). + WithBucket("dogs"). + WithKey("rufus"). + WithR(3). + Build() + +if err != nil { + fmt.Println(err.Error()) + return +} + +if err := cluster.Execute(cmd); err != nil { + fmt.Println(err.Error()) + return +} + +fvc := cmd.(*riak.FetchValueCommand) +rsp := svc.Response +``` + +```curl +curl http://localhost:8098/types/animals/buckets/dogs/keys/rufus?r=3 +``` + +If you're using HTTP, you will most often see the following response +codes: + +* `200 OK` +* `300 Multiple Choices` +* `304 Not Modified` + +The most common error code: + +* `404 Not Found` + +{{% note title="Note" %}} +If you're using a Riak client instead of HTTP, these responses will vary a +great deal, so make sure to check the documentation for your specific client. +{{% /note %}} + +## Not Found + +If there's no object stored in the location where you attempt a read, you'll get the following response: + +```java +java.lang.NullPointerException +``` + +```ruby +Riak::ProtobuffsFailedRequest: Expected success from Riak but received not_found. The requested object was not found. +``` + +```php +$response->getStatusCode(); // 404 +$response->isSuccess(); // false +``` + +```python +riak.RiakError: 'no_type' +``` + +```csharp +result.IsSuccess == false +result.ResultCode == ResultCode.NotFound +``` + +```javascript +rslt.isNotFound === true; +``` + +```erlang +{error,notfound} +``` + +```golang +fvc := cmd.(*riak.FetchValueCommand) +rsp := fvc.Response +rsp.IsNotFound // Will be true +``` + +```curl +not found +``` diff --git a/content/riak/kv/2.2.6/developing/usage/replication.md b/content/riak/kv/2.2.6/developing/usage/replication.md new file mode 100644 index 0000000000..8f8698779b --- /dev/null +++ b/content/riak/kv/2.2.6/developing/usage/replication.md @@ -0,0 +1,588 @@ +--- +title: "Replication" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Replication" + identifier: "usage_replication" + weight: 115 + parent: "developing_usage" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/advanced/replication-properties + - /riak-docs/riak/kv/2.2.6/dev/advanced/replication-properties +--- + +[usage bucket types]: {{<baseurl>}}riak/kv/2.2.6/developing/usage/bucket-types +[concept eventual consistency]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/eventual-consistency +[plan backend leveldb]: {{<baseurl>}}riak/kv/2.2.6/setup/planning/backend/leveldb +[plan backend bitcask]: {{<baseurl>}}riak/kv/2.2.6/setup/planning/backend/bitcask +[use ref strong consistency]: {{<baseurl>}}riak/kv/2.2.6/using/reference/strong-consistency +[concept clusters]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/clusters + +Riak was built to act as a multi-node [cluster][concept clusters]. It +distributes data across multiple physical servers, which enables it to +provide strong availability guarantees and fault tolerance. + +The [CAP theorem](http://en.wikipedia.org/wiki/CAP_theorem), which +undergirds many of the design decisions behind Riak's architecture, +defines distributed systems in terms of three desired properties: +consistency, availability, and partition (i.e. failure) tolerance. Riak +can be used either as an AP, i.e. available/partition-tolerant, system +or as a CP, i.e. consistent/partition-tolerant, system. The former +relies on an [Eventual Consistency][concept eventual consistency] model, while the latter relies on +a special [strong consistency][use ref strong consistency] subsystem. + +Although the [CAP theorem](http://en.wikipedia.org/wiki/CAP_theorem) +dictates that there is a necessary trade-off between data consistency +and availability, if you are using Riak in an eventually consistent +manner, you can fine-tune that trade-off. The ability to make these +kinds of fundamental choices has immense value for your applications and +is one of the features that differentiates Riak from other databases. + +At the bottom of the page, you'll find a [screencast]({{<baseurl>}}riak/kv/2.2.6/developing/app-guide/replication-properties#screencast) that briefly explains how to adjust your +replication levels to match your application and business needs. + +{{% note title="Note on strong consistency" %}} +An option introduced in Riak version 2.0 is to use Riak as a +<a href="http://docs.basho.com/riak/kv/2.2.6/using/reference/strong-consistency/">strongly +consistent</a> system for data in specified buckets. Using Riak in this way is +fundamentally different from adjusting replication properties and fine-tuning +the availability/consistency trade-off, as it sacrifices _all_ availability +guarantees when necessary. Therefore, you should consult the +<a href="http://docs.basho.com/riak/kv/2.2.6/developing/app-guide/strong-consistency/">Using +Strong Consistency</a> documentation, as this option will not be covered in +this tutorial. +{{% /note %}} + +## How Replication Properties Work + +When using Riak, there are two ways of choosing replication properties: + +1. On a per-request basis +2. In a more programmatic fashion, [using bucket types][usage bucket types] + +### Per-request Replication Properties + +The simplest way to apply replication properties to objects stored in +Riak is to specify those properties + +### Replication Properties Through Bucket Types + +Let's say, for example, that you want to apply an `n_val` of 5, an `r` +of 3, and a `w` of 3 to all of the data in some of the [buckets]({{<baseurl>}}riak/kv/2.2.6/learn/concepts/buckets) that +you're using. In order to set those replication properties, you should +create a bucket type that sets those properties. Below is an example: + +```bash +riak-admin bucket-type create custom_props '{"props":{"n_val":5,"r":3,"w":3}}' +riak-admin bucket-type activate custom_props +``` + +Now, any time you store an object in a bucket with the type +`custom_props` those properties will apply to it. + +## Available Parameters + +The table below lists the most frequently used replication parameters +that are available in Riak. Symbolic values like `quorum` are discussed +[below](#symbolic-consistency-names). Each +parameter will be explained in more detail in later sections: + +Parameter | Common name | Default value | Description +:---------|:------------|:--------------|:----------- +`n_val` | N | `3` | Replication factor, i.e. the number of nodes in the cluster on which an object is to be stored +`r` | R | `quorum` | The number of servers that must respond to a read request +`w` | W | `quorum` | Number of servers that must respond to a write request +`pr` | PR | `0` | The number of primary <a href="http://docs.basho.com/riak/kv/2.2.6/learn/concepts/vnodes/">vnodes</a> that must respond to a read request +`pw` | PW | `0` | The number of primary <a href="http://docs.basho.com/riak/kv/2.2.6/learn/concepts/vnodes/">vnodes</a> that must respond to a write request +`dw` | DW | `quorum` | The number of servers that must report that a write has been successfully written to disk +`rw` | RW | `quorum` | If R and W are undefined, this parameter will substitute for both R and W during object deletes. It is extremely unlikely that you will need to adjust this parameter. +`notfound_ok` | | `true` | This parameter determines how Riak responds if a read fails on a node. Setting to `true` (the default) is the equivalent to setting R to 1: if the first node to respond doesn't have a copy of the object, Riak will immediately return a `not found` error. If set to `false`, Riak will continue to look for the object on the number of nodes specified by N (aka `n_val`). +`basic_quorum` | | `false` | If `notfound_ok` is set to `false`, Riak will be more thorough in looking for an object on multiple nodes. Setting `basic_quorum` to `true` in this case will instruct Riak to wait for only a `quorum` of responses to return a `notfound` error instead of N responses. + +## A Primer on N, R, and W + +The most important thing to note about Riak's replication controls is +that they can be at the bucket level. You can use [bucket types]({{<baseurl>}}riak/kv/2.2.6/developing/usage/bucket-types) +to set up bucket `A` to use a particular set of replication properties +and bucket `B` to use entirely different properties. + +At the bucket level, you can choose how many copies of data you want to +store in your cluster (N, or `n_val`), how many copies you wish to read +from at one time (R, or `r`), and how many copies must be written to be +considered a success (W, or `w`). + +In addition to the bucket level, you can also specify replication +properties on the client side for any given read or write. The examples +immediately below will deal with bucket-level replication settings, but +check out the [section below](#client-level-replication-settings) +for more information on setting properties on a per-operation basis. + +The most general trade-off to be aware of when setting these values is +the trade-off between **data accuracy** and **client responsiveness**. +Choosing higher values for N, R, and W will mean higher accuracy because +more nodes are checked for the correct value on read and data is written +to more nodes upon write; but higher values will also entail degraded +responsiveness, especially if one or more nodes is failing, because Riak +has to wait for responses from more nodes. + +## N Value and Replication + +All data stored in Riak will be replicated to the number of nodes in the +cluster specified by a bucket's N value (`n_val`). The default `n_val` +in Riak is 3, which means that data stored in a bucket with the default +N will be replicated to three different nodes, thus storing three +**replicas** of the object. + +In order for this to be effective, you need at least three nodes in your +cluster. The merits of this system, however, can be demonstrated using +your local environment. + +Let's create a bucket type that sets the `n_val` for any bucket with +that type to 2. To do so, you must create and activate a bucket type +that sets this property: + +```bash +riak-admin bucket-type create n_val_equals_2 '{"props":{"n_val":2}}' +riak-admin bucket-type activate n_val_equals_2 +``` + +Now, all buckets that bear the type `n_val_equals_2` will have `n_val` +set to 2. Here's an example write: + +```curl +curl -XPUT http://localhost:8098/types/n_val_equals_2/buckets/test_bucket/keys/test_key \ + -H "Content-Type: text/plain" \ + -d "the n_val on this write is 2" +``` + +Now, whenever we write to a bucket of this type, Riak will write a +replica of the object to two different nodes. + +{{% note title="A Word on Setting the N Value" %}} +`n_val` must be greater than 0 and less than or equal to the number of actual +nodes in your cluster to get all the benefits of replication. We advise +against modifying the `n_val` of a bucket after its initial creation as this +may result in failed reads because the new value may not be replicated to all +the appropriate partitions. +{{% /note %}} + +## R Value and Read Failure Tolerance + +Read requests to Riak are sent to all N nodes that are known to be +currently responsible for the data. The R value (`r`) enables you to +specify how many of those nodes have to return a result on a given read +for the read to be considered successful. This allows Riak to provide +read availability even when nodes are down or laggy. + +You can set R anywhere from 1 to N; lower values mean faster response +time but a higher likelihood of Riak not finding the object you're +looking for, while higher values mean that Riak is more likely to find +the object but takes longer to look. + +As an example, let's create and activate a bucket type with `r` set to +`1`. All reads performed on data in buckets with this type require a +result from only one node. + +```bash +riak-admin bucket-type create r_equals_1 '{"props":{"r":1}}' +riak-admin bucket-type activate r_equals_1 +``` + +Here's an example read request using the `r_equals_1` bucket type: + +```ruby +bucket = client.bucket_type('r_equals_1').bucket('animal_facts') +obj = bucket.get('chimpanzee') +``` + +```java +Location chimpanzeeFact = + new Location(new Namespace("r_equals_1", "animal_facts"), "chimpanzee"); +FetchValue fetch = new FetchValue.Builder(chimpanzeeFact).build(); +FetchValue.Response response = client.execute(fetch); +RiakObject obj = response.getValue(RiakObject.class); +System.out.println(obj.getValue().toString()); +``` + +```php +$response = (new \Basho\Riak\Command\Builder\FetchObject($riak)) + ->buildLocation('chimpanzee', 'animal_facts', 'r_equals_1') + ->build() + ->execute(); + +echo $response->getObject()->getData(); +``` + +```python +bucket = client.bucket_type('r_equals_1').bucket('animal_facts') +bucket.get('chimpanzee') +``` + +```erlang +{ok, Obj} = riakc_pb_socket:get(Pid, + {<<"r_equals_1">>, <<"animal_facts">>}, + <<"chimpanzee">>). +``` + +```curl +curl http://localhost:8098/types/r_equals_1/buckets/animal_facts/keys/chimpanzee +``` + +As explained above, reads to buckets with the `r_equals_1` type will +typically be completed more quickly, but if the first node to respond +to a read request has yet to receive a replica of the object, Riak will +return a `not found` response (which may happen even if the object lives +on one or more other nodes). Setting `r` to a higher value will mitigate +this risk. + +## W Value and Write Fault Tolerance + +As with read requests, writes to Riak are sent to all N nodes that are +know to be currently responsible for the data. The W value (`w`) enables +you to specify how many nodes must complete a write to be considered +successful---a direct analogy to R. This allows Riak to provide write +availability even when nodes are down or laggy. + +As with R, you can set W to any value between 1 and N. The same +performance vs. fault tolerance trade-offs that apply to R apply to W. + +As an example, let's create and activate a bucket type with `w` set to +`3`: + +```bash +riak-admin bucket-type create w_equals_3 '{"props":{"w":3}}' +riak-admin activate w_equals_3 +``` + +Now, we can attempt a write to a bucket bearing the type `w_equals_3`: + +```ruby +bucket = client.bucket_type('w_equals_3').bucket('animal_facts') +obj = Riak::RObject.new(bucket, 'giraffe') +obj.raw_data = 'The species name of the giraffe is Giraffa camelopardalis' +obj.content_type = 'text/plain' +obj.store +``` + +```java +Location storyKey = + new Location(new Namespace("w_equals_3", "animal_facts"), "giraffe"); +RiakObject obj = new RiakObject() + .setContentType("text/plain") + .setValue(BinaryValue.create("The species name of the giraffe is Giraffa camelopardalis")); +StoreValue store = new StoreValue.Builder(obj) + .withLocation("giraffe") + .build(); +client.execute(store); +``` + +```php +(new \Basho\Riak\Command\Builder\StoreObject($riak)) + ->buildLocation('giraffe', 'animal_facts', 'w_equals_3') + ->build() + ->execute(); +``` + +```python +bucket = client.bucket_type('w_equals_3').bucket('animal_facts') +obj = RiakObject(client, bucket, 'giraffe') +obj.content_type = 'text/plain' +obj.data = 'The species name of the giraffe is Giraffa camelopardalis' +obj.store() +``` + +```erlang +Obj = riakc_object:new({<<"w_equals_3">>, <<"animal_facts">>}, + <<"giraffe">>, + <<"The species name of the giraffe is Giraffa camelopardalis">>, + <<"text/plain">>), +riakc_pb_socket:put(Pid, Obj). +``` + +```curl +curl -XPUT \ + -H "Content-type: text/plain" \ + -d "The species name of the giraffe is Giraffa camelopardalis" \ + http://localhost:8098/types/w_equals_3/buckets/animal_facts/keys/giraffe +``` + +Writing our `story.txt` will return a success response from Riak only if +3 nodes respond that the write was successful. Setting `w` to 1, for +example, would mean that Riak would return a response more quickly, but +with a higher risk that the write will fail because the first node it +seeks to write the object to is unavailable. + +## Primary Reads and Writes with PR and PW + +In Riak's replication model, there are N [vnodes]({{<baseurl>}}riak/kv/2.2.6/learn/glossary/#vnode), +called _primary vnodes_, that hold primary responsibility for any given +key. Riak will attempt reads and writes to primary vnodes first, but in +case of failure, those operations will go to failover nodes in order to +comply with the R and W values that you have set. This failover option +is called _sloppy quorum_. + +In addition to R and W, you can also set integer values for the *primary +read* (PR) and _primary write_ (PW) parameters that specify how many +primary nodes must respond to a request in order to report success to +the client. The default for both values is zero. + +Setting PR and/or PW to non-zero values produces a mode of operation +called _strict quorum_. This mode has the advantage that the client is +more likely to receive the most up-to-date values, but at the cost of a +higher probability that reads or writes will fail because primary vnodes +are unavailable. + +{{% note title="Note on PW" %}} +If PW is set to a non-zero value, there is a higher risk (usually very small) +that failure will be reported to the client upon write. But this does not +necessarily mean that the write has failed completely. If there are reachable +primary vnodes, those vnodes will still write the new data to Riak. When the +failed vnode returns to service, it will receive the new copy of the data via +either read repair or active anti-entropy. +{{% /note %}} + +## Durable Writes with DW + +The W and PW parameters specify how many vnodes must _respond_ to a +write in order for it to be deemed successful. What they do not specify +is whether data has actually been written to disk in the storage backend. +The DW parameters enables you to specify a number of vnodes between 1 +and N that must write the data to disk before the request is deemed +successful. The default value is `quorum` (more on symbolic names below). + +How quickly and robustly data is written to disk depends on the +configuration of your backend or backends. For more details, see the +documentation on [Bitcask][plan backend bitcask], [LevelDB][plan backend leveldb], and [multiple backends]({{<baseurl>}}riak/kv/2.2.6/setup/planning/backend/multi). + +## Delete Quorum with RW + +{{% note title="Deprecation notice" %}} +It is no longer necessary to specify an RW value when making delete requests. +We explain its meaning here, however, because RW still shows up as a property +of Riak buckets (as `rw`) for the sake of backwards compatibility. Feel free +to skip this explanation unless you are curious about the meaning of RW. +{{% /note %}} + +Deleting an object requires successfully reading an object and then +writing a tombstone to the object's key that specifies that an object +once resided there. In the course of their operation, all deletes must +comply with any R, W, PR, and PW values that apply along the way. + +If R and W are undefined, however, the RW (`rw`) value will substitute +for both R and W during object deletes. In recent versions of Riak, it +is nearly impossible to make reads or writes that do not somehow specify +oth R and W, and so you will never need to worry about RW. + +## The Implications of `notfound_ok` + +The `notfound_ok` parameter is a bucket property that determines how +Riak responds if a read fails on a node. If `notfound_ok` is set to +`true` (the default value) and the first vnode to respond doesn't have a +copy of the object, Riak will assume that the missing value is +authoritative and immediately return a `not found` result to the client. +This will generally lead to faster response times. + +On the other hand, setting `notfound_ok` to `false` means that the +responding vnode will wait for something other than a `not found` error +before reporting a value to the client. If an object doesn't exist under +a key, the coordinating vnode will wait for N vnodes to respond with +`not found` before it reports `not found` to the client. This setting +makes Riak search more thoroughly for objects but at the cost of slower +response times, a problem can be mitigated by setting `basic_quorum` to +`true`, which is discussed in the next section. + +## Early Failure Return with `basic_quorum` + +Setting `notfound_ok` to `false` on a request (or as a bucket property) +is likely to introduce additional latency. If you read a non-existent +key, Riak will check all 3 responsible vnodes for the value before +returning `not found` instead of checking just one. + +This latency problem can be mitigated by setting `basic_quorum` to +`true`, which will instruct Riak to query a quorum of nodes instead of N +nodes. A quorum of nodes is calculated as floor(N/2) + 1, meaning that 5 +nodes will produce a quorum of 3, 6 nodes a quorum of 4, 7 nodes a +quorum of 4, 8 nodes a quorum of 5, etc. + +The default for `basic_quorum` is `false`, so you will need to +explicitly set it to `true` on reads or in a bucket's properties. While +the scope of this setting is fairly narrow, it can reduce latency in +read-heavy use cases. + +## Symbolic Consistency Names + +Riak provides a number of "symbolic" consistency options for R, W, PR, +RW, and DW that are often easier to use and understand than specifying +integer values. The following symbolic names are available: + +* `all` --- All replicas must reply. This is the same as setting R, W, PR, RW, or DW equal to N. +* `one` --- This is the same as setting 1 as the value for R, W, PR, RW, or DW. +* `quorum` --- A majority of the replicas must respond, that is, half plus one. For the default N value of 3, this calculates to 2, an N value of 5 calculates to 3, and so on. +* `default` --- Uses whatever the per-bucket consistency property is for R, W, PR, RW, or DW, which may be any of the above symbolic values or an integer. + +Not submitting a value for R, W, PR, RW, or DW is the same as using +`default`. + +## Client-level Replication Settings + +Adjusting replication properties at the bucket level by [using bucket types][usage bucket types] +is how you set default properties for _all_ of a bucket's reads and +writes. But you can also set replication properties for specific reads +and writes without setting those properties at the bucket level, instead +specifying them on a per-operation basis. + +Let's say that you want to set `r` to 2 and `notfound_ok` to `true` for +just one read. We'll fetch [John Stockton](http://en.wikipedia.org/wiki/John_Stockton)'s +statistics from the `nba_stats` bucket. + +```ruby +bucket = client.bucket('nba_stats') +obj = bucket.get('john_stockton', r: 2, notfound_ok: true) +``` + +```java +Location johnStocktonStats = + new Namespace(new Namespace("nba_stats"), "john_stockton"); +FetchValue fetch = new FetchValue.Builder(johnStocktonStats) + .withOption(FetchOption.R, new Quorum(2)) + .withOption(FetchOption.NOTFOUND_OK, true) + .build(); +client.execute(fetch); +``` + +```php +(new \Basho\Riak\Command\Builder\FetchObject($riak)) + ->buildLocation('john_stockton', 'nba_stats') + ->withParameter('r', 2) + ->withParameter('notfound_ok', true) + ->build() + ->execute(); +``` + +```python +bucket = client.bucket('nba_stats') +obj = bucket.get('john_stockton', r=2, notfound_ok=True) +``` + +```erlang +{ok, Obj} = riakc_pb_socket:get(Pid, + <<"nba_stats">>, + <<"john_stockton">>, + [{r, 2}, {notfound_ok, true}]). +``` + +```curl +curl http://localhost:8098/buckets/nba_stats/keys/john_stockton?r=2¬found_ok=true +``` + +Now, let's say that you want to attempt a write with `w` set to 3 and +`dw` set to 2. As in the previous example, we'll be using the `default` +bucket type, which enables us to not specify a bucket type upon write. +Here's what that would look like: + +```ruby +bucket = client.bucket('nba_stats') +obj = Riak::RObject.new(bucket, 'michael_jordan') +obj.content_type = 'application/json' +obj.data = '{"stats":{ ... large stats object ... }}' +obj.store(w: 3, dw: 2) +``` + +```java +Location michaelJordanKey = + new Location(new Namespace("nba_stats"), "michael_jordan"); +RiakObject obj = new RiakObject() + .setContentType("application/json") + .setValue(BinaryValue.create("{'stats':{ ... large stats object ... }}")); +StoreValue store = new StoreValue.Builder(obj) + .withLocation(michaelJordanKey) + .withOption(StoreOption.W, new Quorum(3)) + .withOption(StoreOption.DW, new Quorum(2)) + .build(); +client.execute(store); +``` + +```php +(new \Basho\Riak\Command\Builder\StoreObject($riak)) + ->buildJsonObject('{'stats':{ ... large stats object ... }}') + ->buildLocation('john_stockton', 'nba_stats') + ->withParameter('w', 3) + ->withParameter('dw', 2) + ->build() + ->execute(); +``` + +```erlang +Obj = riakc_obj:new(<<"nba_stats">>, + <<"michael_jordan">>, + <<"{'stats':{ ... large stats object ... }}">>, + <<"application/json">>), +riakc_pb_socket:put(Pid, Obj). +``` + +```curl +curl -XPUT \ + -H "Content-Type: application/json" \ + -d '{"stats":{ ... large stats object ... }}' \ + http://localhost:8098/buckets/nba_stats/keys/michael_jordan?w=3&dw=2 +``` + +All of Basho's [official Riak clients]({{<baseurl>}}riak/kv/2.2.6/developing/client-libraries) enable you to +set replication properties this way. For more detailed information, +refer to the tutorial on [basic key/value operations in Riak KV]({{<baseurl>}}riak/kv/2.2.6/developing/getting-started) +or to client-specific documentation: + +* [Ruby](https://github.com/basho/riak-ruby-client/blob/master/README.md) +* [Java](http://basho.github.io/riak-java-client/2.0.0/) +* [Python](http://basho.github.io/riak-python-client/) +* [Erlang](http://basho.github.io/riak-erlang-client/) + +## Illustrative Scenarios + +In case the above explanations were a bit too abstract for your tastes, +the following table lays out a number of possible scenarios for reads +and writes in Riak and how Riak is likely to respond. Some of these +scenarios involve issues surrounding conflict resolution, vector clocks, +and siblings, so we recommend reading the [Vector Clocks]({{<baseurl>}}riak/kv/2.2.6/learn/concepts/causal-context#vector-clocks) documentation for more information. + +#### Read Scenarios + +These scenarios assume that a read request is sent to all 3 primary +vnodes responsible for an object. + +Scenario | What happens in Riak +:--------|:-------------------- +All 3 vnodes agree on the value | Once the first 2 vnodes return the value, that value is returned to the client +2 of 3 vnodes agree on the value, and those 2 are the first to reach the coordinating node | The value is returned to the client. Read repair will deal with the conflict per the later scenarios, which means that a future read may return a different value or <a href="/riak-docs/riak/kv/2.2.6/learn/concepts/causal-context#siblings">siblings</a> +2 conflicting values reach the coordinating node and <a href="/riak-docs/riak/kv/2.2.6/learn/concepts/causal-context#vector-clocks">vector clocks</a> allow for resolution | The vector clocks are used to resolve the conflict and return a single value, which is propagated via read repair to the relevant vnodes +2 conflicting values reach the coordinating node, vector clocks indicate a fork in the object history, and `allow_mult` is set to `false` | The object with the most recent timestamp is returned and propagated via read repair to the relevant vnodes +2 siblings or conflicting values reach the coordinating node, vector clocks indicate a fork in the object history, and `allow_mult` is set to `true` | All keys are returned as siblings, optionally with associated values (depending on how the request is made) + +#### Write Scenarios + +These scenarios assume that a write request is sent to all 3 primary +vnodes responsible for an object. + +Scenario | What happens in Riak +:--------|:-------------------- +A vector clock is included with the write request, and is newer than the vclock attached to the existing object | The new value is written and success is indicated as soon as 2 vnodes acknowledge the write +A vector clock is included with the write request but conflicts with the vclock attached to the existing object, with `allow_mult` set to `true` | The new value is created as a sibling for future reads +A vector clock is included with the write request but conflicts with (or is older than) the vclock attached to the existing object, with `allow_mult` set to `false` | Riak will decide which object "wins" on the basis of timestamps; no sibling will be created +A vector clock is not included with the write request and an object already exists, with `allow_mult` set to `true` | The new value is created as a sibling for future reads +A vector clock is not included with the write request and an object already exists, with `allow_mult` set to `false` | The new value overwrites the existing value + +## Screencast + +Here is a brief screencast that shows just how the N, R, and W values +function in our running 3-node Riak cluster: + +<div style="display:none" class="iframe-video" +id="http://player.vimeo.com/video/11172656"></div> + +<a href="http://vimeo.com/11172656">Tuning CAP Controls in Riak</a> from +<a href="http://vimeo.com/bashotech">Basho Technologies</a> on <a +href="http://vimeo.com">Vimeo</a>. diff --git a/content/riak/kv/2.2.6/developing/usage/search-schemas.md b/content/riak/kv/2.2.6/developing/usage/search-schemas.md new file mode 100644 index 0000000000..44607c5556 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/usage/search-schemas.md @@ -0,0 +1,495 @@ +--- +title: "Creating Search Schemas" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Creating Search Schemas" + identifier: "usage_search_schemas" + weight: 110 + parent: "developing_usage" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/advanced/search-schema + - /riak-docs/riak/kv/2.2.6/dev/advanced/search-schema +--- + +[concept clusters]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/clusters + +> **Note on Search 2.0 vs. Legacy Search** +> +> This document refers to the new Riak Search 2.0 with +[Solr](http://lucene.apache.org/solr/) integration (codenamed +Yokozuna). For information about the deprecated Riak Search, visit [the old Using Riak Search docs](http://docs.basho.com/riak/1.4.10/dev/using/search/). + +Riak Search is built for ease of use, allowing you to write values into +Riak and query for values using Solr. Riak Search does a lot of work +under the hood to convert your values---plain text, JSON, XML, [Riak Data Types]({{<baseurl>}}riak/kv/2.2.6/developing/data-types/), and [more]({{<baseurl>}}riak/kv/2.2.6/developing/usage/custom-extractors)---into something that can be indexed and searched later. +Nonetheless, you must still instruct Riak/Solr how to index a value. Are +you providing and array of strings? An integer? A date? Is your text in +English or Russian? You can provide such instructions to Riak Search by +defining a Solr **schema**. + +## The Default Schema + +Riak Search comes bundled with a default schema named `_yz_default`. The +default schema covers a wide range of possible field types. You can find +the default schema [on GitHub](https://raw.github.com/basho/yokozuna/develop/priv/default_schema.xml). +While using the default schema provides an easy path to starting +development, we recommend that you define your own schema in production. +Take note of `dynamicField name="*"`, which is a catch-all index for any +value. Sufficiently sized objects can potentially take up tremendous +amounts of disk space, so pay special attention to those indexes. + +## Custom Schemas + +We'll show you how you can create custom schemas by way of example. +Let's say that you have already created a schema named `cartoons` in a +file named `cartoons.xml`. This would register the custom schema in Riak +Search: + +```java +import org.apache.commons.io.FileUtils; + +File xml = new File("cartoons.xml"); +String xmlString = FileUtils.readFileToString(xml); +YokozunaSchema schema = new YokozunaSchema("cartoons", xmlString); +StoreSchema storeSchemaOp = new StoreSchema.Builder(schema).build(); +client.execute(storeSchemaOp); +``` + +```ruby +schema_data = File.read("cartoons.xml") +client.create_search_schema("cartoons", schema_data) +``` + +```php +(new \Basho\Riak\Command\Builder\Search\StoreSchema($riak)) + ->withName('users') + ->withSchemaFile('path/to/file.xml') + ->build() + ->execute(); +``` + +```python +xml_file = open('cartoons.xml', 'r') +schema_data = xml_file.read() +client.create_search_schema('cartoons', schema_data) +xml_file.close() +``` + +```csharp +var xml = File.ReadAllText("cartoons.xml"); +var schema = new SearchSchema("cartoons", xml); +var rslt = client.PutSearchSchema(schema); +``` + +```javascript +var fs = require('fs'); + +fs.readFile('cartoons.xml', function (err, data) { + if (err) { + throw new Error(err); + } + + var schemaXml = data.toString('utf8')); + + var options = { + schemaName: 'blog_post_schema', + schema: schemaXml + }; + + client.storeSchema(options, function (err, rslt) { + if (err) { + throw new Error(err); + } + }); +}); +``` + +```erlang +{ok, SchemaData} = file:read_file("cartoons.xml"), +riakc_pb_socket:create_search_schema(Pid, <<"cartoons">>, SchemaData). +``` + +```curl +curl -XPUT http://localhost:8098/search/schema/cartoons \ + -H 'Content-Type:application/xml' \ + --data-binary @cartoons.xml +``` + +## Creating a Custom Schema + +The first step in creating a custom schema is to define exactly what +fields you must index. Part of that step is understanding how Riak +Search extractors function. + +### Extractors + +In Riak Search, extractors are modules responsible for pulling out a +list of fields and values from a Riak object. How this is achieved +depends on the object's content type, but the two common cases are JSON +and XML, which operate similarly. Our examples here will use JSON. + +The following JSON object represents the character +[Lion-o](http://en.wikipedia.org/wiki/List_of_ThunderCats_characters#Lion-O) +from the cartoon Thundercats. He has a name and age, he's the team +leader, and he has a list of aliases in other languages. + +```json +{ + "name":"Lion-o", + "age":30, + "leader":true, + "aliases":[ + {"name":"León-O", "desc_es":"Señor de los ThunderCats"}, + {"name":"Starlion", "desc_fr":"Le jeune seigneur des Cosmocats"}, + ] +} +``` + +The extractor will flatten the above objects into a list of field/value +pairs. Nested objects will be separated with a dot (`.`) and arrays will +simply repeat the fields. The above object will be extracted to the +following list of Solr document fields. + +``` +name=Lion-o +age=30 +leader=true +aliases.name=León-O +aliases.desc_es=Señor de los ThunderCats +aliases.name=Starlion +aliases.desc_fr=Le jeune seigneur des Cosmocats +``` + +This means that our schema should handle `name`, `age`, `leader`, +`aliases.name` (a `dot` is a valid field character), and +`aliases.desc_*` which is a description in the given language of the +suffix (Spanish and French). + +### Required Schema Fields + +Solr schemas can be very complex, containing many types and analyzers. +Refer to the [Solr 4.7 reference +guide](http://archive.apache.org/dist/lucene/solr/ref-guide/apache-solr-ref-guide-4.7.pdf) +for a complete list. You should be aware, however, that there are a few +fields that are required by Riak Search in order to properly distribute +an object across a [cluster][concept clusters]. These fields are all prefixed +with `_yz`, which stands for +[Yokozuna](https://github.com/basho/yokozuna), the original code name +for Riak Search. + +Below is a bare minimum skeleton Solr Schema. It won't do much for you +other than allow Riak Search to properly manage your stored objects. + +```xml +<?xml version="1.0" encoding="UTF-8" ?> +<schema name="schedule" version="1.5"> + <fields> + + <!-- All of these fields are required by Riak Search --> + <field name="_yz_id" type="_yz_str" indexed="true" stored="true" multiValued="false" required="true"/> + <field name="_yz_ed" type="_yz_str" indexed="true" stored="false" multiValued="false"/> + <field name="_yz_pn" type="_yz_str" indexed="true" stored="false" multiValued="false"/> + <field name="_yz_fpn" type="_yz_str" indexed="true" stored="false" multiValued="false"/> + <field name="_yz_vtag" type="_yz_str" indexed="true" stored="false" multiValued="false"/> + <field name="_yz_rk" type="_yz_str" indexed="true" stored="true" multiValued="false"/> + <field name="_yz_rt" type="_yz_str" indexed="true" stored="true" multiValued="false"/> + <field name="_yz_rb" type="_yz_str" indexed="true" stored="true" multiValued="false"/> + <field name="_yz_err" type="_yz_str" indexed="true" stored="false" multiValued="false"/> + </fields> + + <uniqueKey>_yz_id</uniqueKey> + + <types> + <!-- YZ String: Used for non-analyzed fields --> + <fieldType name="_yz_str" class="solr.StrField" sortMissingLast="true" /> + </types> +</schema> +``` + +If you're missing any of the above fields, Riak Search will reject your +custom schema. The value for `<uniqueKey>` _must_ be `_yz_id`. + +In the table below, you'll find a description of the various required +fields. You'll rarely need to use any fields other than `_yz_rt` (bucket +type), `_yz_rb` (bucket) and `_yz_rk` (Riak key). On occasion, `_yz_err` +can be helpful if you suspect that your extractors are failing. +Malformed JSON or XML will cause Riak Search to index a key and set +`_yz_err` to 1, allowing you to reindex with proper values later. + +Field | Name | Description +:-------|:-----|:----------- +`_yz_id` | ID | Unique identifier of this Solr document +`_yz_ed` | Entropy Data | Data related to [active anti-entropy]({{<baseurl>}}riak/kv/2.2.6/learn/concepts/active-anti-entropy) +`_yz_pn` | Partition Number | Used as a filter query parameter to remove duplicate replicas across nodes +`_yz_fpn` | First Partition Number | The first partition in this doc's preflist, used for further filtering on overlapping partitions +`_yz_vtag`| VTag | If there is a sibling, use vtag to differentiate them +`_yz_rk` | Riak Key | The key of the Riak object this doc corresponds to +`_yz_rt` | Riak Bucket Type | The bucket type of the Riak object this doc corresponds to +`_yz_rb` | Riak Bucket | The bucket of the Riak object this doc corresponds to +`_yz_err` | Error Flag | indicating if this doc is the product of a failed object extraction + +### Defining Fields + +With your required fields known and the skeleton schema elements in +place, it's time to add your own fields. Since you know your object +structure, you need to map the name and type of each field (a string, +integer, boolean, etc). + +When creating fields you can either create specific fields via the +`field` element or an asterisk (`*`) wildcard field via `dynamicField`. +Any field that matches a specific field name will win, and if not, it +will attempt to match a dynamic field pattern. + +Besides a field `type`, you also must decide if a value is to be +`indexed` (usually `true`) and `stored`. When a value is `stored` that +means that you can get the value back as a result of a query, but it +also doubles the storage of the field (once in Riak, again in Solr). If +a single Riak object can have more than one copy of the same matching +field, you also must set `multiValued` to `true`. + +```xml +<?xml version="1.0" encoding="UTF-8" ?> +<schema name="schedule" version="1.0"> + <fields> + <field name="name" type="string" indexed="true" stored="true" /> + <field name="age" type="int" indexed="true" stored="false" /> + <field name="leader" type="boolean" indexed="true" stored="false" /> + <field name="aliases.name" type="string" indexed="true" stored="true" multiValued="true" /> + <dynamicField name="*_es" type="text_es" indexed="true" stored="true" multiValued="true" /> + <dynamicField name="*_fr" type="text_fr" indexed="true" stored="true" multiValued="true" /> + + <!-- All of these fields are required by Riak Search --> + <field name="_yz_id" type="_yz_str" indexed="true" stored="true" multiValued="false" required="true"/> + <field name="_yz_ed" type="_yz_str" indexed="true" stored="false" multiValued="false"/> + <field name="_yz_pn" type="_yz_str" indexed="true" stored="false" multiValued="false"/> + <field name="_yz_fpn" type="_yz_str" indexed="true" stored="false" multiValued="false"/> + <field name="_yz_vtag" type="_yz_str" indexed="true" stored="false" multiValued="false"/> + <field name="_yz_rk" type="_yz_str" indexed="true" stored="true" multiValued="false"/> + <field name="_yz_rt" type="_yz_str" indexed="true" stored="true" multiValued="false"/> + <field name="_yz_rb" type="_yz_str" indexed="true" stored="true" multiValued="false"/> + <field name="_yz_err" type="_yz_str" indexed="true" stored="false" multiValued="false"/> + </fields> + + <uniqueKey>_yz_id</uniqueKey> +``` + +Next, take note of the types you used in the fields and ensure that each +of the field types are defined as a `fieldType` under the `types` +element. Basic types such as `string`, `boolean`, `int` have matching +Solr classes. There are dozens more types, including many kinds of +number (`float`, `tdouble`, `random`), `date` fields, and even +geolocation types. + +Besides simple field types, you can also customize analyzers for +different languages. In our example, we mapped any field that ends with +`*_es` to Spanish, and `*_de` to German. + +```xml + <types> + <!-- YZ String: Used for non-analyzed fields --> + <fieldType name="_yz_str" class="solr.StrField" sortMissingLast="true" /> + + <fieldType name="string" class="solr.StrField" sortMissingLast="true" /> + <fieldType name="boolean" class="solr.BoolField" sortMissingLast="true"/> + <fieldType name="int" class="solr.TrieIntField" precisionStep="0" positionIncrementGap="0"/> + + <!-- Spanish --> + <fieldType name="text_es" class="solr.TextField" positionIncrementGap="100"> + <analyzer> + <tokenizer class="solr.StandardTokenizerFactory"/> + <filter class="solr.LowerCaseFilterFactory"/> + <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_es.txt" format="snowball" /> + <filter class="solr.SpanishLightStemFilterFactory"/> + <!-- more aggressive: <filter class="solr.SnowballPorterFilterFactory" language="Spanish"/> --> + </analyzer> + </fieldType> + + <!-- German --> + <fieldType name="text_de" class="solr.TextField" positionIncrementGap="100"> + <analyzer> + <tokenizer class="solr.StandardTokenizerFactory"/> + <filter class="solr.LowerCaseFilterFactory"/> + <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_de.txt" format="snowball" /> + <filter class="solr.GermanNormalizationFilterFactory"/> + <filter class="solr.GermanLightStemFilterFactory"/> + <!-- less aggressive: <filter class="solr.GermanMinimalStemFilterFactory"/> --> + <!-- more aggressive: <filter class="solr.SnowballPorterFilterFactory" language="German2"/> --> + </analyzer> + </fieldType> + </types> +</schema> +``` + +### "Catch-All" Field + +Without a catch-all field, an exception will be thrown if data is +provided to index without a corresponding `<field>` element. The +following is the catch-all field from the default Yokozuna schema and +can be used in a custom schema as well. + +```xml +<dynamicField name="*" type="ignored" /> +``` + +The following is required to be a child of the `types` element in the +schema: + +```xml +<fieldtype name="ignored" stored="false" indexed="false" multiValued="true" class="solr.StrField" /> +``` + +### Dates + +The format of strings that represents a date/time is important as Solr +only understands [ISO8601 UTC date/time +values](http://lucene.apache.org/solr/4_6_1/solr-core/org/apache/solr/schema/DateField.html). +An example of a correctly formatted date/time string is +`1995-12-31T23:59:59Z`. If you provide an incorrectly formatted +date/time value, an exception similar to this will be logged to +`solr.log`: + +```log +2014-02-27 21:30:00,372 [ERROR] <qtp1481681868-421>@SolrException.java:108 org.apache.solr.common.SolrException: Invalid Date String:'Thu Feb 27 21:29:59 +0000 2014' + at org.apache.solr.schema.DateField.parseMath(DateField.java:182) + at org.apache.solr.schema.TrieField.createField(TrieField.java:611) + at org.apache.solr.schema.TrieField.createFields(TrieField.java:650) + at org.apache.solr.schema.TrieDateField.createFields(TrieDateField.java:157) + at org.apache.solr.update.DocumentBuilder.addField(DocumentBuilder.java:47) + ... + ... + ... +``` + +## Field Properties By Use Case + +Sometimes it can be tricky to decide whether a value should be `stored`, +or whether `multiValued` is allowed. This handy table from the [Solr +documentation](https://cwiki.apache.org/confluence/display/solr/Field+Properties+by+Use+Case) +may help you pick field properties. + +An entry of `true` or `false` in the table indicates that the option +must be set to the given value for the use case to function correctly. +If no entry is provided, the setting of that attribute has no impact on +the case. + +<table class="schemausecase"> +<thead> +<tr> +<th>Use Case</th> +<th><code>indexed</code></th> +<th><code>stored</code></th> +<th><code>multiValued</code></th> +<th><code>omitNorms</code></th> +<th><code>termVectors</code></th> +<th><code>termPositions</code></th> +</tr> +</thead> +<tbody> +<tr> +<td>search within field</td> +<td><code>true</code></td> +<td></td> +<td></td> +<td></td> +<td></td> +<td></td> +</tr> +<tr> +<td>retrieve contents</td> +<td></td> +<td><code>true</code></td> +<td></td> +<td></td> +<td></td> +<td></td> +</tr> +<tr> +<td>use as unique key</td> +<td><code>true</code></td> +<td></td> +<td><code>false</code></td> +<td></td> +<td></td> +<td></td> +</tr> +<tr> +<td>sort on field</td> +<td><code>true</code></td> +<td></td> +<td><code>false</code></td> +<td><code>true</code>[1](#notes)</td> +<td></td> +<td></td> +</tr> +<tr> +<td>use field boosts[5](#notes)</td> +<td></td> +<td></td> +<td></td> +<td><code>false</code></td> +<td></td> +<td></td> +</tr> +<tr> +<td>document boosts affect searches within field</td> +<td></td> +<td></td> +<td></td> +<td><code>false</code></td> +<td></td> +<td></td> +</tr> +<tr> +<td>highlighting</td> +<td><code>true</code>[4](#notes)</td> +<td><code>true</code></td> +<td></td> +<td></td> +<td>[2](#notes)</td> +<td><code>true</code>[3](#notes)</td> +</tr> +<tr> +<td>faceting[5](#notes)</td> +<td><code>true</code></td> +<td></td> +<td></td> +<td></td> +<td></td> +<td></td> +</tr> +<tr> +<td>add multiple values, maintaining order</td> +<td></td> +<td></td> +<td><code>true</code></td> +<td></td> +<td></td> +<td></td> +</tr> +<tr> +<td>field length affects doc score</td> +<td></td> +<td></td> +<td></td> +<td><code>false</code></td> +<td></td> +<td></td> +</tr> +<tr> +<td>MoreLikeThis[5](#notes)</td> +<td></td> +<td></td> +<td></td> +<td></td> +<td><code>true</code>[6](#notes)</td> +<td></td> +</tr> +</tbody></table> + +```erlang +{analyzer_factory, {erlang, text_analyzers, noop_analyzer_factory}}} +``` diff --git a/content/riak/kv/2.2.6/developing/usage/search.md b/content/riak/kv/2.2.6/developing/usage/search.md new file mode 100644 index 0000000000..4556ee9b92 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/usage/search.md @@ -0,0 +1,1451 @@ +--- +title: "Using Search" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Searching" + identifier: "usage_searching" + weight: 105 + parent: "developing_usage" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/using/search + - /riak-docs/riak/kv/2.2.6/dev/using/search +--- + +[usage search schema]: ../search-schemas/ +[bucket types]: ../bucket-types/ + +## Setup + +Riak search 2.0 is an integration of Solr (for indexing and querying) +and Riak (for storage and distribution). There are a few points of +interest that a user of Riak search will have to keep in mind in order +to properly store and later query for values. + +1. **Schemas** explain to Solr how to index fields +2. **Indexes** are named Solr indexes against which you will query +3. **Bucket-index association** signals to Riak *when* to index values + (this also includes bucket type-index association) + +{{% note %}} +Riak search uses active anti-entropy (AAE) to ensure that the data is +consistent between the Riak backends and the Solr indexes. When using +Riak search, you should not disable AAE without understanding the risks +of divergence between the data in the Riak backends and the Solr indexes +and how that can impact your application. More information about how +Riak search uses AAE is in the +[Riak search reference](../../../using/reference/search/#active-anti-entropy-aae). +{{% /note %}} + +Riak Search must first be configured with a Solr schema so that Solr +knows how to index value fields. If you don't define one, you're +provided with a default schema named `_yz_default`, which can be found +[on +GitHub](https://raw.githubusercontent.com/basho/yokozuna/develop/priv/default_schema.xml). + +The examples in this document will presume the default. You can read +more about creating custom schemas in [Search Schema][usage search schema], which you'll likely want to use in a production environment. + +Next, you must create a named Solr index through Riak Search. This index +represents a collection of similar data that you connect with to perform +queries. When creating an index, you can optionally provide a schema. If +you do not, the default schema will be used. Here we'll `curl` create an +index named `famous` with the default schema. + +Both schema and index creation will be covered immediately below. + +{{% note title="Note on index names" %}} +Note that index names may only be +[ASCII](http://en.wikipedia.org/wiki/ASCII) values from 32-127 (spaces, +standard punctuation, digits, and word characters). This may change in +the future to allow full [Unicode](http://en.wikipedia.org/wiki/Unicode) +support. +{{% /note %}} + +All `curl` examples in this document assume that you have set an +environment variable named `RIAK_HOST`, which points to a Riak base URL, +such as `http://localhost:8098`. The appropriate value for `RIAK_HOST` +will depend on your [configuration]({{<baseurl>}}riak/kv/2.2.6/configuring/reference#client-interfaces). + +## Creating an Index + +Let's start by creating an index called `famous` that uses the default +schema. + +```java +YokozunaIndex famousIndex = new YokozunaIndex("famous"); +StoreIndex storeIndex = + new StoreIndex.Builder(famousIndex).build(); +client.execute(storeIndex); +``` + +```ruby +client.create_search_index('famous') +``` + +```php +$response = (new \Basho\Riak\Command\Builder\Search\StoreIndex($riak)) + ->withName('famouse') + ->build() + ->execute(); +``` + +```python +client.create_search_index('famous') +``` + +```csharp +var idx = new SearchIndex("famous"); +var rslt = client.PutSearchIndex(idx); +``` + +```javascript +var storeIndex_cb = function (err, rslt) { + if (err) { + throw new Error(err); + } + if (!rslt) { + // error... + } +}; + +var store = new Riak.Commands.YZ.StoreIndex.Builder() + .withIndexName("famous") + .withCallback(storeIndex_cb) + .build(); + +client.execute(store); +``` + +```erlang +riakc_pb_socket:create_search_index(Pid, <<"famous">>). +``` + +```golang +cmd, err := riak.NewStoreIndexCommandBuilder(). + WithIndexName("famous"). + WithTimeout(time.Second * 30). + Build() +if err != nil { + return err +} + +err = cluster.Execute(cmd) +``` + +```curl +export RIAK_HOST="http://localhost:8098" + +curl -XPUT $RIAK_HOST/search/index/famous +``` + +> **Getting started with Riak clients** +> +> If you are connecting to Riak using one of Basho's official [client libraries]({{<baseurl>}}riak/kv/2.2.6/developing/client-libraries), you can find more information about getting started with your client in the [Developing with Riak KV: Getting Started]({{<baseurl>}}riak/kv/2.2.6/developing/getting-started) section. + + +Note that the above command is exactly the same as the following, which +explicitly defines the default schema. + +```java +YokozunaIndex famousIndex = new YokozunaIndex("famous", "_yz_default"); +StoreIndex storeIndex = new StoreIndex.Builder(famousIndex) + .build(); +client.execute(storeIndex); +``` + +```ruby +client.create_search_index("famous", "_yz_default") +``` + +```php +$response = (new \Basho\Riak\Command\Builder\Search\StoreIndex($riak)) + ->withName('scores') + ->usingSchema('_yz_default') + ->build() + ->execute(); +``` + +```python +client.create_search_index('famous', '_yz_default') +``` + +```csharp +var idx = new SearchIndex("famous", "_yz_default"); +var rslt = client.PutSearchIndex(idx); +``` + +```javascript +var store = new Riak.Commands.YZ.StoreIndex.Builder() + .withIndexName("famous") + .withSchemaName("_yz_default") + .withCallback(storeIndex_cb) + .build(); + +client.execute(store); +``` + +```erlang +riakc_pb_socket:create_search_index(Pid, <<"famous">>, <<"_yz_default">>, []). +``` + +```golang +cmd, err := riak.NewStoreIndexCommandBuilder(). + WithIndexName("famous"). + WithSchemaName("_yz_default"). + WithTimeout(time.Second * 30). + Build() +if err != nil { + return err +} + +err = cluster.Execute(cmd) +``` + +```curl +curl -XPUT $RIAK_HOST/search/index/famous \ + -H 'Content-Type: application/json' \ + -d '{"schema":"_yz_default"}' +``` + +## Associating an Index + +The last set-up item that you need to perform is to associate your Solr index +with either a [bucket type][bucket types] or a custom bucket. You +only need do this once per bucket type, and all buckets within that type +will use the same Solr index. For example, to associate a bucket type +named `animals` with the `famous` index, you can set the bucket type +property `search_index` to `animals`. If a Solr index is to be used by +only *one* Riak bucket, you can set the `search_index` property on that +bucket. If more than one bucket is to share a Solr index, a bucket type +should be used. More on bucket types in the section directly below. + +### Associating via Bucket Type + +We suggest that you use [bucket +types][bucket types] to namespace and configure all buckets you +use. Bucket types have a lower overhead within the cluster than the +default bucket namespace but require an additional set-up step on the +command line. + +When creating a new bucket type, you can create a bucket type without +any properties and set individual buckets to be indexed. The step below +creates and activates the bucket type: + +```bash +riak-admin bucket-type create animals '{"props":{}}' +riak-admin bucket-type activate animals +``` + +And this step applies the index to the `cats` bucket, which bears the +`animals` bucket type we just created and activated: + +```curl +curl -XPUT $RIAK_HOST/types/animals/buckets/cats/props \ + -H 'Content-Type: application/json' \ + -d '{"props":{"search_index":"famous"}}' +``` + +Another possibility is to set the `search_index` as a default property +of the bucket type. This means _any_ bucket under that type will +inherit that setting and have its values indexed. + +```bash +riak-admin bucket-type create animals '{"props":{"search_index":"famous"}}' +riak-admin bucket-type activate animals +``` + +If you ever need to turn off indexing for a bucket, set the +`search_index` property to the `_dont_index_` sentinel value. + +### Associating an Index via Custom Bucket Properties + +Although we recommend that you use all new buckets under a bucket type, +if you have existing data with a type-free bucket (i.e. under the +default bucket type) you can set the `search_index` property for a +specific bucket. + +```java +Namespace catsBucket = new Namespace("cats"); +StoreBucketPropsOperation storePropsOp = new StoreBucketPropsOperation.Builder(catsBucket) + .withSearchIndex("famous") + .build(); +client.execute(storePropsOp); +``` + +```ruby +bucket = client.bucket('cats') +bucket.properties = {'search_index' => 'famous'} +``` + +```php +(new \Basho\Riak\Command\Builder\Search\AssociateIndex($riak)) + ->withName('famous') + ->buildBucket('cats') + ->build() + ->execute(); +``` + +```python +bucket = client.bucket('cats') +bucket.set_properties({'search_index': 'famous'}) +``` + +```csharp +var properties = new RiakBucketProperties(); +properties.SetSearchIndex("famous"); +var rslt = client.SetBucketProperties("cats", properties); +``` + +```javascript +var bucketProps_cb = function (err, rslt) { + if (err) { + throw new Error(err); + } + // success +}; + +var store = new Riak.Commands.KV.StoreBucketProps.Builder() + .withBucket("cats") + .withSearchIndex("famous") + .withCallback(bucketProps_cb) + .build(); + +client.execute(store); +``` + +```erlang +riakc_pb_socket:set_search_index(Pid, <<"cats">>, <<"famous">>). +``` + +```golang +cmd, err := riak.NewStoreBucketPropsCommandBuilder(). + WithBucketType("animals"). + WithBucket("cats"). + WithSearchIndex("famous"). + Build() +if err != nil { + return err +} + +err = cluster.Execute(cmd) +``` + +```curl +curl -XPUT $RIAK_HOST/buckets/cats/props \ + -H'content-type:application/json' \ + -d'{"props":{"search_index":"famous"}}' +``` + +Once you have created the index association, any new data will be indexed on +ingest according to your schema. + +## Riak Search Security Setup + +[Security]({{<baseurl>}}riak/kv/2.2.6/using/security/) is a new feature as of +Riak 2.0 that lets an administrator limit access to certain resources. +In the case of search, your options are to limit administration of +schemas or indexes (the `search.admin` permission) to certain users, and +to limit querying (the `search.query` permission) to any index or to a +specific index. The example below shows the various options. + +```bash +riak-admin security grant search.admin on schema to username +riak-admin security grant search.admin on index to username +riak-admin security grant search.query on index to username +riak-admin security grant search.query on index famous to username +``` + +Those permissions can also be revoked: + +```bash +riak-admin security revoke search.admin on schema from username +riak-admin security revoke search.admin on index from username +riak-admin security revoke search.query on index from username +riak-admin security revoke search.query on index famous from username +``` + +## Indexing Values + +> **Note on indexing and lag times** +> +> There is typically a one-second delay between storing an object in Riak +and that object being available in Search queries. You should take this +into account when writing Riak client tests, benchmarking, and so on. +More information can be found in the [Solr +documentation](http://wiki.apache.org/solr/SolrPerformanceFactors). + +With a Solr schema, index, and association in place (and possibly a +security setup as well), we're ready to start using Riak Search. First, +populate the `cat` bucket with values, in this case information about +four cats: Liono, Cheetara, Snarf, and Panthro. + +Depending on the driver you use, you may have to specify the content +type, which for this example is `application/json`. In the case of Ruby +and Python the content type is automatically set for you based on the +object given. + +```java +Namespace animalsBucket = new Namespace("animals"); +String json = "application/json"; + +RiakObject liono = new RiakObject() + .setContentType(json) + .setValue(BinaryValue.create("{\"name_s\":\"Lion-o\",\"age_i\":30,\"leader_b\":true}")); +RiakObject cheetara = new RiakObject() + .setContentType(json) + .setValue(BinaryValue.create("{\"name_s\":\"Cheetara\",\"age_i\":30,\"leader_b\":false}")); +RiakObject snarf = new RiakObject() + .setContentType(json) + .setValue(BinaryValue.create("{\"name_s\":\"Snarf\",\"age_i\":43,\"leader_b\":false}")); +RiakObject panthro = new RiakObject() + .setContentType(json) + .setValue(BinaryValue.create("{\"name_s\":\"Panthro\",\"age_i\":36,\"leader_b\":false}")); +Location lionoLoc = new Location(animalsBucket, "liono"); +Location cheetaraLoc = new Location(animalsBucket, "cheetara"); +Location snarfLoc = new Location(animalsBucket, "snarf"); +Location panthroLoc = new Location(animalsBucket, "panthro"); + +StoreValue lionoStore = new StoreValue.Builder(liono).withLocation(lionoLoc).build(); +// The other StoreValue operations can be built the same way + +client.execute(lionoStore); +// The other storage operations can be performed the same way +``` + +```ruby +bucket = client.bucket_type('animals').bucket("cats") + +cat = bucket.get_or_new("liono") +cat.data = {"name_s" => "Lion-o", "age_i" => 30, "leader_b" => true} +cat.store + +cat = bucket.get_or_new("cheetara") +cat.data = {"name_s" => "Cheetara", "age_i" => 28, "leader_b" => false} +cat.store + +cat = bucket.get_or_new("snarf") +cat.data = {"name_s" => "Snarf", "age_i" => 43} +cat.store + +cat = bucket.get_or_new("panthro") +cat.data = {"name_s" => "Panthro", "age_i" => 36} +cat.store +``` + +```php +$bucket = new \Basho\Riak\Bucket('cats', 'animals'); + +$storeObjectBuilder = (new \Basho\Riak\Command\Builder\StoreObject($riak)) + ->withLocation(new \Basho\Riak\Location('liono', $bucket)) + ->buildJsonObject(['name_s' => 'Lion-o', 'age_i' => 30, 'leader_b' => true]); + +$storeObjectBuilder->build()->execute(); + +$storeObjectBuilder->withLocation(new \Basho\Riak\Location('cheetara', $bucket)) + ->buildJsonObject(['name_s' => 'Cheetara', 'age_i' => 28, 'leader_b' => false]); + +$storeObjectBuilder->build()->execute(); + +$storeObjectBuilder->withLocation(new \Basho\Riak\Location('snarf', $bucket)) + ->buildJsonObject(['name_s' => 'Snarf', 'age_i' => 43]); + +$storeObjectBuilder->build()->execute(); + +$storeObjectBuilder->withLocation(new \Basho\Riak\Location('panthro', $bucket)) + ->buildJsonObject(['name_s' => 'Panthro', 'age_i' => 36]); + +$storeObjectBuilder->build()->execute(); +``` + +```python +bucket = client.bucket_type('animals').bucket('cats') + +cat = bucket.new('liono', {'name_s': 'Lion-o', 'age_i': 30, 'leader_b': True}) +cat.store() + +cat = bucket.new('cheetara', {'name_s':'Cheetara', 'age_i':28, 'leader_b': True}) +cat.store() + +cat = bucket.new('snarf', {'name_s':'Snarf', 'age_i':43}) +cat.store() + +cat = bucket.new('panthro', {'name_s':'Panthro', 'age_i':36}) +cat.store() +``` + +```csharp +var lionoId = new RiakObjectId("animals", "cats", "liono"); +var lionoObj = new { name_s = "Lion-o", age_i = 30, leader = true }; +var lionoRiakObj = new RiakObject(lionoId, lionoObj); + +var cheetaraId = new RiakObjectId("animals", "cats", "cheetara"); +var cheetaraObj = new { name_s = "Cheetara", age_i = 30, leader = false }; +var cheetaraRiakObj = new RiakObject(cheetaraId, cheetaraObj); + +var snarfId = new RiakObjectId("animals", "cats", "snarf"); +var snarfObj = new { name_s = "Snarf", age_i = 43, leader = false }; +var snarfRiakObj = new RiakObject(snarfId, snarfObj); + +var panthroId = new RiakObjectId("animals", "cats", "panthro"); +var panthroObj = new { name_s = "Panthro", age_i = 36, leader = false }; +var panthroRiakObj = new RiakObject(panthroId, panthroObj); + +var rslts = client.Put(new[] { + lionoRiakObj, cheetaraRiakObj, snarfRiakObj, panthroRiakObj +}); +``` + +```javascript +function store_cb(err, rslt, async_cb) { + if (err) { + throw new Error(err); + } + async_cb(null, rslt); +} + +var objs = [ + [ 'liono', { name_s: 'Lion-o', age_i: 30, leader: true } ], + [ 'cheetara', { name_s: 'Cheetara', age_i: 30, leader: false } ], + [ 'snarf', { name_s: 'Snarf', age_i: 43, leader: false } ], + [ 'panthro', { name_s: 'Panthro', age_i: 36, leader: false } ], +]; + +var storeFuncs = []; +objs.forEach(function (o) { + var storeFunc = function (async_cb) { + var key = o[0]; + var value = o[1]; + var riakObj = new Riak.Commands.KV.RiakObject(); + riakObj.setContentType('application/json'); + riakObj.setBucketType('animals'); + riakObj.setBucket('cats'); + riakObj.setKey(key); + riakObj.setValue(value); + client.storeValue({ value: riakObj }, function (err, rslt) { + store_cb(err, rslt, async_cb); + }); + }; + storeFuncs.push(storeFunc); +}); + +async.parallel(storeFuncs, function (err, rslts) { + if (err) { + throw new Error(err); + } + // NB: all objects stored and indexed... +}); +``` + +```erlang +CO = riakc_obj:new({<<"animals">>, <<"cats">>}, <<"liono">>, + <<"{\"name_s\":\"Lion-o\", \"age_i\":30, \"leader_b\":true}">>, + "application/json"), +riakc_pb_socket:put(Pid, CO), + +C1 = riakc_obj:new({<<"animals">>, <<"cats">>}, <<"cheetara">>, + <<"{\"name_s\":\"Cheetara\", \"age_i\":28, \"leader_b\":false}">>, + "application/json"), +riakc_pb_socket:put(Pid, C1), + +C2 = riakc_obj:new({<<"animals">>, <<"cats">>}, <<"snarf">>, + <<"{\"name_s\":\"Snarf\", \"age_i\":43}">>, + "application/json"), +riakc_pb_socket:put(Pid, C2), + +C3 = riakc_obj:new({<<"animals">>, <<"cats">>}, <<"panthro">>, + <<"{\"name_s\":\"Panthro\", \"age_i\":36}">>, + "application/json"), +riakc_pb_socket:put(Pid, C3), +``` + +```golang +o1 := &riak.Object{ + Key: "liono", + Value: []byte("{\"name_s\":\"Lion-o\",\"age_i\":30,\"leader_b\":true}"), +} +o2 := &riak.Object{ + Key: "cheetara", + Value: []byte("{\"name_s\":\"Cheetara\",\"age_i\":30,\"leader_b\":false}"), +} +o3 := &riak.Object{ + Key: "snarf", + Value: []byte("{\"name_s\":\"Snarf\",\"age_i\":43,\"leader_b\":false}"), +} +o4 := &riak.Object{ + Key: "panthro", + Value: []byte("{\"name_s\":\"Panthro\",\"age_i\":36,\"leader_b\":false}"), +} + +objs := [...]*riak.Object{o1, o2, o3, o4} + +wg := &sync.WaitGroup{} +for _, obj := range objs { + obj.ContentType = "application/json" + obj.Charset = "utf-8" + obj.ContentEncoding = "utf-8" + + cmd, err := riak.NewStoreValueCommandBuilder(). + WithBucketType("animals"). + WithBucket("cats"). + WithContent(obj). + Build() + if err != nil { + return err + } + + args := &riak.Async{ + Command: cmd, + Wait: wg, + } + if err := cluster.ExecuteAsync(args); err != nil { + return err + } +} + +wg.Wait() +``` + +```curl +curl -XPUT $RIAK_HOST/types/animals/buckets/cats/keys/liono \ + -H 'Content-Type: application/json' \ + -d '{"name_s":"Lion-o", "age_i":30, "leader_b":true}' + +curl -XPUT $RIAK_HOST/types/animals/buckets/cats/keys/cheetara \ + -H 'Content-Type: application/json' \ + -d '{"name_s":"Cheetara", "age_i":28, "leader_b":false}' + +curl -XPUT $RIAK_HOST/types/animals/buckets/cats/keys/snarf \ + -H 'Content-Type: application/json' \ + -d '{"name_s":"Snarf", "age_i":43}' + +curl -XPUT $RIAK_HOST/types/animals/buckets/cats/keys/panthro \ + -H 'Content-Type: application/json' \ + -d '{"name_s":"Panthro", "age_i":36}' +``` + +If you've used Riak before, you may have noticed that this is no +different from storing values without Riak Search. That's because we +designed Riak Search with the following design goal in mind: + +#### Write it like Riak, query it like Solr + +But how does Riak Search know how to index values, given that you can +store opaque values in Riak? For that, we employ extractors. + +## Extractors + +Extractors are modules in Riak that accept a Riak value with a certain +content type and convert it into a list of fields that can be indexed by +Solr. This is done transparently and automatically as part of the +indexing process. You can even create your own [custom extractors]({{<baseurl>}}riak/kv/2.2.6/developing/usage/custom-extractors). + +Our current example uses the JSON extractor, but Riak Search also +extracts indexable fields from the following content types: + +* JSON (`application/json`) +* XML (`application/xml`, `text/xml`) +* Plain text (`text/plain`) +* [Riak Data Types]({{<baseurl>}}riak/kv/2.2.6/developing/data-types/) + * counter (`application/riak_counter`) + * map (`application/riak_map`) + * set (`application/riak_set`) +* noop (unknown content type) + +More on Riak Data Types can be found in [Riak Data Types and Search]({{<baseurl>}}riak/kv/2.2.6/developing/usage/searching-data-types). + +In the examples we've seen, the JSON field `name_s` is translated to a +Solr index document field insert. Solr will index any field that it +recognizes, based on the index's schema. The default schema +(`_yz_default`) uses the suffix to decide the field type (`_s` +represents a string, `_i` is an integer, `_b` is binary and so on). + +If the content type allows for nested values (e.g. JSON and XML), the +extractors will flatten each field, separated by dots. For example, if +you have this XML: + +```xml +<person> + <pets> + <pet> + <name_s>Spot</name_s> + </pet> + </pets> +</person> +``` + +The extractor will convert it to the Solr field `person.pets.pet.name_s` +with value `Spot`. Lists of values are assumed to be Solr multi-valued +fields. + +```json +{"people_ss":["Ryan", "Eric", "Brett"]} +``` + +The above JSON will insert a list of three values into Solr to be +indexed: `people_ss=Ryan`, `people_ss=Eric`, `people_ss=Brett`. + +You can also create your own custom extractors if your data doesn't fit +one of the default types. A full tutorial can be found in [Custom Search Extractors]({{<baseurl>}}riak/kv/2.2.6/developing/usage/custom-extractors). + +### Automatic Fields + +When a Riak object is indexed, Riak Search automatically inserts a few +extra fields as well. These are necessary for a variety of technical +reasons, and for the most part you don't need to think about them. +However, there are a few fields which you may find useful: + +- `_yz_rk` (Riak key) +- `_yz_rt` (Riak bucket type) +- `_yz_rb` (Riak bucket) +- `_yz_err` (extraction error) + +You can query on the basis of these fields, just like any other normal +Solr fields. Most of the time, however, you'll use `_yz_rk` as a query +result, which tells you the Riak key that matches the query you just +ran. Let's see this in detail by running some queries in the next +section. + +## Querying + +After the schema, index, association, and population/extraction/indexing +are taken care of, you can get down to the fun part of querying your +data. + +### Simple Query + +The basic query parameter is `q` via HTTP, or the first parameter of +your chosen driver's `search` function (there are examples from all of +our client libraries below). All distributed Solr queries are supported, +which actually includes most of the single-node Solr queries. This +example searches for all documents in which the `name_s` value begins +with `Lion` by means of a glob (wildcard) match. + +```java +SearchOperation searchOp = new SearchOperation + .Builder(BinaryValue.create("famous"), "name_s:Lion*") + .build(); +cluster.execute(searchOp); +// This will display the actual results as a List of Maps: +List<Map<String, List<String>>> results = searchOp.get().getAllResults(); +// This will display the number of results: +System.out.println(results); +``` + +```ruby +results = client.search("famous", "name_s:Lion*") +p results +p results['docs'] +``` + +```php +$response = (new \Basho\Riak\Command\Builder\Search\FetchObjects($riak)) + ->withIndexName('famous') + ->withQuery('name_s:Lion*') + ->build() + ->execute(); + +$response->getNumFound(); // 1 + +var_dump($response->getDocs()); +``` + +```python +results = client.fulltext_search('famous', 'name_s:Lion*') +print results +print results['docs'] +``` + +```csharp +var search = new RiakSearchRequest +{ + Query = new RiakFluentSearch("famous", "name_s") + .Search("Lion*") + .Build() +}; + +var rslt = client.Search(search); +RiakSearchResult searchResult = rslt.Value; +foreach (RiakSearchResultDocument doc in searchResult.Documents) +{ + var args = new[] { + doc.BucketType, + doc.Bucket, + doc.Key, + string.Join(", ", doc.Fields.Select(f => f.Value).ToArray()) + }; + Debug.WriteLine( + format: "BucketType: {0} Bucket: {1} Key: {2} Values: {3}", + args: args); +} +``` + +```javascript +function search_cb(err, rslt) { + if (err) { + throw new Error(err); + } + logger.info("docs:", JSON.stringify(rslt.docs)); +} + +var search = new Riak.Commands.YZ.Search.Builder() + .withIndexName('famous') + .withQuery('name_s:Lion*') + .withCallback(search_cb) + .build(); +client.execute(search); +``` + +```erlang +{ok, Results} = riakc_pb_socket:search(Pid, <<"famous">>, <<"name_s:Lion*">>), +io:fwrite("~p~n", [Results]), +Docs = Results#search_results.docs, +io:fwrite("~p~n", [Docs]). + +%% Please note that this example relies on an Erlang record definition +%% for the search_result record found here: +%% https://github.com/basho/riak-erlang-client/blob/master/include/riakc.hrl +``` + +```golang +cmd, err := riak.NewSearchCommandBuilder(). + WithIndexName("famous"). + WithQuery("name_s:Lion*"). + Build(); +if err != nil { + return err +} + +if err := cluster.Execute(cmd); err != nil { + return err +} + +sc := cmd.(*riak.SearchCommand) +if json, jerr := json.MarshalIndent(sc.Response.Docs, "", " "); jerr != nil { + return jerr +} else { + fmt.Println(string(json)) +} +``` + +```curl +curl "$RIAK_HOST/search/query/famous?wt=json&q=name_s:Lion*" | jsonpp +``` + +The response to a query will be an object containing details about the +response, such as a query's max score and a list of documents which +match the given query. It's worth noting two things: + +* The documents returned are Search documents (a set of Solr + field/values), not a Riak value +* The HTTP response is a direct Solr response, while the drivers use + Protocol Buffers and are encoded with different field names + +This is a common HTTP `response` value: + +```json +{ + "numFound": 1, + "start": 0, + "maxScore": 1.0, + "docs": [ + { + "leader_b": true, + "age_i": 30, + "name_s": "Lion-o", + "_yz_id": "default_cats_liono_37", + "_yz_rk": "liono", + "_yz_rt": "default", + "_yz_rb": "cats" + } + ] +} +``` + +The most important field returned is `docs`, which is the list of +objects that each contain fields about matching index documents. The +values you'll use most often are `_yz_rt` (Riak bucket type), `_yz_rb` +(Riak bucket), `_yz_rk` (Riak key), and `score` which represent the +similarity of the matching doc to the query via [Lucene +scoring](https://lucene.apache.org/core/4_6_0/core/org/apache/lucene/search/package-summary.html#scoring). + +In this example the query fields are returned because they're stored in +Solr. This depends on your schema. If they are not stored, you'll have +to perform a separate Riak GET operation to retrieve the value using the +`_yz_rk` value. + +```java +// Using the results object from above +Map<String, List<String> doc = results.get(0); +String bucketType = doc.get("_yz_rt").get(0); +String bucket = doc.get("yz_rb").get(0); +String key = doc.get("_yz_rk").get(0); +Namespace namespace = new Namespace(bucketType, bucket); +Location objectLocation = new Location(namespace, key); +FetchValue fetchOp = new FetchValue.Builder(objectLocation) + .build(); +RiakObject obj = client.execute(fetchOp).getValue(RiakObject.class); +System.out.println(obj.getValue()); + +// {"name_s": "Lion-o", "age_i": 30, "leader_b": true} +``` + +```ruby +doc = results['docs'].first +btype = Riak::BucketType.new(client, doc["_yz_rt"]) # animals +bucket = Riak::Bucket.new(client, doc["_yz_rb"]) # cats +object = bucket.get( doc["_yz_rk"] ) # liono +p object.data + +# {"name_s" => "Lion-o", "age_i" => 30, "leader_b" => true} +``` + +```php +$doc = $response->getDocs()[0]; +$btype = $doc->_yz_rt; // animals +$bucket = $doc->_yz_rb; // cats +$key = $doc->_yz_rk; // liono +$name = $doc->name_s; // Lion-o + +$object = (new \Basho\Riak\Command\Builder\FetchObject($riak)) + ->buildLocation($key, $bucket, $btype) + ->build() + ->execute() + ->getObject(); + +var_dump($object->getData()); +``` + +```python +doc = results['docs'][0] +bucket = client.bucket_type(doc['_yz_rt']).bucket(doc['_yz_rb']) # animals/cats +object = bucket.get(doc['_yz_rk']) # liono +print object.data + +# {"name_s": "Lion-o", "age_i": 30, "leader_b": true} +``` + +```csharp +RiakSearchResult searchResult = searchRslt.Value; + +RiakSearchResultDocument doc = searchResult.Documents.First(); +var id = new RiakObjectId(doc.BucketType, doc.Bucket, doc.Key); +var rslt = client.Get(id); + +RiakObject obj = rslt.Value; +Debug.WriteLine(Encoding.UTF8.GetString(obj.Value)); + +// {"name_s":"Lion-o","age_i":30,"leader_b":true} +``` + +```javascript +var doc = rslt.docs.pop(); +var args = { + bucketType: doc._yz_rt, + bucket: doc._yz_rb, + key: doc._yz_rk, + convertToJs: true +}; +client.fetchValue(args, function (err, rslt) { + if (err) { + throw new Error(err); + } + logger.info(rslt.values[0].value); +}); +``` + +```erlang +[{Index,Doc}|_] = Docs, +BType = proplists:get_value(<<"_yz_rt">>, Doc), %% <<"animals">> +Bucket = proplists:get_value(<<"_yz_rb">>, Doc), %% <<"cats">> +Key = proplists:get_value(<<"_yz_rk">>, Doc), %% <<"liono">> +{ok, Obj} = riakc_pb_socket:get(Pid, {BType, Bucket}, Key), +Val = riakc_obj:get_value(Obj), +io:fwrite("~s~n", [Val]). + +%% {"name_s":"Lion-o", "age_i":30, "leader_b":true} +``` + +```golang +doc := sc.Response.Docs[0] // NB: SearchDoc struct type + +cmd, err = riak.NewFetchValueCommandBuilder(). + WithBucketType(doc.BucketType). + WithBucket(doc.Bucket). + WithKey(doc.Key). + Build() +if err != nil { + return err +} + +if err := cluster.Execute(cmd); err != nil { + return err +} +``` + +```curl +curl $RIAK_HOST/types/animals/buckets/cats/keys/liono + +# Response: + +{"name_s":"Lion-o", "age_i":30, "leader_b":true} +``` + +This was one simple glob query example. There are many query options, a +more complete list of which can be found by digging into [searching +Solr](https://cwiki.apache.org/confluence/display/solr/Searching). Let's +look at a few others. + +### Range Queries + +Range queries are searches within a +[range](https://cwiki.apache.org/confluence/display/solr/The+Standard+Query+Parser#TheStandardQueryParser-DifferencesbetweenLuceneQueryParserandtheSolrStandardQueryParser) +of numerical or +date/[datemath](http://lucene.apache.org/solr/4_6_0/solr-core/org/apache/solr/util/DateMathParser.html) +values. + +To find the ages of all famous cats who are 30 or younger: `age_i:[0 TO +30]`. If you wanted to find all cats 30 or older, you could include a +glob as a top end of the range: `age_i:[30 TO *]`. + +```java +String index = "famous"; +String query = "age_i:[30 TO *]"; +SearchOperation searchOp = new SearchOperation + .Builder(BinaryValue.create(index), query) + .build(); +cluster.execute(searchOp); +SearchOperation.Response results = searchOp.get(); +``` + +```ruby +client.search("famous", "age_i:[30 TO *]") +``` + +```php +$response = (new \Basho\Riak\Command\Builder\Search\FetchObjects($riak)) + ->withIndexName('famous') + ->withQuery('age_i:[30 TO *]') + ->build() + ->execute(); +``` + +```python +client.fulltext_search('famous', 'age_i:[30 TO *]') +``` + +```csharp +var search = new RiakSearchRequest("famous", "age_i:[30 TO *]"); + +/* + * Fluent interface: + * + * var search = new RiakSearchRequest + * { + * Query = new RiakFluentSearch("famous", "age_i") + * .Between("30", "*") + * .Build() + * }; + */ +var rslt = client.Search(search); +``` + +```javascript +var search = new Riak.Commands.YZ.Search.Builder() + .withIndexName('famous') + .withQuery('age_i:[30 TO *]') + .withCallback(search_cb) + .build(); +client.execute(search); +``` + +```erlang +riakc_pb_socket:search(Pid, <<"famous">>, <<"age_i:[30 TO *]">>), +``` + +```golang +cmd, err := riak.NewSearchCommandBuilder(). + WithIndexName("famous"). + WithQuery("age_i:[30 TO *]"). + Build(); +if err != nil { + return err +} + +if err := cluster.Execute(cmd); err != nil { + return err +} +``` + +```curl +curl "$RIAK_HOST/search/query/famous?wt=json&q=age_i:%5B30%20TO%20*%5D" | jsonpp +``` + +<!-- TODO: pubdate:[NOW-1YEAR/DAY TO NOW/DAY+1DAY] --> + +### Boolean + +You can perform logical conjunctive, disjunctive, and negative +operations on query elements as, respectively, `AND`, `OR`, and `NOT`. +Let's say we want to see who is capable of being a US Senator (at least +30 years old, and a leader). It requires a conjunctive query: +`leader_b:true AND age_i:[25 TO *]`. + +```java +String index = "famous"; +String query = "leader_b:true AND age_i:[30 TO *]"; +Search searchOp = new Search.Builder(index, query).build(); +cluster.execute(searchOp); +SearchOperation.Response results = searchOp.get(); +``` + +```ruby +client.search("famous", "leader_b:true AND age_i:[30 TO *]") +``` + +```php +$response = (new \Basho\Riak\Command\Builder\Search\FetchObjects($riak)) + ->withIndexName('famous') + ->withQuery('leader_b:true AND age_i:[30 TO *]') + ->build() + ->execute(); +``` + +```python +client.fulltext_search('famous', 'leader_b:true AND age_i:[30 TO *]') +``` + +```csharp +var search = new RiakSearchRequest +{ + Query = new RiakFluentSearch("famous", "leader_b") + .Search("true").AndBetween("age_i", "30", "*") + .Build() +}; +``` + +```javascript +var search = new Riak.Commands.YZ.Search.Builder() + .withIndexName('famous') + .withQuery('leader_b:true AND age_i:[30 TO *]') + .withCallback(search_cb) + .build(); +client.execute(search); +``` + +```erlang +riakc_pb_socket:search(Pid, <<"famous">>, <<"leader_b:true AND age_i:[30 TO *]">>), +``` + +```golang +cmd, err := riak.NewSearchCommandBuilder(). + WithIndexName("famous"). + WithQuery("leader_b:true AND age_i:[30 TO *]"). + Build(); +if err != nil { + return err +} + +if err := cluster.Execute(cmd); err != nil { + return err +} +``` + +```curl +curl "$RIAK_HOST/search/query/famous?wt=json&q=leader_b:true%20AND%20age_i:%5B25%20TO%20*%5D" | jsonpp +``` + +### Deleting Indexes + +Indexes may be deleted if they have no buckets associated with them: + +```java +String index = "famous"; +YzDeleteIndexOperation deleteOp = new YzDeleteIndexOperation.Builder(index) + .build(); +cluster.execute(deleteOp); +``` + +```ruby +client.delete_search_index('famous') +``` + +```php +(new Command\Builder\Search\DeleteIndex($riak)) + ->withName('famous') + ->build() + ->execute(); +``` + +```python +client.delete_search_index('famous') +``` + +```csharp +var rslt = client.DeleteSearchIndex("famous"); +``` + +```javascript +function delete_cb(err, rslt) { + if (err) { + throw new Error(err); + } + if (rslt === true) { + // success + } else { + // error + } +} + +// NB: first make sure that no bucket types or buckets are using the index +var search = new Riak.Commands.YZ.DeleteIndex.Builder() + .withIndexName('famous') + .withCallback(delete_cb) + .build(); +client.execute(search); +``` + +```erlang +riakc_pb_socket:delete_search_index(Pid, <<"famous">>, []), +``` + +```golang +cmd, err := riak.NewStoreBucketPropsCommandBuilder(). + WithBucketType("animals"). + WithBucket("cats"). + WithSearchIndex("_dont_index_"). + Build() +if err != nil { + return err +} + +if err := cluster.Execute(cmd); err != nil { + return err +} + +cmd, err = riak.NewDeleteIndexCommandBuilder(). + WithIndexName("famous"). + Build() +if err != nil { + return err +} + +if err := cluster.Execute(cmd); err != nil { + return err +} +``` + +```curl +curl -XDELETE $RIAK_HOST/search/index/famous +``` + +If an index does have a bucket associated with it, then that index's +`search_index` property must be changed to either a different index name +or to the sentinel value `_dont_index_`. + +```curl +curl -XPUT $RIAK_HOST/types/animals/buckets/cats/props \ + -H 'Content-Type: application/json' \ + -d '{"props":{"search_index":"_dont_index_"}}' +``` + +#### Pagination + +A common requirement you may face is paginating searches, where an +ordered set of matching documents are returned in non-overlapping +sequential subsets (in other words, *pages*). This is easy to do with +the `start` and `rows` parameters, where `start` is the number of +documents to skip over (the offset) and `rows` are the number of results +to return in one go. + +For example, assuming we want two results per page, getting the second +page is easy, where `start` is calculated as (rows per page) * (page +number - 1). + +```java +int rowsPerPage = 2; +int page = 2; +int start = rowsPerPage * (page - 1); + +SearchOperation searchOp = new SearchOperation + .Builder(BinaryValue.create("famous"), "*:*") + .withStart(start) + .withNumRows(rowsPerPage) + .build(); +client.execute(searchOp); +StoreOperation.Response response = searchOp.get(); +``` + +```ruby +ROWS_PER_PAGE=2 +page = 2 +start = ROWS_PER_PAGE * (page - 1) + +client.search("famous", "*:*", {:start => start, :rows => ROWS_PER_PAGE}) +``` + +```php +$maxRows = 2; +$page = 2; +$start = $rowsPerPAge * (page - 1); + +(new \Basho\Riak\Command\Builder\Search\FetchObjects($riak)) + ->withIndexName('famous') + ->withQuery('*:*') + ->withMaxRows($maxRows) + ->withStartRow($start) + ->build() + ->execute(); +``` + +```python +ROWS_PER_PAGE=2 +page = 2 +start = ROWS_PER_PAGE * (page - 1) + +client.fulltext_search('famous', '*:*', start=start, rows=ROWS_PER_PAGE) +``` + +```csharp +int rowsPerPage = 2; +int page = 2; +int start = rowsPerPage * (page - 1); + +var search = new RiakSearchRequest +{ + Start = start, + Rows = rowsPerPage, + Query = new RiakFluentSearch("famous", "*") + .Search("*") + .Build(), +}; + +var rslt = client.Search(search); +``` + +```javascript +var rowsPerPage = 2; +var page = 2; +var start = rowsPerPage * (page - 1); + +var search = new Riak.Commands.YZ.Search.Builder() + .withIndexName('famous') + .withQuery('*:*') + .withStart(start) + .withNumRows(rowsPerPage) + .withCallback(search_cb) + .build(); +client.execute(search); +``` + +```erlang +-define(ROWS_PER_PAGE, 2). + +Page = 2, +Start = ?ROWS_PER_PAGE * (Page - 1), + +riakc_pb_socket:search(Pid, <<"famous">>, <<"*:*">>, [{start, Start},{rows, ?ROWS_PER_PAGE}]), +``` + +```golang +rowsPerPage := uint32(2) +page := uint32(2) +start := rowsPerPage * (page - uint32(1)) + +cmd, err := riak.NewSearchCommandBuilder(). + WithIndexName("famous"). + WithQuery("*:*"). + WithStart(start). + WithNumRows(rowsPerPage). + Build(); +if err != nil { + return err +} + +if err := cluster.Execute(cmd); err != nil { + return err +} +``` + +```curl +ROWS_PER_PAGE=2 +PAGE=2 +START=$(($ROWS_PER_PAGE * ($PAGE-1))) + +curl +curl "$RIAK_HOST/search/query/famous?wt=json&q=*:*&start=$START&rows=$ROWS_PER_PAGE" | jsonpp +``` + +### Pagination Warning + +Distributed pagination in Riak Search cannot be used reliably when +sorting on fields that can have different values per replica of the same +object, namely `score` and `_yz_id`. In the case of sorting by these +fields, you may receive redundant objects. In the case of `score`, the +top-N can return different results over multiple runs. + +If you are paginating simply to get all keys that match and don't care +about the score, then you can sort on type-bucket-key (eg. `_yz_rt asc`, +`_yz_rb asc`, `_yz_rk asc`) to get consistent results. + +If you want to sort by score without repeating results then you must set +`rows` >= `numFound`. This requires having some idea of how many rows +will match before running the query. + +[This issue](https://github.com/basho/yokozuna/issues/355) is caused by +the way Search must minimally distribute a query across multiple Solr +nodes (called a *coverage plan*) and then filter duplicate results to +retrieve a full result set. Since this plan is frequently recalculated, +successive page queries may use a different plan, and thus calculate +alternate `score`s or filter different `_yz_id` values. We have plans to +fix this shortcoming in a future version of Riak. + +### MapReduce + +Riak Search allows for piping search results as inputs for +[MapReduce]({{<baseurl>}}riak/kv/2.2.6/developing/usage/mapreduce/) jobs. This is a useful cross-section for +performing post-calculations of results or aggregations of ad-hoc +queries. The Riak Search MapReduce integration works similarly to +regular MapReduce, with the notable exception that your input is not a +bucket, but rather index and query arguments to the `yokozuna` module +and `mapred_search` function (an Erlang `module:function` pair that adds +the Riak Search hook to MapReduce). + +```json +{ + "inputs": { + "module": "yokozuna", + "function": "mapred_search", + "arg": ["famous","NOT leader_b:true"] + }, + "query": [ + { + "map": { + "language": "javascript", + "keep": false, + "source": "function(v) { return [1]; }" + } + }, + { + "reduce": { + "language": "javascript", + "keep": true, + "name": "Riak.reduceSum" + } + } + ] +} +``` + +In this example we're searching for all famous cats that are not +leaders and counting up the results using Javascript for both map and +reduce. It should return the reduced sum of `[3]`. + +```curl +curl -XPOST $RIAK_HOST/mapred \ + -H 'Content-Type: application/json' \ + -d '{"inputs":{"module":"yokozuna","function":"mapred_search","arg":["famous","NOT leader_b:true"]},"query":[{"map":{"language":"javascript","keep":false,"source":"function(v) { return [1]; }"}},{"reduce":{"language":"javascript","keep":true,"name":"Riak.reduceSum"}}]}' +``` diff --git a/content/riak/kv/2.2.6/developing/usage/searching-data-types.md b/content/riak/kv/2.2.6/developing/usage/searching-data-types.md new file mode 100644 index 0000000000..9ac124bbcb --- /dev/null +++ b/content/riak/kv/2.2.6/developing/usage/searching-data-types.md @@ -0,0 +1,1683 @@ +--- +title: "Searching with Data Types" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Searching with Data Types" + identifier: "usage_search_data_types" + weight: 111 + parent: "developing_usage" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/search/search-data-types + - /riak-docs/riak/kv/2.2.6/dev/search/search-data-types +--- + +Although [Riak Data Types]({{<baseurl>}}riak/kv/2.2.6/developing/data-types) function differently from other +Riak objects in some respects, when you're using Search you can think of +them as normal Riak objects with special metadata attached (metadata +that you don't need to worry about as a user). Riak's [counters]({{<baseurl>}}riak/kv/2.2.6/developing/data-types/#counters), [sets]({{<baseurl>}}riak/kv/2.2.6/developing/data-types/#sets), and [maps]({{<baseurl>}}riak/kv/2.2.6/developing/data-types/#maps) +can be indexed and have their contents searched just like other Riak +objects. + +## Data Type MIME Types + +Like all objects stored in Riak, Riak Data Types are assigned content +types. Unlike other Riak objects, this happens automatically. When you +store, say, a counter in Riak, it will automatically be assigned the +type `application/riak_counter`. The table below provides the full list +of content types: + +Data Type | Content Type +:---------|:------------ +Counters | `application/riak_counter` +Sets | `application/riak_set` +Maps | `application/riak_map` + +When using Search, you won't need to worry about this, as Riak Data +Types are automatically indexed on the basis of these content types. + +## Data Type Schemas + +There are two types of schemas related to Riak Data Types: + +* **Top-level schemas** relate to Data Types that are stored at the key + level (counters and sets) +* **Embedded schemas** relate to Data Types nested inside of maps + (flags, counters, registers, and sets) + +As you can see from the [default Search +schema](https://github.com/basho/yokozuna/blob/develop/priv/default_schema.xml#L96), +each of the Data Types has its own default schema, with the exception of +maps, which means that the `_yz_default` schema will automatically index +Data Types on the basis of their assigned content type. This means that +there is no extra work involved in indexing Riak Data Types. You can +simply store them and begin querying, provided that they are properly +indexed, which is covered in the [examples](#riak-data-types-and-search) section below. + +As mentioned above, there are no default schemas available for maps. +This is because maps are essentially carriers for the other Data Types. +Even when maps are embedded within other maps, all of the data that you +might wish to index and search is contained in counters, sets, +registers, and flags. + +The sections immediately below provide the default schemas for each Riak +Data Type. Because you will not need to manipulate these default schemas +to search Data Types, they are provided only for reference. + +### Top-level Schemas + +The default schema for [counters]({{<baseurl>}}riak/kv/2.2.6/developing/data-types/#counters) indexes each +counter as an integer. + +```xml +<field name="counter" type="int" indexed="true" stored="true" multiValued="false" /> +``` + +Constructing queries for counters involves prefacing the query with +`counter`. Below are some examples: + +Query | Syntax +:-----|:------ +Counters with a value over 10 | `counter:[10 TO *]` +Counters with a value below 10 and above 50 | `counter:[* TO 10] AND counter:[50 TO *]` +Counters with a value of 15 | `counter:15` +All counters within the index | `counter:*` + +The schema for [sets]({{<baseurl>}}riak/kv/2.2.6/developing/data-types/#sets) indexes each element of a set as +a string and indexes the set itself as multi-valued. + +```xml +<field name="set" type="string" indexed="true" stored="false" multiValued="true" /> +``` + +To query sets, preface the query with `set`. The table below shows some +examples: + +Query | Syntax +:-----|:------ +Sets that contain the value `apple` | `set:apple` +Sets that contain an item beginning with `level` | `set:level*` +Sets that contain both `apple` and `orange` | `set:apple AND set:orange` +All sets within the index | `set:*` + +### Embedded Schemas + +For searching within [maps]({{<baseurl>}}riak/kv/2.2.6/developing/data-types/#maps), there are four schemas +for embedded, aka dynamic, fields. Flags are indexed as booleans: + +```xml +<dynamicField name="*_flag" type="boolean" indexed="true" stored="true" multiValued="false" /> +``` + +Counters, like their top-level counterparts, are indexed as integers: + +```xml +<dynamicField name="*_counter" type="int" indexed="true" stored="true" multiValued="false" /> +``` + +Registers are indexed as strings, but unlike sets they are not +multi-valued. + +```xml +<dynamicField name="*_register" type="string" indexed="true" stored="true" multiValued="false" /> +``` + +Finally, sets at the embedded level are indexed as multi-valued strings. + +```xml +<dynamicField name="*_set" type="string" indexed="true" stored="true" multiValued="true" /> +``` + +To query embedded fields, you must provide the name of the field. The +table below provides some examples: + +Query | Syntax +:-----|:------ +Maps containing a set called `hobbies` | `hobbies_set:*` +Maps containing a `score` counter over 50 | `score_counter:[50 TO *]` +Maps containing disabled `advanced` flags | `advanced_flag:false` +Maps containing enabled `advanced` flags and `score` counters under 10 | `advanced_flag:true AND score_counter:[* TO 10]` + +You can also query maps within maps, which is covered in the **Querying +maps within maps** section below. + +## Data Types and Search Examples + +In this section, we'll start with two simple examples, one involving +counters and the other involving sets. Later on, we'll introduce a +slightly more complex map example. + +## Counters Example + +Let's say that we're storing scores in a multiplayer online game in +Riak. The game is called Boulderdash and it involves smashing digital +boulders armed with nothing but witty retorts and arcane trivia +knowledge. We'll create and activate a [bucket type]({{<baseurl>}}riak/kv/2.2.6/developing/usage/bucket-types) for [storing counters]({{<baseurl>}}riak/kv/2.2.6/developing/data-types/#counters) simply called +`counters`, like so: + +```bash +riak-admin bucket-type create counters '{"props":{"datatype":"counter"}}' +riak-admin bucket-type activate counters +``` + +Now, we'll create a search index called `scores` that uses the default +schema (as in some of the examples above): + +```java +YokozunaIndex scoresIndex = new YokozunaIndex("scores", "_yz_default"); +StoreIndex storeIndex = new StoreIndex.Builder(scoresIndex) + .build(); +client.execute(storeIndex); +``` + +```ruby +client.create_search_index('scores', '_yz_default') +``` + +```php +$response = (new \Basho\Riak\Command\Builder\Search\StoreIndex($riak)) + ->withName('scores') + ->usingSchema('_yz_default') + ->build() + ->execute(); +``` + +```python +client.create_search_index('scores', '_yz_default') +``` + +```csharp +var idx = new SearchIndex("scores", "_yz_default"); +var rslt = client.PutSearchIndex(idx); +``` + +```javascript +var options = { + schemaName: '_yz_default', + indexName: 'scores' +}; +client.storeIndex(options, function (err, rslt) { +}); +``` + +```erlang +riakc_pb_socket:create_search_index(Pid, <<"scores">>, <<"_yz_default">>, []). +``` + +```curl +curl -XPUT $RIAK_HOST/search/index/hobbies \ + -H 'Content-Type: application/json' \ + -d '{"schema":"_yz_default"}' +``` + +Now, we can modify our `counters` bucket type to associate that bucket +type with our `scores` index: + +```bash +riak-admin bucket-type update counters '{"props":{"search_index":"scores"}}' +``` + +At this point, all of the counters that we stored in any bucket with the +bucket type `counters` will be indexed in our `scores` index. So let's +start playing with some counters. All counters will be stored in the +bucket `people`, while the key for each counter will be the username of +each player: + +```java +Namespace peopleBucket = new Namespace("counters", "people"); + +Location christopherHitchensCounter = new Location(peopleBucket, "christ_hitchens"); +CounterUpdate cu = new CounterUpdate(10); +UpdateCounter update = new UpdateCounter.Builder(christopherHitchensCounter, cu) + .build(); +client.execute(update); + +Location joanRiversCounter = new Location(peopleBucket, "joan_rivers"); +CounterUpdate cu = new CounterUpdate(25); +UpdateCounter update = new UpdateCounter.Builder(joanRiversCounter, cu) + .build(); +client.execute(update); +``` + +```ruby +bucket = client.bucket('people') + +christopher_hitchens_counter = Riak::Crdt::Counter.new(bucket, 'chris_hitchens', 'counters') +christopher_hitchens_counter.increment(10) + +joan_rivers_counter = Riak::Crdt::Counter.new(bucket, 'joan_rivers', 'counters') +joan_rivers_counter.increment(25) +``` + +```php +$builder = (new \Basho\Riak\Command\Builder\IncrementCounter($riak)) + ->withIncrement(10) + ->buildLocation('chris_hitchens', 'people', 'counters'); + +$builder->build->execute(); + +$builder->withIncrement(25) + ->buildLocation('joan_rivers', 'people', 'counters') + ->build() + ->execute(); +``` + +```python +from riak.datatypes import Counter + +bucket = client.bucket_type('counters').bucket('people') + +christopher_hitchens_counter = Counter(bucket, 'chris_hitchens') +christopher_hitchens_counter.increment(10) +christopher_hitchens_counter.store() + +joan_rivers_counter = Counter(bucket, 'joan_rivers') +joan_rivers_counter.increment(25) +joan_rivers_counter.store() +``` + +```csharp +// https://github.com/basho/riak-dotnet-client/blob/develop/src/RiakClientExamples/Dev/Search/SearchDataTypes.cs + +var cmd = new UpdateCounter.Builder() + .WithBucketType("counters") + .WithBucket("people") + .WithKey("christ_hitchens") + .WithIncrement(10) + .Build(); +RiakResult rslt = client.Execute(cmd); + +cmd = new UpdateCounter.Builder() + .WithBucketType("counters") + .WithBucket("people") + .WithKey("joan_rivers") + .WithIncrement(25) + .Build(); +rslt = client.Execute(cmd); +``` + +```javascript +var funcs = [ + function (async_cb) { + var options = { + bucketType: 'counters', + bucket: 'people', + key: 'christ_hitchens', + increment: 10 + }; + + client.updateCounter(options, function (err, rslt) { + throwIfErr(err); + async_cb(); + }); + }, + function (async_cb) { + var options = { + bucketType: 'counters', + bucket: 'people', + key: 'joan_rivers', + increment: 25 + }; + + client.updateCounter(options, function (err, rslt) { + throwIfErr(err); + async_cb(); + }); + } +]; + +async.parallel(funcs, function (err, rslts) { + throwIfErr(err); +}); +``` + +```erlang +ChristopherHitchensCounter = riakc_counter:new(), +HitchensCounter1 = riakc_counter:increment(10, ChristopherHitchensCounter), +JoanRiversCounter = riakc_counter:new(), +RiversCounter1 = riakc_counter:increment(25, JoanRiversCounter), +riakc_pb_socket:update_type(Pid, + {<<"counters">>, <<"people">>}, + <<"chris_hitchens">>, + riakc_counter:to_op(HitchensCounter1)), +riakc_pb_socket:update_type(Pid, + {<<"counters">>, <<"people">>}, + <<"joan_rivers">>, + riakc_counter:to_op(RiversCounter1)). +``` + +```curl +# We do not recommend working with Riak Data Types via curl. Try using +# one of our client libraries instead. +``` + +So now we have two counters, one with a value of 10 and the other with a +value of 25. Let's query to see how many counters have a value greater +than 20, just to be sure: + +```java +String index = "scores"; +String query = "counter:[20 TO *]"; +SearchOperation searchOp = new SearchOperation.Builder(BinaryValue.create(index), query) + .build(); +cluster.execute(searchOp); +SearchOperation.Response results = searchOp.get(); +``` + +```ruby +results = client.search('scores', 'counter:[20 TO *]') +# This should return a Hash with fields like 'num_found' and 'docs' + +results['num_found'] +# 1 +``` + +```php +$response = (new \Basho\Riak\Command\Builder\Search\FetchObjects($riak)) + ->withIndexName('scores') + ->withQuery('counter:[20 TO *]') + ->build() + ->execute(); + +$response->getNumFound(); // 1 +``` + +```python +results = client.fulltext_search('scores', 'counter:[20 TO *]') +# This should return a dict with fields like 'num_found' and 'docs' + +results['num_found'] +# 1 +``` + +```csharp +var search = new RiakSearchRequest("scores", "counter:[20 TO *]"); +var rslt = client.Search(search); +RiakSearchResult searchResult = rslt.Value; +Console.WriteLine("Num found: {0}", searchResult.NumFound); +``` + +```javascript +function search_cb(err, rslt) { + logger.info("counter numFound: '%d', docs: '%s'", + rslt.numFound, JSON.stringify(rslt.docs)); + + var doc = rslt.docs[0]; + var key = doc['_yz_rk']; + var bucket = doc['_yz_rb']; + var bucketType = doc['_yz_rt']; +} + +var searchCmd = new Riak.Commands.YZ.Search.Builder() + .withIndexName('scores') + .withQuery('counter:[20 TO *]') + .withCallback(search_cb) + .build(); + +client.execute(searchCmd); +``` + +```erlang +{ok, Results} = riakc_pb_socket:search(Pid, <<"scores">>, <<"counter:[20 TO *]">>), +NumberFound = Results#search_results.num_found. +%% 1 +``` + +```curl +curl "$RIAK_HOST/search/query/scores?wt=json&q=counter:[20 TO *]" | jsonpp +``` + +And there we are: only one of our two stored sets has a value over 20. +To find out which set that is, we can dig into our results: + +```java +// Using the "results" object from above: +int numberFound = results.numResults(); +Map<String, List<String>> foundObject = results.getAllResults().get(0); +String key = foundObject.get("_yz_rk").get(0); // "joan_rivers" +String bucket = foundObject.get("_yz_rb").get(0); // "people" +String bucketType = foundObject.get("_yz_rt").get(0); // "counters" +``` + +```ruby +doc = results['docs'][0] + +# The key +doc['_yz_rk'] # 'joan_rivers' + +# The bucket +doc['_yz_rb'] # 'people' + +# The bucket type +doc['_yz_rt'] # 'counters' +``` + +```php +$doc = $response->getDocs()[0]; + +# The key +$doc['_yz_rk'] # 'joan_rivers' + +# The bucket +$doc['_yz_rb'] # 'people' + +# The bucket type +$doc['_yz_rt'] # 'counters' +``` + +```python +doc = results['docs'][0] + +# The key +doc['_yz_rk'] # 'joan_rivers' + +# The bucket +doc['_yz_rb'] # 'people' + +# The bucket type +doc['_yz_rt'] # 'counters' +``` + +```csharp +var search = new RiakSearchRequest("scores", "counter:[20 TO *]"); +var rslt = client.Search(search); + +RiakSearchResult searchResult = rslt.Value; +Console.WriteLine("Num found: {0}", searchResult.NumFound); + +var firstDoc = searchResult.Documents.First(); +Console.WriteLine("Key: {0} Bucket: {1} Type: {2}", + firstDoc.Key, firstDoc.Bucket, firstDoc.BucketType); +``` + +```javascript +var doc = rslt.docs[0]; + +var key = doc['_yz_rk']; +var bucket = doc['_yz_rb']; +var bucketType = doc['_yz_rt']; +``` + +```erlang +Doc = lists:nth(1, Docs), +Key = proplists:get_value(<<"_yz_rk">>, Doc), +Bucket = proplists:get_value(<<"_yz_rb">>, Doc), +BucketType = proplists:get_value(<<"_yz_rt", Doc). +``` + +```curl +# Use the JSON object from above to locate bucket, key, and bucket type +# information +``` + +Alternatively, we can see how many counters have values below 15: + +```java +String index = "scores"; +String query = "counter:[* TO 15]"; +SearchOperation searchOp = new SearchOperation + .Builder(BinaryValue.create("scores"), "counter:[* TO 15]") + .build(); +cluster.execute(searchOp); +SearchOperation.Response results = searchOp.get(); +``` + +```ruby +results = client.search('scores', 'counter:[* TO 15]') +``` + +```php +$response = (new \Basho\Riak\Command\Builder\Search\FetchObjects($riak)) + ->withIndexName('scores') + ->withQuery('counter:[* TO 15]') + ->build() + ->execute(); + +$response->getNumFound(); // 1 +``` + +```python +results = client.fulltext_search('scores', 'counter:[* TO 15]') +``` + +```csharp +var search = new RiakSearchRequest("scores", "counter:[* TO 15]"); +var rslt = client.Search(search); +``` + +```javascript +var searchCmd = new Riak.Commands.YZ.Search.Builder() + .withIndexName('scores') + .withQuery('counter:[* TO 15]') + .withCallback(search_cb) + .build(); + +client.execute(searchCmd); +``` + +```erlang +{ok, Results} = riakc_pb_socket:search(Pid, <<"scores">>, <<"counter:[* TO 15]"). +``` + +```curl +curl "$RIAK_HOST/search/query/scores?wt=json&q=counter:[* TO 15]" | jsonpp +``` + +Or we can see how many counters have a value of 17 exactly: + +```java +// Using the same method as above, just changing the query: +String query = "counter:17"; +``` + +```ruby +results = client.search('scores', 'counter:17') +``` + +```php +$response = (new \Basho\Riak\Command\Builder\Search\FetchObjects($riak)) + ->withIndexName('scores') + ->withQuery('counter:17') + ->build() + ->execute(); +``` + +```python +results = client.fulltext_search('scores', 'counter:17') +``` + +```csharp +var search = new RiakSearchRequest("scores", "counter:17"); +var rslt = client.Search(search); +``` + +```javascript +var searchCmd = new Riak.Commands.YZ.Search.Builder() + .withIndexName('scores') + .withQuery('counter:17') + .withCallback(search_cb) + .build(); + +client.execute(searchCmd); +``` + +```erlang +{ok, Results} = riakc_pb_socket:search(Pid, <<"scores">>, <<"counter:17">>). +``` + +```curl +curl "$RIAK_HOST/search/query/scores?wt=json&q=counter:17" | jsonpp +``` + +## Sets Example + +Let's say that we're storing information about the hobbies of a group of +people in sets. We'll create and activate a [bucket type]({{<baseurl>}}riak/kv/2.2.6/developing/usage/bucket-types) for [storing sets]({{<baseurl>}}riak/kv/2.2.6/developing/data-types/#sets) simply called `sets`, +like so: + +```bash +riak-admin bucket-type create sets '{"props":{"datatype":"set"}}' +riak-admin bucket-type activate sets +``` + +Now, we'll create a Search index called `hobbies` that uses the default +schema (as in some of the examples above): + +```java +YokozunaIndex hobbiesIndex = new YokozunaIndex("hobbies"); +StoreIndex storeIndex = + new StoreIndex.Builder(hobbiesIndex).build(); +client.execute(storeIndex); +``` + +```ruby +client.create_search_index('hobbies', '_yz_default') +``` + +```php +$response = (new \Basho\Riak\Command\Builder\Search\StoreIndex($riak)) + ->withName('hobbies') + ->usingSchema('_yz_default') + ->build() + ->execute(); +``` + +```python +client.create_search_index('hobbies', '_yz_default') +``` + +```csharp +var searchIndex = new SearchIndex("hobbies", "_yz_default"); +var rslt = client.PutSearchIndex(searchIndex); +``` + +```javascript +var options = { + schemaName: '_yz_default', + indexName: 'hobbies' +}; +client.storeIndex(options, function (err, rslt) { +}); +``` + +```erlang +riakc_pb_socket:create_search_index(Pid, <<"hobbies">>, <<"_yz_default">>). +``` + +```curl +curl -XPUT $RIAK_HOST/search/index/hobbies \ + -H 'Content-Type: application/json' \ + -d '{"schema": "_yz_default"}' +``` + +Now, we can modify our `sets` bucket type to associate that bucket type +with our `hobbies` index: + +```bash +riak-admin bucket-type update sets '{"props":{"search_index":"hobbies"}}' +``` + +Now, all of the sets that we store in any bucket with the bucket type +`sets` will be automatically indexed as a set. So let's say that we +store three sets for two different people describing their respective +hobbies, in the bucket `people`: + +```java +Namespace peopleBucket = new Namespace("sets", "people"); + +Location mikeDitkaSet = new Location(peopleBucket, "ditka"); +SetUpdate su1 = new SetUpdate() + .add("football") + .add("winning"); +UpdateSet update1 = new UpdateSet.Builder(mikeDitkaSet, su1).build(); + +Location ronnieJamesDioSet = new Location(peopleBucket, "dio"); +SetUpdate su2 = new SetUpdate() + .add("wailing") + .add("rocking") + .add("winning"); +UpdateSet update2 = new UpdateSet.Builder(ronnieJamesDioSet, su2).build(); + +client.execute(update1); +client.execute(update2); +``` + +```ruby +bucket = client.bucket('people') + +mike_ditka_set = Riak::Crdt::Set.new(bucket, 'ditka', 'sets') +mike_ditka_set.add('football') +mike_ditka_set.add('winning') + +ronnie_james_dio_set = Riak::Crdt::Set.new(bucket, 'dio', 'sets') +ronnie_james_dio_set.add('wailing') +ronnie_james_dio_set.add('rocking') +ronnie_james_dio_set.add('winning') +``` + +```php +$builder = (new \Basho\Riak\Command\Builder\UpdateSet($riak)) + ->add('football') + ->add('winning') + ->buildLocation('ditka', 'people', 'counters'); + +$builder->build->execute(); + +$builder->add('wailing') + ->add('rocking') + ->add('winning') + ->buildLocation('dio', 'people', 'counters'); + ->build() + ->execute(); +``` + +```python +from riak.datatypes import Set + +bucket = client.bucket_type('sets').bucket('people') + +mike_ditka_set = Set(bucket, 'ditka') +mike_ditka_set.add('football') +mike_ditka_set.add('winning') +mike_ditka_set.store() + +ronnie_james_dio_set = Set(bucket, 'dio') +ronnie_james_dio_set.add('wailing') +ronnie_james_dio_set.add('rocking') +ronnie_james_dio_set.add('winning') +ronnie_james_dio_set.store() +``` + +```csharp +// https://github.com/basho/riak-dotnet-client/blob/develop/src/RiakClientExamples/Dev/Search/SearchDataTypes.cs + +var cmd = new UpdateSet.Builder() + .WithBucketType("sets") + .WithBucket("people") + .WithKey("ditka") + .WithAdditions(new[] { "football", "winning" }) + .Build(); +RiakResult rslt = client.Execute(cmd); + +cmd = new UpdateSet.Builder() + .WithBucketType("sets") + .WithBucket("people") + .WithKey("dio") + .WithAdditions(new[] { "wailing", "rocking", "winning" }) + .Build(); +rslt = client.Execute(cmd); +``` + +```javascript +var funcs = [ + function (async_cb) { + var options = { + bucketType: 'sets', + bucket: 'people', + key: 'ditka', + additions: ['football', 'winning'] + }; + + client.updateSet(options, function (err, rslt) { + throwIfErr(err); + async_cb(); + }); + }, + function (async_cb) { + var options = { + bucketType: 'sets', + bucket: 'people', + key: 'dio', + additions: ['wailing', 'rocking', 'winning'] + }; + + client.updateSet(options, function (err, rslt) { + throwIfErr(err); + async_cb(); + }); + } +]; + +async.parallel(funcs, function (err, rslts) { + throwIfErr(err); +}); +``` + +```erlang +MikeDitkaSet = riakc_set:new(), +riakc_set:add_element(<<"football">>, MikeDitkaSet), +riakc_set:add_element(<<"winning">>, MikeDitkaSet), +RonnieJamesDioSet = riakc_set:new(), +riakc_set:add_element(<<"wailing">>, RonnieJamesDioSet), +riakc_set:add_element(<<"rocking">>, RonnieJamesDioSet), +riakc_set:add_element(<<"winning">>, RonnieJamesDioSet), + +riakc_pb_socket:update_type(Pid, + {<<"sets">>, <<"people">>}, + <<"ditka">>, + riakc_set:to_op(MikeDitkaSet)), +riakc_pb_socket:update_type(Pid, + {<<"sets">>, <<"people">>}, + <<"dio">>, + riakc_set:to_op(RonnieJamesDioSet)). +``` + +Now, we can query our `hobbies` index to see if anyone has the hobby +`football`: + +```java +// Using the same method explained above, just changing the query: +String query = "set:football"; +``` + +```ruby +results = client.search('hobbies', 'set:football') +# This should return a dict with fields like 'num_found' and 'docs' +``` + +```php +$response = (new \Basho\Riak\Command\Builder\Search\FetchObjects($riak)) + ->withIndexName('hobbies') + ->withQuery('set:football') + ->build() + ->execute(); +``` + +```python +results = client.fulltext_search('hobbies', 'set:football') +# This should return a dict with fields like 'num_found' and 'docs' +``` + +```csharp +var search = new RiakSearchRequest("hobbies", "set:football"); +var rslt = client.Search(search); + +RiakSearchResult searchResult = rslt.Value; +Console.WriteLine("Num found: {0}", searchResult.NumFound); + +var firstDoc = searchResult.Documents.First(); +Console.WriteLine("Key: {0} Bucket: {1} Type: {2}", + firstDoc.Key, firstDoc.Bucket, firstDoc.BucketType); +``` + +```javascript +function search_cb(err, rslt) { + logger.info("sets numFound: '%d', docs: '%s'", + rslt.numFound, JSON.stringify(rslt.docs)); + + var doc = rslt.docs[0]; + var key = doc['_yz_rk']; + var bucket = doc['_yz_rb']; + var bucketType = doc['_yz_rt']; +} + +var searchCmd = new Riak.Commands.YZ.Search.Builder() + .withIndexName('hobbies') + .withQuery('set:football') + .withCallback(search_cb) + .build(); + +client.execute(searchCmd); +``` + +```erlang +{ok, Results} = riakc_pb_socket:search(Pid, <<"hobbies">>, <<"set:football">>). +``` + +```curl +curl "$RIAK_HOST/search/query/hobbies?wt=json&q=set:football" | jsonpp +``` + +Let's see how many sets contain the element `football`: + +```java +// Using the same method explained above for getting search results: +int numberFound = results.numResults(); // 1 +``` + +```ruby +results['num_found'] +# 1 +``` + +```php +$response->getNumFound(); // 1 +``` + +```python +results['num_found'] +# 1 +``` + +```csharp +RiakSearchResult searchResult = rslt.Value; +Console.WriteLine("Num found: {0}", searchResult.NumFound); +``` + +```javascript +rslt.numFound; +// 1 +``` + +```erlang +NumberFound = Results#search_results.num_found. +%% 1 +``` + +```curl +``` + +Success! We stored two sets, only one of which contains the element +`football`. Now, let's see how many sets contain the element `winning`: + +```java +// Using the same method explained above, just changing the query: +String query = "set:winning"; + +// Again using the same method from above: +int numberFound = results.numResults(); // 2 +``` + +```ruby +results = client.search('hobbies', 'set:winning') +results['num_found'] +# 2 +``` + +```php +$response = (new \Basho\Riak\Command\Builder\Search\FetchObjects($riak)) + ->withIndexName('hobbies') + ->withQuery('set:winning') + ->build() + ->execute(); + +$response->getNumFound(); // 2 +``` + +```python +results = client.fulltext_search('hobbies', 'set:winning') +results['num_found'] +# 2 +``` + +```csharp +var search = new RiakSearchRequest("hobbies", "set:winning"); +``` + +```javascript +var searchCmd = new Riak.Commands.YZ.Search.Builder() + .withIndexName('hobbies') + .withQuery('set:winning') + .withCallback(search_cb) + .build(); + +client.execute(searchCmd); +``` + +```erlang +{ok, Results} = riakc_pb_socket:search(Pid, <<"hobbies">>, <<"set:winning">>). +NumberFound = Results#search_results.num_found. +%% 2 +``` + +Just as expected, both sets we stored contain the element `winning`. + +## Maps Example + +This example will build on the example in the [Using Data Types]({{<baseurl>}}riak/kv/2.2.6/developing/data-types) +tutorial. That tutorial walks you through storing CMS-style user data in +Riak [maps]({{<baseurl>}}riak/kv/2.2.6/developing/data-types/#maps), and we'd suggest that you +familiarize yourself with that tutorial first. More specifically, user +data is stored in the following fields in each user's map: + +* first name in a `first_name` register +* last name in a `last_name` register +* whether the user is an enterprise customer in an `enterprise_customer` + flag +* the number of times the user has visited the company page in a + `page_visits` counter +* a list of the user's interests in an `interests` set + +First, let's create and activate a bucket type simply called `maps` that +is set up to store Riak maps: + +```bash +riak-admin bucket-type create maps '{"props":{"datatype":"map"}}' +riak-admin bucket-type activate maps +``` + +Now, let's create a search index called `customers` using the default +schema: + +```java +YokozunaIndex customersIndex = new YokozunaIndex("customers", "_yz_default"); +StoreIndex storeIndex = + new StoreIndex.Builder(customersIndex).build(); +client.execute(storeIndex); +``` + +```ruby +client.create_search_index('customers', '_yz_default') +``` + +```php +(new Command\Builder\Search\StoreIndex($riak)) + ->withName('customers') + ->usingSchema('_yz_default') + ->build() + ->execute(); +``` + +```python +client.create_search_index('customers', '_yz_default') +``` + +```csharp +var searchIndex = new SearchIndex("customers", "_yz_default"); +var rslt = client.PutSearchIndex(searchIndex); +``` + +```javascript +var options = { + schemaName: '_yz_default', + indexName: 'customers' +}; +client.storeIndex(options, function (err, rslt) { +}); +``` + +```erlang +riakc_pb_socket:create_search_index(Pid, <<"customers">>, <<"_yz_default">>). +``` + +```curl +curl -XPUT $RIAK_HOST/search/index/customers \ + -H 'Content-Type: application/json' \ + -d '{"schema":"_yz_default"}' +``` + +With our index created, we can associate our new `customers` index with +our `maps` bucket type: + +```bash +riak-admin bucket-type update maps '{"props":{"search_index":"customers"}}' +``` + +Now we can create some maps along the lines suggested above: + +```java +Namespace customersBucket = new Namespace("maps", "customers"); + +Location idrisElbaMap = new Location(customersBucket, "idris_elba"); +MapUpdate mu = new MapUpdate() + .update("first_name", new RegisterUpdate("Idris")) + .update("last_name", new RegisterUpdate("Elba")) + .update("enterprise_customer", new FlagUpdate(false)) + .update("page_visits", new CounterUpdate(10)) + .update("interests", new SetUpdate().add("acting", "being Stringer Bell")); + +Location joanJettMap = new Location(customersBucket, "joan_jett"); +MapUpdate mu2 = new MapUpdate() + .update("first_name", new RegisterUpdate("Joan")) + .update("last_name", new RegisterUpdate("Jett")) + // Joan Jett is not an enterprise customer, so we don't need to + // explicitly disable the "enterprise_customer" flag, as all + // flags are disabled by default + .update("page_visits", new CounterUpdate(25)) + .update("interests", new SetUpdate().add("loving rock and roll").add("being in the Blackhearts")); + +UpdateMap update1 = new UpdateMap.Builder(idrisElbaMap, mu1).build(); +UpdateMap update2 = new UpdateMap.Builder(joanJettMap, mu2).build(); +client.execute(update1); +client.execute(update2); +``` + +```ruby +bucket = client.bucket('customers') + +idris_elba = Riak::Crdt::Map.new(bucket, 'idris_elba', 'maps') + +idris_elba.batch do |ie| + ie.registers['first_name'] = 'Idris' + ie.registers['last_name'] = 'Elba' + ie.flags['enterprise_customer'] = true + ie.counters['page_visits'].increment(10) + ['acting', 'being Stringer Bell'].each do |interest| + ie.sets['interests'].add(interest) + end +end + +joan_jett = Riak::Crdt::Map.new(bucket, 'joan_jett', 'maps') +joan_jett.batch do |jj| + jj.registers['first_name'] = 'Joan' + jj.registers['last_name'] = 'Jett' + ## Joan Jett is not an enterprise customers, so we don't need to + ## explicitly disable this flag, as all flags are disabled by default + jj.counters['page_visits'].increment(25) + ['loving rock and roll', 'being in the Blackhearts'].each do |interest| + jj.sets['interests'].add(interest) + end +end +``` + +```php +$counterBuilder = (new \Basho\Riak\Command\Builder\IncrementCounter($riak)) + ->withIncrement(10); + +$setBuilder = (new \Basho\Riak\Command\Builder\UpdateSet($riak)); + +foreach(['acting', 'being Stringer Bell'] as $interest) { + $setBuilder->add($interest); +} + +(new \Basho\Riak\Command\Builder\UpdateMap($riak)) + ->updateRegister('first_name', 'Idres') + ->updateRegister('last_name', 'Elba') + ->updateFlag('enterprise_customer', true) + ->updateSet('interests', $setBuilder) + ->updateCounter('page_visits', $counterBuilder) + ->buildLocation('idris_elba', 'customers', 'maps') + ->build() + ->execute(); + +$setBuilder = (new \Basho\Riak\Command\Builder\UpdateSet($riak)); + +foreach(['loving rock and roll', 'being in the Blackhearts'] as $interest) { + $setBuilder->add($interest); +} + +(new \Basho\Riak\Command\Builder\UpdateMap($riak)) + ->updateRegister('first_name', 'Joan') + ->updateRegister('last_name', 'Jett') + ->updateSet('interests', $setBuilder) + ->updateCounter('page_visits', $counterBuilder->withIncrement(25)) + ->buildLocation('joan_jett', 'customers', 'maps') + ->build() + ->execute(); +``` + +```python +bucket = client.bucket_type('maps').bucket('customers') + +idris_elba = Map(bucket, 'idris_elba') +idris_elba.registers['first_name'].assign('Idris') +idris_elba.registers['last_name'].assign('Elba') +idris_elba.flags['enterprise_customer'].enable() +idris_elba.counters['page_visits'].increment(10) +for interest in ['acting', 'being Stringer Bell']: + idris_elba.sets['interests'].add(interest) +idris_elba.store() + +joan_jett = Map(bucket, 'joan_jett') +joan_jett.registers['first_name'].assign('Joan') +joan_jett.registers['last_name'].assign('Jett') +# Joan Jett is not an enterprise customers, so we don't need to +# explictly disable this flag, as all flags are disabled by default +idris_elba.counters['page_visits'].increment(25) +for interest in ['loving rock and roll', 'being in the Blackhearts']: + joan_jett.sets['interests'].add(interest) +joan_jett.store() +``` + +```csharp +// https://github.com/basho/riak-dotnet-client/blob/develop/src/RiakClientExamples/Dev/Search/SearchDataTypes.cs + +// Note: similar code for Joan Jett + +const string firstNameRegister = "first_name"; +const string lastNameRegister = "last_name"; +const string enterpriseCustomerFlag = "enterprise_customer"; +const string pageVisitsCounter = "page_visits"; +const string interestsSet = "interests"; + +var idrisAdds = new[] { "acting", "being Stringer Bell" }; + +var mapOp = new UpdateMap.MapOperation() + .SetRegister(firstNameRegister, "Idris") + .SetRegister(lastNameRegister, "Elba") + .SetFlag(enterpriseCustomerFlag, false) + .IncrementCounter(pageVisitsCounter, 10) + .AddToSet(interestsSet, idrisAdds); + +var cmd = new UpdateMap.Builder() + .WithBucketType("maps") + .WithBucket("customers") + .WithKey("idris_elba") + .WithMapOperation(mapOp) + .Build(); + +RiakResult rslt = client.Execute(cmd); +``` + +```javascript +var funcs = [ + function (async_cb) { + var options = { + bucketType: 'maps', + bucket: 'customers', + key: 'idris_elba' + }; + + var mapOp = new Riak.Commands.CRDT.UpdateMap.MapOperation(); + mapOp.setRegister('first_name', 'Idris'); + mapOp.setRegister('last_name', 'Elba'); + mapOp.setFlag('enterprise_customer', false); + mapOp.incrementCounter('page_visits', 10); + mapOp.addToSet('interests', 'acting'); + mapOp.addToSet('interests', 'being Stringer Bell'); + + options.op = mapOp; + + client.updateMap(options, function (err, rslt) { + throwIfErr(err); + async_cb(); + }); + }, + function (async_cb) { + var options = { + bucketType: 'maps', + bucket: 'customers', + key: 'joan_jett' + }; + + var mapOp = new Riak.Commands.CRDT.UpdateMap.MapOperation(); + mapOp.setRegister('first_name', 'Joan'); + mapOp.setRegister('last_name', 'Jett'); + mapOp.setFlag('enterprise_customer', false); + mapOp.incrementCounter('page_visits', 25); + mapOp.addToSet('interests', 'loving rock and roll'); + mapOp.addToSet('interests', 'being in the Blackhearts'); + + options.op = mapOp; + + client.updateMap(options, function (err, rslt) { + throwIfErr(err); + async_cb(); + }); + } +]; + +async.parallel(funcs, function (err, rslts) { + throwIfErr(err); +}); +``` + +### Searching Counters Within Maps + +We now have two maps stored in Riak that we can query. Let's query to +see how many users have page visit counters above 15. Unlike the +counters example above, we have to specify _which_ counter we're +querying: + +```java +// Using the same method explained above, just changing the query: +String query = "page_visits_counter:[15 TO *]"; + +// Again using the same method from above: +int numberFound = results.numResults(); // 1 +``` + +```ruby +results = client.search('customers', 'page_visits_counter:[15 TO *]') +results['num_found'] +# 1 +``` + +```php +$response = (new \Basho\Riak\Command\Builder\Search\FetchObjects($riak)) + ->withIndexName('customers') + ->withQuery('page_visits_counter:[15 TO *]') + ->build() + ->execute(); + +$response->getNumFound(); // 1 +``` + +```python +results = client.fulltext_search('customers', 'page_visits_counter:[15 TO *]') +results['num_found'] +# 1 +``` + +```csharp +var search = new RiakSearchRequest("customers", "page_visits_counter:[15 TO *]"); +var rslt = client.Search(search); +``` + +```javascript +function search_cb(err, rslt) { + logger.info("numFound: '%d', docs: '%s'", + rslt.numFound, JSON.stringify(rslt.docs)); + + var doc = rslt.docs[0]; + var key = doc['_yz_rk']; + var bucket = doc['_yz_rb']; + var bucketType = doc['_yz_rt']; +} + +var searchCmd = new Riak.Commands.YZ.Search.Builder() + .withIndexName('customers') + .withQuery('page_visits_counter:[15 TO *]') + .withCallback(search_cb) + .build(); + +client.execute(searchCmd); +``` + +As expected, one of our two stored maps has a `page_visits` counter +above 15. Let's make sure that we have the right result: + +```java +// Using the same method from above: +String query = "page_visits_counter:[15 TO *]"; + +// Again using the same method from above: +String registerValue = + results.getAllResults().get(0).get("first_name_register").get(0); // Joan +``` + +```ruby +results['docs'][0]['first_name_register'] +# 'Joan' +``` + +```php +$response->getDocs()[0]->first_name_register']; // Joan +``` + +```python +results['docs'][0]['first_name_register'] +# u'Joan' +``` + +```csharp +var search = new RiakSearchRequest("customers", "page_visits_counter:[15 TO *]"); +var rslt = client.Search(search); +var firstDoc = searchResult.Documents.First(); +``` + +```javascript +var doc = rslts.docs[0]; +doc.page_visits_register; +``` + +Success! Now we can test out searching sets. + +### Searching Sets Within Maps + +Each of the maps we stored thus far had an `interests` set. First, let's +see how many of our maps even _have_ sets called `interests` using a +wildcard query: + +```java +// Using the same method from above: +String query = "interests_set:*"; +``` + +```ruby +results = client.search('customers', 'interests_set:*') +# 2 +``` + +```php +$response = (new \Basho\Riak\Command\Builder\Search\FetchObjects($riak)) + ->withIndexName('customers') + ->withQuery('interests_set:*') + ->build() + ->execute(); + +$response->getNumFound(); // 2 +``` + +```python +results = client.fulltext_search('customers', 'interests_set:*') +results['num_found'] +# 2 +``` + +```csharp +var search = new RiakSearchRequest("customers", "interests_set:*"); +var rslt = client.Search(search); +``` + +```javascript +var searchCmd = new Riak.Commands.YZ.Search.Builder() + .withIndexName('customers') + .withQuery('interests_set:*') + .withCallback(search_cb) + .build(); + +client.execute(searchCmd); +``` + +As expected, both stored maps have an `interests` set. Now let's see how +many maps have items in `interests` sets that begin with `loving`: + +```java +// Using the same method from above: +String query = "interests_set:loving*"; + +// Again using the same method from above: +int numberFound = results.numResults(); // 1 +String registerValue = + results.getAllResults().get(0).get("first_name_register").get(0); // Joan +``` + +```ruby +results = client.search('customers', 'interests_set:loving*') +results['num_found'] # 1 +results['docs'][0]['first_name_register'] # 'Joan' +``` + +```php +$response = (new \Basho\Riak\Command\Builder\Search\FetchObjects($riak)) + ->withIndexName('customers') + ->withQuery('interests_set:loving*') + ->build() + ->execute(); + +$response->getDocs()[0]->first_name_register']; // Joan +``` + +```python +results = client.fulltext_search('customers', 'interests_set:loving*') +results['num_found'] # 1 +results['docs'][0]['first_name_register'] # u'Joan' +``` + +```csharp +var search = new RiakSearchRequest("customers", "interests_set:loving*"); +var rslt = client.Search(search); +``` + +```javascript +var searchCmd = new Riak.Commands.YZ.Search.Builder() + .withIndexName('customers') + .withQuery('interests_set:loving*') + .withCallback(search_cb) + .build(); + +client.execute(searchCmd); +``` + +As expected, only our Joan Jett map has one item in its `interests` set +that starts with `loving`. + +### Searching Maps Within Maps + +Before we can try to search maps within maps, we need to actually store +some. Let's add a `alter_ego` map to both of the maps we've stored thus +far. Each person's alter ego will have a first name only. + +```java +Location idrisElbaMap = new Location(customersBucket, "idris_elba"); +MapUpdate alterEgoUpdateName = new MapUpdate() + .update("name", new RegisterUpdate("John Luther")); +MapUpdate alterEgoUpdate = new MapUpdate() + .update("alter_ego", alterEgoUpdateName); +UpdateMap addSubMap = new UpdateMap.Builder(idrisElbaMap, alterEgoUpdate); +client.execute(addSubMap); +``` + +```ruby +idris_elba.maps['alter_ego'].registers['name'] = 'John Luther' + +joan_jett.maps['alter_ego'].registers['name'] = 'Robert Plant' +``` + +```php +$mapBuilder = (new \Basho\Riak\Command\Builder\UpdateMap($riak)) + ->updateRegister('name', 'John Luther') + +(new \Basho\Riak\Command\Builder\UpdateMap($riak)) + ->updateMap('alter_ego', $mapBuilder) + ->buildLocation('idris_elba', 'customers', 'maps') + ->build() + ->execute(); + +$mapBuilder->updateRegister('name', 'Robert Plant') + +(new \Basho\Riak\Command\Builder\UpdateMap($riak)) + ->updateMap('alter_ego', $mapBuilder) + ->buildLocation('joan_jett', 'customers', 'maps') + ->build() + ->execute(); +``` + +```python +idris_elba.maps['alter_ego'].registers['name'].assign('John Luther') +idris_elba.store() + +joan_jett.maps['alter_ego'].registers['name'].assign('Robert Plant') +joan_jett.store() +``` + +```csharp +// https://github.com/basho/riak-dotnet-client/blob/develop/src/RiakClientExamples/Dev/Search/SearchDataTypes.cs + +const string nameRegister = "name"; +const string alterEgoMap = "alter_ego"; + +var mapOp = new UpdateMap.MapOperation(); +mapOp.Map(alterEgoMap).SetRegister(nameRegister, "John Luther"); + +var cmd = new UpdateMap.Builder() + .WithBucketType("maps") + .WithBucket("customers") + .WithKey("idris_elba") + .WithMapOperation(mapOp) + .Build(); + +RiakResult rslt = client.Execute(cmd); +``` + +```javascript +var funcs = [ + function (async_cb) { + var options = { + bucketType: 'maps', + bucket: 'customers', + key: 'idris_elba' + }; + + var mapOp = new Riak.Commands.CRDT.UpdateMap.MapOperation(); + var alterEgoMap = mapOp.map('alter_ego'); + alterEgoMap.setRegister('name', 'John Luther'); + + options.op = mapOp; + + client.updateMap(options, function (err, rslt) { + throwIfErr(err); + async_cb(); + }); + }, + function (async_cb) { + var options = { + bucketType: 'maps', + bucket: 'customers', + key: 'joan_jett' + }; + + var mapOp = new Riak.Commands.CRDT.UpdateMap.MapOperation(); + var alterEgoMap = mapOp.map('alter_ego'); + alterEgoMap.setRegister('name', 'Robert Plant'); + + options.op = mapOp; + + client.updateMap(options, function (err, rslt) { + throwIfErr(err); + async_cb(); + }); + } +]; + +async.parallel(funcs, function (err, rslts) { + throwIfErr(err); +}); +``` + +Querying maps within maps involves construct queries that separate the +different levels of depth with a single dot. Here's an example query for +finding maps that have a `name` register embedded within an `alter_ego` +map: + +```java +// Using the same method from above: +String query = "alter_ego_map.name_register:*"; + +// Again using the same method from above: +int numberFound = results.numResults(); // 2 +``` + +```ruby +results = client.search('customers', 'alter_ego_map.name_register:*') +results['num_found'] # 2 +``` + +```php +$response = (new \Basho\Riak\Command\Builder\Search\FetchObjects($riak)) + ->withIndexName('customers') + ->withQuery('alter_ego_map.name_register:*') + ->build() + ->execute(); + +$response->getNumFound(); // 2 +``` + +```python +results = client.fulltext_search('customers', 'alter_ego_map.name_register:*') +results['num_found'] # 2 +``` + +```csharp +var search = new RiakSearchRequest("customers", "alter_ego_map.name_register:*"); +var rslt = client.Search(search); +``` + +```javascript +var searchCmd = new Riak.Commands.YZ.Search.Builder() + .withIndexName('customers') + .withQuery('alter_ego_map.name_register:*') + .withCallback(search_cb) + .build(); + +client.execute(searchCmd); +``` + +Once we know how to query embedded fields like this, we can query those +just like any other. Let's find out which maps have an `alter_ego` +sub-map that contains a `name` register that ends with `PLant`, and +display that customer's first name: + +```java +// Using the same method from above: +String query = "alter_ego_map.name_register:*Plant"; + +// Again using the same method from above: +int numberFound = results.numResults(); // 1 +String registerValue = + results.getAllResults().get(0).get("first_name_register").get(0); // Joan +``` + +```ruby +results = client.search('customers', 'alter_ego_map.name_register:*Plant') +results['num_found'] # 1 +results['docs'][0]['first_name_register'] # 'Joan' +``` + +```php +$response = (new \Basho\Riak\Command\Builder\Search\FetchObjects($riak)) + ->withIndexName('customers') + ->withQuery('alter_ego_map.name_register:*Plant') + ->build() + ->execute(); + +$response->getNumFound(); // 1 +$response->getDocs()[0]->first_name_register']; // Joan +``` + +```python +results = client.fulltext_search('customers', 'alter_ego_map.name_register:*Plant') +results['num_found'] # 1 +results['docs'][0]['first_name_register'] # u'Joan +``` + +```csharp +var search = new RiakSearchRequest("customers", "alter_ego_map.name_register:*Plant"); +var rslt = client.Search(search); +``` + +```javascript +var searchCmd = new Riak.Commands.YZ.Search.Builder() + .withIndexName('customers') + .withQuery('alter_ego_map.name_register:*Plant') + .withCallback(search_cb) + .build(); + +client.execute(searchCmd); +``` + +Success! We've now queried not just maps but also maps within maps. diff --git a/content/riak/kv/2.2.6/developing/usage/secondary-indexes.md b/content/riak/kv/2.2.6/developing/usage/secondary-indexes.md new file mode 100644 index 0000000000..297b763702 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/usage/secondary-indexes.md @@ -0,0 +1,2026 @@ +--- +title: "Using Secondary Indexes (2i)" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Using Secondary Indexes" + identifier: "usage_2i" + weight: 107 + parent: "developing_usage" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/using/2i + - /riak-docs/riak/kv/2.2.6/dev/using/2i +--- + +[plan backend leveldb]: {{<baseurl>}}riak/kv/2.2.6/setup/planning/backend/leveldb +[plan backend memory]: {{<baseurl>}}riak/kv/2.2.6/setup/planning/backend/memory +[use ref strong consistency]: {{<baseurl>}}riak/kv/2.2.6/using/reference/strong-consistency + +> **Note: Riak Search preferred for querying** +> +> If you're interested in non-primary-key-based querying in Riak, i.e. if +you're looking to go beyond straightforward K/V operations, we now +recommend [Riak Search]({{<baseurl>}}riak/kv/2.2.6/developing/usage/search/) rather than secondary indexes for +a variety of reasons. Most importantly, Riak Search has a far more +capacious querying API and can be used with all of Riak's storage +backends. + +Secondary indexes (2i) in Riak enable you to tag objects stored in Riak, +at write time, with one or more queryable values. Those values can then +be used to find multiple objects in Riak. If you're storing [user data]({{<baseurl>}}riak/kv/2.2.6/developing/data-modeling/#user-accounts), for example, you could tag each object +associated with that user with a username or other unique marker. Once +tagged, you could find all objects in a Riak bucket sharing that tag. +Secondary indexes can be either a binary or string, such as +`sensor_1_data` or `admin_user` or `click_event`, or an integer, such as +`99` or `141121`. + +[Riak Search]({{<baseurl>}}riak/kv/2.2.6/developing/usage/search/) serves analogous purposes but is quite +different because it parses key/value data itself and builds indexes on +the basis of Solr schemas. + +Please note that 2i can be used only with the [LevelDB][plan backend leveldb] and [Memory][plan backend memory] +backends. + +## Features + +* Allows two types of secondary attributes: integers and strings (aka + binaries) +* Allows querying by exact match or range on one index +* Allows pagination of results +* Allows streaming of results +* Query results can be used as input to a [MapReduce]({{<baseurl>}}riak/kv/2.2.6/developing/usage/mapreduce/) + query + +> **Note on 2i and strong consistency** +Secondary indexes do not currently work with the [strong consistency][use ref strong consistency] +feature introduced in Riak version 2.0. If you store objects in +[strongly consistent buckets]({{<baseurl>}}riak/kv/2.2.6/developing/app-guide/strong-consistency/#creating-a-strongly-consistent-bucket-type) and attach +secondary index metadata to those objects, you can still perform +strongly consistent operations on those objects but the secondary +indexes will be ignored. + +## When to Use Secondary Indexes + +Secondary indexes are useful when you want to find data on the basis of +something other than objects' bucket type, bucket, and key, i.e. when +you want objects to be discoverable based on more than their location +alone. + +2i works best for objects whose value is stored in an opaque blob, like +a binary file, because those objects don't offer any clues that enable +you to discover them later. Indexing enables you to tag those objects +and find all objects with the same tag in a specified bucket later on. + +2i is thus recommended when your use case requires an easy-to-use search +mechanism that does not require a schema (as does [Riak Search]({{<baseurl>}}riak/kv/2.2.6/using/reference/search/#schemas)) and a basic query interface, i.e. an interface that +enables an application to tell Riak things like "fetch all objects +tagged with the string `Milwaukee_Bucks`" or "fetch all objects tagged +with numbers between 1500 and 1509." + +2i is also recommended if your use case requires anti-entropy. Since +secondary indexes are just metadata attached to key/value objects, 2i +piggybacks off of read-repair. + +## When Not to Use Secondary Indexes + +* If your ring size exceeds 512 partitions, 2i can cause performance + issues in large clusters. +* When you need more than the exact match and range searches that 2i + supports. If that's the case, we recommend checking out [Riak Search]({{<baseurl>}}riak/kv/2.2.6/developing/usage/search/). +* When you want to use composite queries. A query like + `last_name=zezeski AND state=MD` would have to be split into two + queries and the results merged (or it would need to involve + [MapReduce]({{<baseurl>}}riak/kv/2.2.6/developing/usage/mapreduce/)). + +## Query Interfaces and Examples + +Typically, the result set from a 2i query is a list of object keys from +the specified bucket that include the index values in question. As we'll +see below, when executing range queries in Riak 1.4 or higher, it is +possible to retrieve the index values along with the object keys. + +### Inserting Objects with Secondary Indexes + +In this example, the key `john_smith` is used to store user data in the +bucket `users`, which bears the `default` bucket type. Let's say that an +application would like add a Twitter handle and an email address to this +object as secondary indexes. + +```java +Location johnSmithKey = new Location(new Namespace("default", "users"), "john_smith"); + +// In the Java client (and all clients), if you do not specify a bucket type, +// the client will use the default type. And so the following store command +// would be equivalent to the one above: +Location johnSmithKey = new Location(new Namespace("users"), "john_smith"); + +RiakObject obj = new RiakObject() + .setContentType("application/json") + .setValue(BinaryValue.create("{'user_data':{ ... }}")); + +obj.getIndexes().getIndex(StringBinIndex.named("twitter")).add("jsmith123"); +obj.getIndexes().getIndex(StringBinIndex.named("email")).add("jsmith@basho.com"); + +StoreValue store = new StoreValue.Builder(obj) + .withLocation(johnSmithKey) + .build(); +client.execute(store); +``` + +```ruby +bucket = client.bucket_type('default').bucket('users') +obj = Riak::RObject.new(bucket, 'john_smith') +obj.content_type = 'application/json' +obj.raw_data = '{"user_data":{ ... }}' + +# String/binary indexes must be set as an array of strings +obj.indexes['twitter_bin'] = %w{ jsmith123 } +obj.indexes['email_bin'] = %w{ jsmith@basho.com } +obj.store + +# In the Ruby client (and all clients), if you do not specify a bucket +# type, the client will use the default type. And so the following set +# of commands would be equivalent to the one above: + +bucket = client.bucket('users') +# repeat the same commands for building the object +obj.store +``` + +```php +$object = (new \Basho\Riak\Object('{"user_data":{ ... }}', ['Content-type' => 'application/json'])) + ->addValueToIndex('twitter_bin', 'jsmith123') + ->addValueToIndex('email_bin', 'jsmith@basho.com'); + +(new \Basho\Riak\Command\Builder\StoreObject($riak)) + ->withObject($object) + ->buildLocation('john_smith', 'users', 'default') + ->build() + ->execute(); +``` + +```python +bucket = client.bucket_type('default').bucket('users') +# In the Python client (and all clients), if you do not specify a bucket type, +# the client will use the default type. And so the following store command +# would be equivalent to the one above: +bucket = client.bucket('users') + +obj = RiakObject(client, bucket, 'john_smith') +obj.content_type = 'text/plain' +obj.data = '...user data...' +obj.add_index('twitter_bin', 'jsmith123') +obj.add_index('email_bin', 'jsmith@basho.com') +obj.store() +``` + +```csharp +var id = new RiakObjectId("default", "users", "john_smith"); +var obj = new RiakObject(id, "...user data...", + RiakConstants.ContentTypes.TextPlain); +obj.BinIndex("twitter").Set("jsmith123"); +obj.BinIndex("email").Set"jsmith@basho.com"); +var rslt = client.Put(obj); +``` + +```javascript +var riakObj = new Riak.Commands.KV.RiakObject(); +riakObj.setContentType('text/plain'); +riakObj.setBucket('users'); +riakObj.setKey('john_smith'); +riakObj.setValue('...user data...'); +riakObj.addToIndex('twitter_bin', 'jsmith123'); +riakObj.addToIndex('email_bin', 'jsmith@basho.com'); +client.storeValue({ value: riakObj }, function (err, rslt) { + if (err) { + throw new Error(err); + } +}); +``` + +```erlang +Obj = riakc_obj:new({<<"default">>, <<"users">>}, + <<"john_smith">>, + <<"...user data...">>, + <<"text/plain">>), +%% In the Erlang client (and all clients), if you do not specify a bucket type, +%% the client will use the default type. And so the following object would be +%% equivalent to the one above: + +Obj = riakc_obj:new(<<"users">>, + <<"john_smith">>, + <<"...user data...">>, + <<"text/plain">>), +MD1 = riakc_obj:get_update_metadata(Obj), +MD2 = riakc_obj:set_secondary_index( + MD1, + [{{binary_index, "twitter"}, [<<"jsmith123">>]}, + {{binary_index, "email"}, [<<"jsmith@basho.com">>]}]), +Obj2 = riakc_obj:update_metadata(Obj, MD2), +riakc_pb_socket:put(Pid, Obj2). +``` + +```golang +obj := &riak.Object{ + ContentType: "text/plain", + Charset: "utf-8", + ContentEncoding: "utf-8", + BucketType: "indexes", + Bucket: "users", + Key: "john_smith", + Value: []byte("…user data…"), +} + +obj.AddToIndex("twitter_bin", "jsmith123") +obj.AddToIndex("email_bin", "jsmith@basho.com") + +cmd, err := riak.NewStoreValueCommandBuilder(). + WithContent(obj). + Build() +if err != nil { + return err +} + +if err := cluster.Execute(cmd); err != nil { + return err +} +``` + +```curl +curl -XPOST localhost:8098/types/default/buckets/users/keys/john_smith \ + -H 'x-riak-index-twitter_bin: jsmith123' \ + -H 'x-riak-index-email_bin: jsmith@basho.com' \ + -H 'Content-Type: application/json' \ + -d '{"userData":"data"}' +``` + +> **Getting started with Riak clients** +> +> If you are connecting to Riak using one of Basho's official [client libraries]({{<baseurl>}}riak/kv/2.2.6/developing/client-libraries), you can find more information about getting started with +your client in the [Developing with Riak KV: Getting Started]({{<baseurl>}}riak/kv/2.2.6/developing/getting-started) section. + +This has accomplished the following: + +* The object has been stored with a primary bucket/key of + `users`/`john_smith` +* The object now has a secondary index called `twitter_bin` with a value + of `jsmith123` +* The object now has a secondary index called `email_bin` with a value + of `jsmith@basho.com` + +### Querying Objects with Secondary Indexes + +Let's query the `users` bucket on the basis of Twitter handle to make +sure that we can find our stored object: + +```java +Namespace usersBucket = new Namespace("users"); +BinIndexQuery biq = new BinIndexQuery.Builder(usersBucket, "twitter", "jsmith123") + .build(); +BinIndexQuery.Response response = client.execute(biq); +List<BinIndexQuery.Response.Entry> entries = response.getEntries(); +for (BinIndexQuery.Response.Entry entry : entries) { + System.out.println(entry.getRiakObjectLocation().getKey()); +} +``` + +```ruby +bucket = client.bucket('users') +bucket.get_index('twitter_bin', 'jsmith123') + +# This is equivalent to the following: +bucket = client.bucket_type('default').bucket('users') +bucket.get_index('twitter_bin', 'jsmith123') +``` + +```php +$response = (new \Basho\Riak\Command\Builder\QueryIndex($riak)) + ->buildBucket('users') + ->withIndexName('twitter_bin') + ->withScalarValue('jsmith123') + ->build() + ->execute() + ->getResults(); +``` + +```python +bucket = client.bucket('users') # equivalent to client.bucket_type('default').bucket('users') +bucket.get_index('twitter_bin', 'jsmith123').results +``` + +```csharp +var idxId = new RiakIndexId("default", "users", "twitter"); +var rslt = client.GetSecondaryIndex(idxId, "jsmith123"); +var idxRslt = rslt.Value; +foreach (var keyTerm in idxRslt.IndexKeyTerms) +{ + Debug.WriteLine(keyTerm.Key); +} +``` + +```javascript +var query_keys = []; +function query_cb(err, rslt) { + if (err) { + throw new Error(err); + } + + if (rslt.done) { + query_keys.forEach(function (key) { + logger.info("2i query key: '%s'", key); + }); + } + + if (rslt.values.length > 0) { + Array.prototype.push.apply(query_keys, + rslt.values.map(function (value) { + return value.objectKey; + })); + } +} + +var cmd = new Riak.Commands.KV.SecondaryIndexQuery.Builder() + .withBucket('users') + .withIndexName('twitter_bin') + .withIndexKey('jsmith123') + .withCallback(query_cb) + .build(); +client.execute(cmd); +``` + +```erlang +{ok, Results} = + riakc_pb_socket:get_index(Pid, + <<"users">>, %% bucket + {binary_index, "twitter"}, %% index name + <<"jsmith123">>). %% index +``` + +```golang +cmd, err := riak.NewSecondaryIndexQueryCommandBuilder(). + WithBucketType("indexes"). + WithBucket("users"). + WithIndexName("twitter_bin"). + WithIndexKey("jsmith123"). + Build() +if err != nil { + return err +} + +if err := cluster.Execute(cmd); err != nil { + return err +} +``` + +```curl +curl localhost:8098/buckets/users/index/twitter_bin/jsmith123 +``` + +The response: + +```java +john_smith +``` + +```ruby +["john_smith"] +``` + +```php +['john_smith'] +``` + +```python +['john_smith'] +``` + +```csharp +john_smith +``` + +```javascript +john_smith +``` + +```erlang +{ok,{index_results_v1,[<<"john_smith">>], + undefined,undefined}}. +``` + +```golang +john_smith +``` + +```curl +{ + "keys": [ + "john_smith" + ] +} +``` + +## Examples + +To run the following examples, make sure that Riak is configured to use +an index-capable storage backend, such as [LevelDB][plan backend leveldb] or [Memory][plan backend memory]. + +## Indexing Objects + +The following example indexes four different objects. Notice that we're +storing both integer and string (aka binary) fields. Field names are +automatically lowercased, some fields have multiple values, and +duplicate fields are automatically de-duplicated, as in the following +example: + +```java +Namespace peopleBucket = new Namespace("indexes", "people"); + +RiakObject larry = new RiakObject() + .setValue(BinaryValue.create("My name is Larry")); +larry.getIndexes().getIndex(StringBinIndex.named("field1")).add("val1"); +larry.getIndexes().getIndex(LongIntIndex.named("field2")).add(1001L); +StoreValue storeLarry = new StoreValue.Builder(larry) + .withLocation(peopleBucket.setKey("larry")) + .build(); +client.execute(storeLarry); + +RiakObject moe = new RiakObject() + .setValue(BinaryValue.create("Ny name is Moe")); +moe.getIndexes().getIndex(StringBinIdex.named("Field1")).add("val2"); +moe.getIndexes().getIndex(LongIntIndex.named("Field2")).add(1002L); +StoreValue storeMoe = new StoreValue.Builder(moe) + .withLocation(peopleBucket.setKey("moe")) + .build(); +client.execute(storeMoe); + +RiakObject curly = new RiakObject() + .setValue(BinaryValue.create("My name is Curly")); +curly.getIndexes().getIndex(StringBinIndex.named("FIELD1")).add("val3"); +curly.getIndexes().getIndex(LongIntIndex.named("FIELD2")).add(1003L); +StoreValue storeCurly = new StoreValue.Builder(curly) + .withLocation(peopleBucket.setKey("curly")) + .build(); +client.execute(storeCurly); + +RiakObject veronica = new RiakObject() + .setValue(BinaryValue.create("My name is Veronica")); +veronica.getIndexes().getIndex(StringBinIndex.named("field1")) + .add("val4").add("val4"); +veronica.getIndexes().getIndex(LongIntIndex.named("field2")) + .add(1004L).add(1005L).add(1006L).add(1004L).add(1004L).add(1007L); +StoreValue storeVeronica = new StoreValue.Builder(veronica) + .withLocation(peopleBucket.setKey("veronica")) + .build(); +client.execute(storeVeronica); +``` + +```ruby +bucket = client.bucket_type('indexes').bucket('people') + +obj1 = Riak::RObject.new(bucket, 'larry') +obj1.content_type = 'text/plain' +obj1.raw_data = 'My name is Larry' +obj1.indexes['field1_bin'] = %w{ val1 } +# Like binary/string indexes, integer indexes must be set as an array, +# even if you wish to add only a single index +obj1.indexes['field2_int'] = [1001] +obj1.store + +obj2 = Riak::RObject.new(bucket, 'moe') +obj2.content_type = 'text/plain' +obj2.raw_data = 'My name is Larry' +obj2.indexes['Field1_bin'] = %w{ val2 } +obj2.indexes['Field2_int'] = [1002] +obj2.store + +obj3 = Riak::RObject.new(bucket, 'curly') +obj3.content_type = 'text/plain' +obj3.raw_data = 'My name is Curly' +obj3.indexes['FIELD1_BIN'] = %w{ val3 } +obj3.indexes['FIELD2_INT'] = [1003] +obj3.store + +obj4 = Riak::RObject.new(bucket, 'veronica') +obj4.content_type = 'text/plain' +obj4.raw_data = 'My name is Veronica' +obj4.indexes['field1_bin'] = %w{ val4 val4 val4a val4b } +obj4.indexes['field2_int'] = [1004, 1004, 1005, 1006] +obj4.indexes['field2_int'] = [1004] +obj4.indexes['field2_int'] = [1004] +obj4.indexes['field2_int'] = [1004] +obj4.indexes['field2_int'] = [1007] +obj4.store +``` + +```php +$bucket = new \Basho\Riak\Bucket('people', 'indexes'); + +$object = (new \Basho\Riak\Object'My name is Larry', ['Content-type' => 'text/plain'])) + ->addValueToIndex('field1_bin', 'val1') + ->addValueToIndex('field2_int', 1001); + +(new \Basho\Riak\Command\Builder\StoreObject($riak)) + ->withObject($object) + ->withLocation(new \Basho\Riak\Location('larry', $bucket)) + ->build() + ->execute(); + +$object = (new \Basho\Riak\Object'My name is Moe', ['Content-type' => 'text/plain'])) + ->addValueToIndex('Field1_bin', 'val2') + ->addValueToIndex('Field2_int', 1002); + +(new \Basho\Riak\Command\Builder\StoreObject($riak)) + ->withObject($object) + ->withLocation(new \Basho\Riak\Location('moe', $bucket)) + ->build() + ->execute(); + +$object = (new \Basho\Riak\Object'My name is Curly', ['Content-type' => 'text/plain'])) + ->addValueToIndex('FIELD1_BIN', 'val3') + ->addValueToIndex('FIELD2_int', 1003); + +(new \Basho\Riak\Command\Builder\StoreObject($riak)) + ->withObject($object) + ->withLocation(new \Basho\Riak\Location('curly', $bucket)) + ->build() + ->execute(); + +$object = (new \Basho\Riak\Object'My name is Veronica', ['Content-type' => 'text/plain'])) + ->addValueToIndex('field1_bin', 'val4') + ->addValueToIndex('field1_bin', 'val4') + ->addValueToIndex('field1_bin', 'val4a') + ->addValueToIndex('field1_bin', 'val4b') + ->addValueToIndex('field2_int', 1004) + ->addValueToIndex('field2_int', 1005) + ->addValueToIndex('field2_int', 1006) + ->addValueToIndex('field2_int', 1004) + ->addValueToIndex('field2_int', 1004) + ->addValueToIndex('field2_int', 1007); + +(new \Basho\Riak\Command\Builder\StoreObject($riak)) + ->withObject($object) + ->withLocation(new \Basho\Riak\Location('veronica', $bucket)) + ->build() + ->execute(); +``` + +```python +bucket = client.bucket_type('indexes').bucket('people') + +obj1 = RiakObject(client, bucket, 'larry') +obj1.content_type = 'text/plain' +obj1.data = 'My name is Larry' +obj1.add_index('field1_bin', 'val1').add_index('field2_int', 1001) +obj1.store() + +obj2 = RiakObject(client, bucket, 'moe') +obj2.content_type = 'text/plain' +obj2data = 'Moe' +obj2.add_index('Field1_bin', 'val2').add_index('Field2_int', 1002) +obj2.store() + +obj3 = RiakObject(client, bucket, 'curly') +obj3.content_type = 'text/plain' +obj3.data = 'Curly' +obj3.add_index('FIELD1_BIN', 'val3').add_index('FIELD2_INT', 1003) +obj3.store() + +obj4 = RiakObject(client, bucket, 'veronica') +obj4.content_type = 'text/plain' +obj4.data = 'Veronica' +obj4.add_index('field1_bin', 'val4').add_index('field1_bin', 'val4a').add_index('field1_bin', 'val4b').add_index('field2_int', 1004).add_index('field2_int', 1004).add_index('field2_int', 1005).add_index('field2_int', 1006).add_index('field2_int', 1004).add_index('field2_int', 1004).add_index('field2_int', 1004).add_index('field2_int', 1007) +obj4.store() +``` + +```csharp +var larryId = new RiakObjectId("indexes", "people", "larry"); +var larry = new RiakObject(larryId, "My name is Larry", + RiakConstants.ContentTypes.TextPlain); + +larry.BinIndex("field1").Set("val1"); +larry.IntIndex("field2").Set(1001); + +client.Put(larry); + +var moeId = new RiakObjectId("indexes", "people", "moe"); +var moe = new RiakObject(moeId, "My name is Moe", + RiakConstants.ContentTypes.TextPlain); + +moe.BinIndex("Field1").Set("val2"); +moe.IntIndex("Field2").Set(1002); + +client.Put(moe); + +var curlyId = new RiakObjectId("indexes", "people", "curly"); +var curly = new RiakObject(curlyId, "My name is Curly", + RiakConstants.ContentTypes.TextPlain); + +curly.BinIndex("FIELD1").Set("val3"); +curly.IntIndex("FIELD2").Set(1003); + +client.Put(curly); + +var veronicaId = new RiakObjectId("indexes", "people", "veronica"); +var veronica = new RiakObject(veronicaId, "My name is Veronica", + RiakConstants.ContentTypes.TextPlain); + +veronica.BinIndex("FIELD1").Set(new string[] { "val4", "val4a", "val4b" }); +veronica.IntIndex("FIELD2").Set(new BigInteger[] { + 1004, 1005, 1006, 1004, 1004, 1007 +}); + +client.Put(veronica); +``` + +```javascript +function store_cb(err, rslt, async_cb) { + if (err) { + throw new Error(err); + } + async_cb(null, rslt); +} + +var storeFuncs = [ + function (async_cb) { + var riakObj = new Riak.Commands.KV.RiakObject(); + riakObj.setContentType('text/plain'); + riakObj.setBucketType('indexes'); + riakObj.setBucket('people'); + riakObj.setKey('larry'); + riakObj.setValue('My name is Larry'); + riakObj.addToIndex('field1_bin', 'val1'); + riakObj.addToIndex('field2_int', 1001); + client.storeValue({ value: riakObj }, function (err, rslt) { + store_cb(err, rslt, async_cb); + }); + }, + function (async_cb) { + var riakObj = new Riak.Commands.KV.RiakObject(); + riakObj.setContentType('text/plain'); + riakObj.setBucketType('indexes'); + riakObj.setBucket('people'); + riakObj.setKey('moe'); + riakObj.setValue('My name is Moe'); + riakObj.addToIndex('Field1_bin', 'val2'); + riakObj.addToIndex('Field2_int', 1002); + client.storeValue({ value: riakObj }, function (err, rslt) { + store_cb(err, rslt, async_cb); + }); + }, + function (async_cb) { + var riakObj = new Riak.Commands.KV.RiakObject(); + riakObj.setContentType('text/plain'); + riakObj.setBucketType('indexes'); + riakObj.setBucket('people'); + riakObj.setKey('curly'); + riakObj.setValue('My name is Curly'); + riakObj.addToIndex('FIELD1_BIN', 'val3'); + riakObj.addToIndex('FIELD2_INT', 1003); + client.storeValue({ value: riakObj }, function (err, rslt) { + store_cb(err, rslt, async_cb); + }); + }, + function (async_cb) { + var riakObj = new Riak.Commands.KV.RiakObject(); + riakObj.setContentType('text/plain'); + riakObj.setBucketType('indexes'); + riakObj.setBucket('people'); + riakObj.setKey('veronica'); + riakObj.setValue('My name is Veronica'); + riakObj.addToIndex('FIELD1_bin', 'val4'); + riakObj.addToIndex('FIELD1_bin', 'val4'); + riakObj.addToIndex('FIELD1_bin', 'val4a'); + riakObj.addToIndex('FIELD1_bin', 'val4b'); + riakObj.addToIndex('FIELD2_int', 1004); + riakObj.addToIndex('FIELD2_int', 1005); + riakObj.addToIndex('FIELD2_int', 1006); + riakObj.addToIndex('FIELD2_int', 1004); + riakObj.addToIndex('FIELD2_int', 1004); + riakObj.addToIndex('FIELD2_int', 1007); + client.storeValue({ value: riakObj }, function (err, rslt) { + store_cb(err, rslt, async_cb); + }); + } +]; +async.parallel(storeFuncs, function (err, rslts) { + if (err) { + throw new Error(err); + } +}); +``` + +```erlang +Larry = riakc_obj:new( + {<<"indexes">>, <<"people">>}, + <<"larry">>, + <<"My name is Larry">>, + <<"text/plain">>), +LarryMetadata = riakc_obj:get_update_metadata(Larry), +LarryIndexes = riakc_obj:set_secondary_index( + LarryMetadata, + [{{binary_index, "field1"}, [<<"val1">>]}, {{integer_index, "field2"}, [1001]}] +), +LarryWithIndexes = riakc_obj:update_metadata(Larry, LarryIndexes). + +Moe = riakc_obj:new( + {<<"indexes">>, <<"people">>}, + <<"moe">>, + <<"My name is Moe">>, + <<"text/plain">>), +MoeMetadata = riakc_obj:get_update_metadata(Moe), +MoeIndexes = riakc_obj:set_secondary_index( + MoeMetadata, + [{{binary_index, "Field1"}, [<<"val2">>]}, {{integer_index, "Field2"}, [1002]}] +), +MoeWithIndexes = riakc_obj:update_metadata(Moe, MoeIndexes). + +Curly = riakc_obj:new( + {<<"indexes">>, <<"people">>}, + <<"curly">>, + <<"My name is Curly">>, + <<"text/plain">>), +CurlyMetadata = riakc_obj:get_update_metadata(Curly), +CurlyIndexes = riakc_obj:set_secondary_index( + CurlyMetadata, + [{{binary_index, "FIELD1"}, [<<"val3">>]}, {{integer_index, "FIELD2"}, [1003]}] +), +CurlyWithIndexes = riakc_obj:update_metadata(Curly, CurlyIndexes). + +Veronica = riakc_obj:new( + {<<"indexes">>, <<"people">>}, + <<"veronica">>, + <<"My name is Veronica">>, + <<"text/plain">>), +VeronicaMetadata = riakc_obj:get_update_metadata(Veronica), +VeronicaIndexes = riakc_obj:set_secondary_index( + VeronicaMetadata, + [{{binary_index, "field1"}, [<<"val4">>]}, {{binary_index, "field1"}, [<<"val4">>]}, {{integer_index, "field2"}, [1004]}, {{integer_index, "field2"}, [1004]}, {{integer_index, "field2"}, [1005]}, {{integer_index, "field2"}, [1006]}, {{integer_index, "field2"}, [1004]}, {{integer_index, "field2"}, [1004]}, {{integer_index, "field2"}, [1007]}] +), +VeronicaWithIndexes = riakc_obj:update_metadata(Veronica, VeronicaIndexes). +``` + +```golang +o1 := &riak.Object{ + Key: "larry", + Value: []byte("My name is Larry"), +} +o1.AddToIndex("field1_bin", "val1") +o1.AddToIntIndex("field2_int", 1001) + +o2 := &riak.Object{ + Key: "moe", + Value: []byte("My name is Moe"), +} +o2.AddToIndex("Field1_bin", "val2") +o2.AddToIntIndex("Field2_int", 1002) + +o3 := &riak.Object{ + Key: "curly", + Value: []byte("My name is Curly"), +} +o3.AddToIndex("FIELD1_BIN", "val3") +o3.AddToIntIndex("FIELD2_INT", 1003) + +o4 := &riak.Object{ + Key: "veronica", + Value: []byte("My name is Veronica"), +} +o4.AddToIndex("FIELD1_bin", "val4") +o4.AddToIndex("FIELD1_bin", "val4") +o4.AddToIndex("FIELD1_bin", "val4a") +o4.AddToIndex("FIELD1_bin", "val4b") +o4.AddToIntIndex("FIELD2_int", 1004) +o4.AddToIntIndex("FIELD2_int", 1005) +o4.AddToIntIndex("FIELD2_int", 1006) +o4.AddToIntIndex("FIELD2_int", 1004) +o4.AddToIntIndex("FIELD2_int", 1004) +o4.AddToIntIndex("FIELD2_int", 1007) + +objs := [...]*riak.Object{o1, o2, o3, o4} + +wg := &sync.WaitGroup{} +for _, obj := range objs { + obj.ContentType = "text/plain" + obj.Charset = "utf-8" + obj.ContentEncoding = "utf-8" + + cmd, err := riak.NewStoreValueCommandBuilder(). + WithBucketType("indexes"). + WithBucket("people"). + WithContent(obj). + Build() + if err != nil { + return err + } + + args := &riak.Async{ + Command: cmd, + Wait: wg, + } + if err := cluster.ExecuteAsync(args); err != nil { + return err + } +} + +wg.Wait() +``` + +```curl +curl -v -XPUT localhost:8098/types/indexes/buckets/people/keys/larry \ + -H "x-riak-index-field1_bin: val1" \ + -H "x-riak-index-field2_int: 1001" \ + -d 'My name is Larry' + +curl -v -XPUT localhost:8098/types/indexes/buckets/people/keys/moe \ + -H "x-riak-index-Field1_bin: val2" \ + -H "x-riak-index-Field2_int: 1002" \ + -d 'My name is Moe' + +curl -v -XPUT localhost:8098/types/indexes/buckets/people/keys/curly \ + -H "X-RIAK-INDEX-FIELD1_BIN: val3" \ + -H "X-RIAK-INDEX-FIELD2_INT: 1003" \ + -d 'My name is Curly' + +curl -v -XPUT 127.0.0.1:8098/types/indexes/buckets/people/keys/veronica \ + -H "x-riak-index-field1_bin: val4, val4, val4a, val4b" \ + -H "x-riak-index-field2_int: 1004, 1004, 1005, 1006" \ + -H "x-riak-index-field2_int: 1004" \ + -H "x-riak-index-field2_int: 1004" \ + -H "x-riak-index-field2_int: 1004" \ + -H "x-riak-index-field2_int: 1007" \ + -d 'My name is Veronica' +``` + +The above objects will end up having the following secondary indexes, +respectively: + +* `Larry` --- Binary index `field1_bin` and integer index `field2_int` +* `Moe` --- Binary index `field1_bin` and integer index `field2_int` + (note that the index names are set to lowercase by Riak) +* `Curly` --- Binary index `field1_bin` and integer index `field2_int` + (note again that the index names are set to lowercase) +* `Veronica` --- Binary index `field1_bin` with the values `val4`, + `val4a`, and `val4b` and integer index `field2_int` with the values + `1004`, `1005`, `1006`, and `1007` (note that redundancies have been removed) + +As these examples show, there are safeguards in Riak that both normalize +the names of indexes and prevent the accumulation of redundant indexes. + +## Invalid Field Names and Types + +The following examples demonstrate what happens when an index field is +specified with an invalid field name or type. The system responds with +`400 Bad Request` and a description of the error. + +Invalid field name: + +```java +// The Java client will not allow you to provide invalid index names, +// because you are not required to add "_bin" or "_int" to the end of +// those names +``` + +```ruby +bucket = client.bucket_type('indexes').bucket('people') +obj = Riak::RObject.new(bucket, 'larry') +obj.indexes['field2_foo'] = [1001] + +# The Ruby client will let you get away with this...at first. But when +# you attempt to store the object, you will get an error response such +# as this: + +NoMethodError: undefined method 'map' for 1001:Fixnum +``` + +```php +// throws \InvalidArgumentException +$object = (new \Basho\Riak\Object('{"user_data":{ ... }}', ['Content-type' => 'application/json'])) + ->addValueToIndex('twitter', 'jsmith123'); +``` + +```python +bucket = client.bucket_type('indexes').bucket('people') +obj = RiakObject(client, bucket, 'larry') +obj.add_index('field2_foo', 1001) + +# Result: +riak.RiakError: "Riak 2i fields must end with either '_bin' or '_int'." +``` + +```csharp +// The Riak .NET Client will not allow you to provide invalid index names, +// because you are not required to add "_bin" or "_int" to the end of +// those names +``` + +```javascript +var cmd = new Riak.Commands.KV.SecondaryIndexQuery.Builder() + .withBucketType('indexes') + .withBucket('people') + .withIndexName('field2_foo') + .withIndexKey('jsmith123') + .withCallback(query_cb) + .build(); +client.execute(cmd); + +// Produces the following stack trace (truncated): +error: query_cb err: 'Error processing incoming message: error:function_clause:[{riak_api_pb_server, + send_error, + [{unknown_field_type, + <<"field2_foo">>}, + {state, + {gen_tcp,inet}, + #Port<0.68338>, + undefined, + ... + ... + ... +``` + +```erlang +Obj = riakc_obj:new( + {<<"indexes">>, <<"people">>}, + <<"larry">>, + <<"some data">>, + <<"text/plain">> +), +MD1 = riakc_obj:get_update_metadata(Obj), +MD2 = riakc_obj:set_secondary_index(MD1, [{{foo_index, "field2"}, [1001]}]). + +%% The Erlang client will return an error message along these lines: +** exception error: no function clause matching + riakc_obj:set_secondary_index( ... ). +``` + +```golang +cmd, err := riak.NewSecondaryIndexQueryCommandBuilder(). + WithBucketType("indexes"). + WithBucket("users"). + WithIndexName("field2_foo"). + WithIndexKey("jsmith123"). + Build() +if err != nil { + return err +} + +if err := cluster.Execute(cmd); err != nil { + fmt.Println("[DevUsing2i] field name error:", err) +} else { + return errors.New("[DevUsing2i] expected an error!") +} + +// Produces the following stack trace (truncated): +error: query_cb err: 'Error processing incoming message: error:function_clause:[{riak_api_pb_server, + send_error, + [{unknown_field_type, + <<"field2_foo">>}, + {state, + {gen_tcp,inet}, + #Port<0.68338>, + undefined, + ... + ... + ... +``` + +```curl +curl -XPUT 127.0.0.1:8098/types/indexes/buckets/people/keys/larry \ + -H "x-riak-index-field2_foo: 1001" \ + -d 'data1' + +# Response +Unknown field type for field: 'field2_foo'. +``` + +Incorrect data type: + +```java +Location key = new Location(new Namespace("people"), "larry"); +RiakObject obj = new RiakObject(); +obj.getIndexes().getIndex(LongIntIndex.named("field2")).add("bar"); + +// The Java client will return a response indicating a type mismatch. +// The output may look something like this: + +Error:(46, 68) java: no suitable method found for add(java.lang.String) + method com.basho.riak.client.query.indexes.RiakIndex.add(java.lang.Long) is not applicable + (argument mismatch; java.lang.String cannot be converted to java.lang.Long) + method com.basho.riak.client.query.indexes.RiakIndex.add(java.util.Collection<java.lang.Long>) is not applicable + (argument mismatch; java.lang.String cannot be converted to java.util.Collection<java.lang.Long>) +``` + +```ruby +bucket = client.bucket_type('indexes').bucket('people') +obj = Riak::RObject.new(bucket, 'larry') +obj.indexes['field2_int'] = %w{ bar } + +# The Ruby client will let you get away with this...at first. But when +# you attempt to store the object, you will get an error response such +# as this: + +NoMethodError: undefined method 'map' for 1001:Fixnum +``` + +```php +// throws \InvalidArgumentException +$object = (new \Basho\Riak\Object('{"user_data":{ ... }}', ['Content-type' => 'application/json'])) + ->addValueToIndex('twitter_int', 'not_an_int'); + +// throws \InvalidArgumentException +$object = (new \Basho\Riak\Object('{"user_data":{ ... }}', ['Content-type' => 'application/json'])) + ->addValueToIndex('twitter_int', ['arrays', 'are', 'not', 'strings']); + +// does not throw an exception, it will just write ints as a string +// only requirement is that value is scalar (int, float, string, bool) +$object = (new \Basho\Riak\Object('{"user_data":{ ... }}', ['Content-type' => 'application/json'])) + ->addValueToIndex('twitter_bin', 12); +``` + +```python +bucket = client.bucket_type('indexes').bucket('people') +obj = RiakObject(client, bucket, 'larry') +obj.add_index('field2_int', 'bar') + +# The Python client will let you get away with this...at first. But when you +# attempt to store the object, you will get an error response such as this: +riak.RiakError: '{precommit_fail,[{field_parsing_failed,{<<"field2_int">>,<<"bar">>}}]}' +``` + +```csharp +var id = new RiakObjectId("indexes", "people", "larry"); +var obj = new RiakObject(id, "test value", "text/plain"); +var intIdx = obj.IntIndex("test-int-idx"); +intIdx.Add("invalid-value"); + +// The .NET client will throw a FormatException at this point +// The output may look something like this: + +The value could not be parsed. +``` + +```javascript +var riakObj = new Riak.Commands.KV.RiakObject(); +riakObj.setContentType('text/plain'); +riakObj.setBucketType('indexes'); +riakObj.setBucket('people'); +riakObj.setKey('larry'); +riakObj.addToIndex('field2_int', 'bar'); +try { + client.storeValue({ value: riakObj }, function (err, rslt) { + logger.error("incorrect_data_type err: '%s'", err); + }); +} catch (e) { + logger.error("incorrect_data_type err: '%s'", e); +} + +// Output: +buffer.js:67 + throw new TypeError('must start with number, buffer, array or string'); + ^ +TypeError: must start with number, buffer, array or string + at new Buffer (buffer.js:67:11) +``` + +```erlang +Obj = riakc_obj:new( + {<<"indexes">>, <<"people">>}, + <<"larry">>, + <<"some data">>, + <<"text/plain">> +), +MD1 = riakc_obj:get_update_metadata(Obj), +MD2 = riakc_obj:set_secondary_index(MD1, [{{integer_index, "field2"}, [<<"bar">>]}]). + +%% The Erlang client will return an error message along these lines: +** exception error: bad argument + in function integer_to_list/1 + called as integer_to_list(<<"bar">>) ... +``` + +```golang +obj := &riak.Object{ + BucketType: "indexes", + Bucket: "people", + Key: "larry", + ContentType: "text/plain", + Charset: "utf-8", + ContentEncoding: "utf-8", + Value: []byte("My name is Larry"), +} +obj.AddToIndex("field2_int", "bar") + +cmd, err := riak.NewStoreValueCommandBuilder(). + WithContent(obj). + Build() +if err != nil { + return err +} + +if err := cluster.Execute(cmd); err != nil { + fmt.Println("[DevUsing2i] index data type error:", err) +} else { + return errors.New("[DevUsing2i] expected an error!") +} + +// The riak.Error object will contain: +{precommit_fail,[{field_parsing_failed,{<<"field2_int">>,<<"bar">>}}]} +``` + +```curl +curl -XPUT 127.0.0.1:8098/types/indexes/buckets/people/keys/larry \ + -H "x-riak-index-field2_int: bar" \ + -d 'data1' + +# Response +HTTP/1.1 400 Bad Request + +Could not parse field 'field2_int', value 'bar'. +``` + +## Querying + +> **Note on 2i queries and the R parameter** +> +> For all 2i queries, the [R]({{<baseurl>}}riak/kv/2.2.6/developing/app-guide/replication-properties#r-value-and-read-failure-tolerance) parameter is set to 1, +which means that queries that are run while [handoffs]({{<baseurl>}}riak/kv/2.2.6/learn/glossary/#hinted-handoff) and related operations are underway may not +return all keys as expected. +> +> To avoid issues such as the above, a new option has been added to the `riak.conf` file to allow you to disable or enable node participation in 2i queries. `participate_in_coverage=disabled` will prevent the node in question from participating. Recommended usage of this feature is to prevent newly added nodes to the cluster that have yet to receive all of their data from participating in 2i queries and generating non-consistent results. Changing the `participate_in_coverage` setting requires Riak to be restarted on that node for the change to take effect. The default setting is `enabled`. + +### Exact Match + +The following examples perform an exact match index query. + +Query a binary index: + +```java +Namespace myBucket = new Namespace("indexes", "people"); +BinIndexQuery biq = new BinIndexQuery.Builder(myBucket, "field1", "val1").build(); +BinIndexQuery.Response response = client.execute(biq); +``` + +```ruby +bucket = client.bucket_type('indexes').bucket('people') +bucket.get_index('field1_bin', 'val1') +``` + +```php +(new \Basho\Riak\Command\Builder\QueryIndex($riak)) + ->buildBucket('people', 'indexes') + ->withIndexName('field1_bin') + ->withScalarValue('val1') + ->build() + ->execute() + ->getResults(); +``` + +```python +bucket = client.bucket_type('indexes').bucket('people') +bucket.get_index('field1_bin', 'val1') +``` + +```csharp +var riakIndexId = new RiakIndexId("indexes", "people", "field1"); +// Note: using a string argument indicates a binary index query: +var indexRiakResult = client.GetSecondaryIndex(riakIndexId, "val1"); +var indexResult = indexRiakResult.Value; +``` + +```javascript +var binIdxCmd = new Riak.Commands.KV.SecondaryIndexQuery.Builder() + .withBucketType('indexes') + .withBucket('people') + .withIndexName('field1_bin') + .withIndexKey('val1') + .withCallback(query_cb) + .build(); +client.execute(binIdxCmd); +``` + +```erlang +{ok, Results} = riakc_pb_socket:get_index( + Pid, + {<<"indexes">>, <<"people">>}, %% bucket type and bucket name + {binary_index, "field2"}, + <<"val1">> +). +``` + +```golang +c1, err := riak.NewSecondaryIndexQueryCommandBuilder(). + WithBucketType("indexes"). + WithBucket("people"). + WithIndexName("field1_bin"). + WithIndexKey("val1"). + Build() +if err != nil { + return err +} +``` + +```curl +curl localhost:8098/types/indexes/buckets/people/index/field1_bin/val1 +``` + +Query an integer index: + +```java +Namespace myBucket = new Namespace("indexes", "people"); +IntIndexQuery iiq = new IntIndexQuery.Builder(myBucket, "field2", 1001L) + .build(); +IntIndexQuery.Response response = client.execute(iiq); +``` + +```ruby +bucket = client.bucket_type('indexes').bucket('people') +bucket.get_index('field2_int', 1001) +``` + +```php +(new \Basho\Riak\Command\Builder\QueryIndex($riak)) + ->buildBucket('people', 'indexes') + ->withIndexName('field2_int') + ->withScalarValue(1001) + ->build() + ->execute() + ->getResults(); +``` + +```python +bucket = client.bucket_type('indexes').bucket('people') +bucket.get_index('field2_int', 1001) +``` + +```csharp +var riakIndexId = new RiakIndexId("indexes", "people", "field2"); +// Note: using an integer argument indicates an int index query: +var indexRiakResult = client.GetSecondaryIndex(riakIndexId, 1001); +var indexResult = indexRiakResult.Value; +``` + +```javascript +var intIdxCmd = new Riak.Commands.KV.SecondaryIndexQuery.Builder() + .withBucketType('indexes') + .withBucket('people') + .withIndexName('field2_int') + .withIndexKey(1001) + .withCallback(query_cb) + .build(); +client.execute(intIdxCmd); +``` + +```erlang +{ok, Results} = riakc_pb_socket:get_index( + Pid, + {<<"indexes">>, <<"people">>}, %% bucket type and bucket name + {integer_index, "field2"}, + 1001 +). +``` + +```golang +cmd, err := riak.NewSecondaryIndexQueryCommandBuilder(). + WithBucketType("indexes"). + WithBucket("people"). + WithIndexName("field2_int"). + WithIntIndexKey(1001). + Build() +if err != nil { + return err +} +``` + +```curl +curl localhost:8098/types/indexes/buckets/people/index/field2_int/1001 +``` + +The following example performs an exact match query and pipes the +results into a MapReduce job: + +```curl +curl -XPOST localhost:8098/mapred \ + -H "Content-Type: application/json" \ + -d @-<<EOF +{ + "inputs": { + "bucket": "people", + "index": "field2_bin", + "key":"val3" + }, + "query": [ + { + "reduce": { + "language":"erlang", + "module": "riak_kv_mapreduce", + "function": "reduce_identity", + "keep": true + } + } + ] +} +EOF +``` + +### Range + +The following examples perform a range query. + +Query a binary index... + +```java +Namespace myBucket = new Namespace("indexes", "people"); +BinIndexQuery biq = new BinIndexQuery.Builder(myBucket, "field1", "val2", "val4") + .build(); +BinIndexQuery.Response response = client.execute(biq); +``` + +```ruby +bucket = client.bucket_type('indexes').bucket('people') +bucket.get_index('field1_bin', 'val2'..'val4') +``` + +```php +(new \Basho\Riak\Command\Builder\QueryIndex($riak)) + ->buildBucket('people', 'indexes') + ->withIndexName('field1_bin') + ->withRangeValue('val2', 'val4') + ->build() + ->execute() + ->getResults(); +``` + +```python +bucket = client.bucket_type('indexes').bucket('people') +bucket.get_index('field1_bin', 'val2', 'val4') +``` + +```csharp +var riakIndexId = new RiakIndexId("indexes", "people", "field1"); +var indexRiakResult = client.GetSecondaryIndex(riakIndexId, "val2", "val4"); +var indexResult = indexRiakResult.Value; +``` + +```javascript +var binIdxCmd = new Riak.Commands.KV.SecondaryIndexQuery.Builder() + .withBucketType('indexes') + .withBucket('people') + .withIndexName('field1_bin') + .withRange('val2', 'val4') + .withCallback(query_cb) + .build(); +client.execute(binIdxCmd); +``` + +```erlang +{ok, Results} = riakc_pb_socket:get_index_range( + Pid, + {<<"indexes">>, <<"people">>}, %% bucket type and bucket name + {binary_index, "field1"}, %% index name + <<"val2">>, <<"val4">> %% range query for keys between "val2" and "val4" +). +``` + +```golang +c1, err := riak.NewSecondaryIndexQueryCommandBuilder(). + WithBucketType("indexes"). + WithBucket("people"). + WithIndexName("field1_bin"). + WithRange("val2", "val4"). + Build() +if err != nil { + return err +} +``` + +```curl +curl localhost:8098/types/indexes/buckets/people/index/field1_bin/val2/val4 +``` + +Or query an integer index... + +```java +Namespace myBucket = new Namespace("indexes", "people"); +IntIndexQuery iiq = new IntIndexQuery.Builder(myBucket, "field2", 1002L, 1004L) + .build(); +IntIndexQuery.Response response = client.execute(iiq); +``` + +```ruby +bucket = client.bucket_type('indexes').bucket('people') +bucket.get_index('field2_int', 1002..1004) +``` + +```php +(new \Basho\Riak\Command\Builder\QueryIndex($riak)) + ->buildBucket('people', 'indexes') + ->withIndexName('field2_int') + ->withRangeValue(1002, 1004) + ->build() + ->execute() + ->getResults(); +``` + +```python +bucket = client.bucket_type('indexes').bucket('people') +bucket.get_index('field2_int', 1002, 1004) +``` + +```csharp +var riakIndexId = new RiakIndexId("indexes", "people", "field2"); +var indexRiakResult = client.GetSecondaryIndex(riakIndexId, 1002, 1004); +var indexResult = indexRiakResult.Value; +``` + +```javascript +var intIdxCmd = new Riak.Commands.KV.SecondaryIndexQuery.Builder() + .withBucketType('indexes') + .withBucket('people') + .withIndexName('field2_int') + .withRange(1002, 1004) + .withCallback(query_cb) + .build(); +client.execute(intIdxCmd); +``` + +```erlang +{ok, Results} = riakc_pb_socket:get_index_range( + Pid, + {<<"indexes">>, <<"people">>}, %% bucket type and bucket name + {integer_index, "field2"}, %% index name + 1002, 1004 %% range query for keys between "val2" and "val4" +). +``` + +```golang +cmd, err := riak.NewSecondaryIndexQueryCommandBuilder(). + WithBucketType("indexes"). + WithBucket("people"). + WithIndexName("field2_int"). + WithIntRange(1002, 1004). + Build() +``` + +```curl +curl localhost:8098/types/indexes/buckets/people/index/field2_int/1002/1004 +``` + +The following example performs a range query and pipes the results into +a MapReduce job: + +```curl +curl -XPOST localhost:8098/mapred\ + -H "Content-Type: application/json" \ + -d @-<<EOF +{ + "inputs": { + "bucket": "people", + "index": "field2_bin", + "start": "1002", + "end": "1004" + }, + "query": [ + { + "reduce": { + "language": "erlang", + "module": "riak_kv_mapreduce", + "function": "reduce_identity", + "keep": true + } + } + ] +} +EOF +``` + +#### Range with terms + +When performing a range query, it is possible to retrieve the matched +index values alongside the Riak keys using `return_terms=true`. An +example from a small sampling of Twitter data with indexed hash tags: + +```java +Namespace tweetsBucket = new Namespace("indexes", "tweets"); +BinIndexQuery biq = new BinIndexQuery.Builder(tweetsBucket, "hashtags", "rock", "rocl") + .withKeyAndIndex(true) + .build(); +BinIndexQuery.Response response = client.execute(biq); +``` + +```ruby +bucket = client.bucket_type('indexes').bucket('tweets') +bucket.get_index('hashtags_bin', 'rock'..'rocl', return_terms: true) +``` + +```php +(new \Basho\Riak\Command\Builder\QueryIndex($riak)) + ->buildBucket('tweets', 'indexes') + ->withIndexName('hashtags') + ->withRangeValue('rock', 'rocl') + ->withReturnTerms() + ->build() + ->execute() + ->getResults(); +``` + +```python +bucket = client.bucket_type('indexes').bucket('tweets') +bucket.get_index('hashtags_bin', 'rock', 'rocl', return_terms=True) +``` + +```csharp +var riakIndexId = new RiakIndexId("indexes", "tweets", "hashtags"); +var options = new RiakIndexGetOptions(); +options.SetReturnTerms(true); +var indexRiakResult = client.GetSecondaryIndex(riakIndexId, "rock", "rocl", options); +var indexResult = indexRiakResult.Value; +``` + +```javascript +var binIdxCmd = new Riak.Commands.KV.SecondaryIndexQuery.Builder() + .withBucketType('indexes') + .withBucket('tweets') + .withIndexName('hashtags_bin') + .withRange('rock', 'rocl') + .withReturnKeyAndIndex(true) + .withCallback(query_cb) + .build(); +client.execute(binIdxCmd); +``` + +```erlang +{ok, Results} = riakc_pb_socket:get_index_range( + Pid, + {<<"indexes">>, <<"people">>}, %% bucket type and bucket name + {binary_index, "hashtags"}, %% index name + <<"rock">>, <<"rocl">> %% range query for keys between "val2" and "val4" +). +``` + +```golang +cmd, err := riak.NewSecondaryIndexQueryCommandBuilder(). + WithBucketType("indexes"). + WithBucket("tweets"). + WithIndexName("hashtags_bin"). + WithRange("rock", "rocl"). + Build() +if err != nil { + return err +} + +if err := cluster.Execute(cmd); err != nil { + return err +} +``` + +```curl +curl localhost:8098/types/indexes/buckets/tweets/index/hashtags_bin/rock/rocl?return_terms=true +``` + +Response: + +```json +{ + "results": [ + { + "rock": "349224101224787968" + }, + { + "rocks": "349223639880699905" + } + ] +} +``` + +### Pagination + +When asking for large result sets, it is often desirable to ask the +servers to return chunks of results instead of a firehose. You can do so +using `max_results=<n>`, where `n` is the number of results you'd like +to receive. + +Assuming more keys are available, a `continuation` value will be +included in the results to allow the client to request the next page. + +Here is an example of a range query with both `return_terms` and +pagination against the same Twitter data set. + +```java +Namespace tweetsBucket = new Namespace("indexes", "tweets"); +BinIndexQuery biq = new BinIndexQuery.Builder(tweetsBucket, "hashtags", "ri", "ru") + .withMaxResults(5) + .build(); +BinIndexQuery.Response response = client.execute(biq); +``` + +```ruby +bucket = client.bucket_type('indexes').bucket('tweets') +bucket.get_index('hashtags_bin', 'ri'..'ru', max_results: 5) +``` + +```php +(new \Basho\Riak\Command\Builder\QueryIndex($riak)) + ->buildBucket('tweets', 'indexes') + ->withIndexName('hashtags') + ->withRangeValue('ri', 'ru') + ->withMaxResults(5) + ->build() + ->execute() + ->getResults(); +``` + +```python +bucket = client.bucket_type('indexes').bucket('tweets') +bucket.get_index('hashtags_bin', 'ri', 'ru', max_results=5) +``` + +```csharp +var idxId = new RiakIndexId("indexes", "tweets", "hashtags"); +var options = new RiakIndexGetOptions(); +options.SetMaxResults(5); +var rslt = client.GetSecondaryIndex(idxId, "ri", "ru", options); + +options.SetContinuation(rslt.Continuation); +rslt = client.GetSecondaryIndex(idxId, "ri", "ru", options); +``` + +```javascript +function do_query(continuation) { + var binIdxCmdBuilder = new Riak.Commands.KV.SecondaryIndexQuery.Builder() + .withBucketType('indexes') + .withBucket('tweets') + .withIndexName('hashtags_bin') + .withRange('ri', 'ru') + .withMaxResults(5) + .withCallback(pagination_cb); + + if (continuation) { + binIdxCmdBuilder.withContinuation(continuation); + } + + client.execute(binIdxCmdBuilder.build()); +} + +var query_keys = []; +function pagination_cb(err, rslt) { + if (err) { + logger.error("query_cb err: '%s'", err); + return; + } + + if (rslt.done) { + query_keys.forEach(function (key) { + logger.info("2i query key: '%s'", key); + }); + query_keys = []; + + if (rslt.continuation) { + do_query(rslt.continuation); + } + } + + if (rslt.values.length > 0) { + Array.prototype.push.apply(query_keys, + rslt.values.map(function (value) { + return value.objectKey; + })); + } +} + +do_query(); +``` + +```erlang +{ok, Results} = riakc_pb_socket:get_index_range( + Pid, + {<<"indexes">>, <<"tweets">>}, %% bucket type and bucket name + {binary_index, "hashtags"}, %% index name + <<"ri">>, <<"ru">>, %% range query from "ri" to "ru" + {max_results, 5} +). +``` + +```golang +func doPaginatedQuery(cluster *riak.Cluster, continuation []byte) error { + builder := riak.NewSecondaryIndexQueryCommandBuilder(). + WithBucketType("indexes"). + WithBucket("tweets"). + WithIndexName("hashtags_bin"). + WithRange("ri", "ru"). + WithMaxResults(5) + + if continuation != nil && len(continuation) > 0 { + builder.WithContinuation(continuation) + } + + cmd, err := builder.Build() + if err != nil { + return err + } + + if err := cluster.Execute(cmd); err != nil { + return err + } + + printIndexQueryResults(cmd) + + sciq := cmd.(*riak.SecondaryIndexQueryCommand) + if sciq.Response == nil { + return errors.New("[DevUsing2i] expected response but did not get one") + } + + rc := sciq.Response.Continuation + if rc != nil && len(rc) > 0 { + return doPaginatedQuery(cluster, sciq.Response.Continuation) + } + + return nil +} + +func queryingPagination(cluster *riak.Cluster) error { + return doPaginatedQuery(cluster, nil) +} +``` + +```curl +curl localhost:8098/types/indexes/buckets/tweets/index/hashtags_bin/ri/ru?max_results=5&return_terms=true +``` + +Here is an example JSON response (your client-specific response may differ): + +```json +{ + "continuation": "g2gCbQAAAAdyaXBqYWtlbQAAABIzNDkyMjA2ODcwNTcxMjk0NzM=", + "results": [ + { "rice": "349222574510710785" }, + { "rickross": "349222868095217664" }, + { "ridelife": "349221819552763905" }, + { "ripjake": "349220649341952001" }, + { "ripjake": "349220687057129473" } + ] +} +``` + +Take the continuation value from the previous result set and feed it +back into the query. + +```java +Namespace tweetsBucket = new Namespace("indexes", "tweets"); +BinIndexQuery biq = new BinIndexQuery.Builder(tweetsBucket, "hashtags", "ri", "ru") + .withContinuation(BinaryValue.create("g2gCbQAAAAdyaXBqYWtlbQAAABIzNDkyMjA2ODcwNTcxMjk0NzM")) + .withMaxResults(5) + .withKeyAndIndex(true) + .build(); +BinIndexQuery.Response response = client.execute(biq); +``` + +```ruby +bucket = client.bucket_type('indexes').bucket('tweets') +bucket.get_index( + 'hashtags_bin', + 'ri'..'ru', + continuation: 'g2gCbQAAAAdyaXBqYWtlbQAAABIzNDkyMjA2ODcwNTcxMjk0NzM', + max_results: 5, + return_terms: true +) +``` + +```php +(new \Basho\Riak\Command\Builder\QueryIndex($riak)) + ->buildBucket('tweets', 'indexes') + ->withIndexName('hashtags') + ->withRangeValue('ri', 'ru') + ->withMaxResults(5) + ->withContinuation('g2gCbQAAAAdyaXBqYWtlbQAAABIzNDkyMjA2ODcwNTcxMjk0NzM') + ->build() + ->execute() + ->getResults(); +``` + +```python +bucket = client.bucket_type('indexes').bucket('tweets') +bucket.get_index( + 'hashtags_bin', + 'ri', 'ru', + continuation='g2gCbQAAAAdyaXBqYWtlbQAAABIzNDkyMjA2ODcwNTcxMjk0NzM', + max_results=5, + return_terms=True +) +``` + +```csharp +// rslt is the previous 2i fetch result +var idxId = new RiakIndexId("indexes", "tweets", "hashtags"); +var options = new RiakIndexGetOptions(); +options.SetMaxResults(5); +options.SetContinuation(rslt.Continuation); +rslt = client.GetSecondaryIndex(idxId, "ri", "ru", options); +``` + +```javascript +// See above example +``` + +```erlang +{ok, Results} = riakc_pb_socket:get_index_range( + Pid, + {<<"indexes">>, <<"tweets">>}, %% bucket type and bucket name + {binary_index, "hashtags"}, %% index name + <<"ri">>, <<"ru">>, %% range query from "ri" to "ru" + [ + {continuation, <<"g2gCbQAAAAdyaXBqYWtlbQAAABIzNDkyMjA2ODcwNTcxMjk0NzM">>}, + {max_results, 5}, + {return_terms, true} + ] +). +``` + +```golang +// See above example +``` + +```curl +curl localhost:8098/types/indexes/buckets/tweets/index/hashtags_bin/ri/ru?continuation=g2gCbQAAAAdyaXBqYWtlbQAAABIzNDkyMjA2ODcwNTcxMjk0NzM=&max_results=5&return_terms=true +``` + +The result: + +```json +{ + "continuation": "g2gCbQAAAAlyb2Jhc2VyaWFtAAAAEjM0OTIyMzcwMjc2NTkxMjA2NQ==", + "results": [ + { + "ripjake": "349221198774808579" + }, + { + "ripped": "349224017347100672" + }, + { + "roadtrip": "349221207155032066" + }, + { + "roastietime": "349221370724491265" + }, + { + "robaseria": "349223702765912065" + } + ] +} +``` + +### Streaming + +It is also possible to stream results: + +```java +// Available in Riak Java Client 2.1.0 and later +int pollTimeoutMS = 200; +Namespace ns = new Namespace("indexes", "tweets"); +String indexName = "hashtags"; + +BinIndexQuery indexQuery = + new BinIndexQuery.Builder(ns, indexName, "ri", "ru").build(); + +final RiakFuture<BinIndexQuery.StreamingResponse, BinIndexQuery> streamingFuture = + client.executeAsyncStreaming(indexQuery, pollTimeoutMS); + +// For streaming commands, the future's value will be available before +// the future is complete, so you may begin to pull results from the +// provided iterator as soon as possible. +final BinIndexQuery.StreamingResponse streamingResponse = streamingFuture.get(); + +for (BinIndexQuery.Response.Entry e : streamingResponse) +{ + // Do something with key... +} + +streamingFuture.await(); +Assert.assertTrue(streamingFuture.isDone()); +``` + +```ruby +bucket = client.bucket_type('indexes').bucket('people') +bucket.get_index('myindex_bin', 'foo', stream: true) +``` + +```php +/* + It is not currently possible to stream results using the PHP client +*/ +``` + +```python +bucket = client.bucket_type('indexes').bucket('people') +keys = [] +for key in bucket.stream_index('myindex_bin', 'foo'): + keys.append(key) +``` + +```csharp +var riakIndexId = new RiakIndexId("indexes", "tweets", "hashtags"); +var indexRiakResult = client.StreamGetSecondaryIndex(riakIndexId, "ri", "ru"); +var indexResult = indexRiakResult.Value; +foreach (var key in indexResult.IndexKeyTerms) +{ + // Do something with key... +} +``` + +```javascript +var binIdxCmd = new Riak.Commands.KV.SecondaryIndexQuery.Builder() + .withBucketType('indexes') + .withBucket('tweets') + .withIndexName('hashtags_bin') + .withRange('ri', 'ru') + .withStreaming(true); + .withCallback(query_cb) // See query_cb in other examples + .build(); +client.execute(binIdxCmd); +``` + +```erlang +{ok, KeyStream} = riakc_pb_socket:get_index_eq( + Pid, + {<<"indexes">>, <<"people">>}, %% bucket type and bucket name + {binary_index, "myindex"}, %% index name and type + <<"foo">>, %% value of the index + [{stream, true}] %% enable streaming +). +``` + +```golang +cmd, err := riak.NewSecondaryIndexQueryCommandBuilder(). + WithBucketType("indexes"). + WithBucket("tweets"). + WithIndexName("hashtags_bin"). + WithRange("ri", "ru"). + WithStreaming(true). + WithCallback(streamingCallback). + Build() +if err != nil { + return err +} + +if err := cluster.Execute(cmd); err != nil { + return err +} +``` + +```curl +curl localhost:8098/types/indexes/buckets/people/index/myindex_bin/foo?stream=true +``` + +Streaming can also be combined with `pagination` and `return_terms`. + +### Sorting + +As of Riak 1.4, the result set is sorted on index values (when executing +range queries) and object keys. See the pagination example above: hash +tags (2i keys) are returned in ascending order, and the object keys +(Twitter IDs) for the messages which contain the `ripjake` hash tag are +also returned in ascending order. + +### Retrieve all Bucket Keys via the `$bucket` Index + +The following example retrieves the keys for all objects stored in the +bucket `people` using an exact match on the special `$bucket` index. + +```curl +curl localhost:8098/types/indexes/buckets/people/index/\$bucket/_ +``` + +### Count Bucket Objects via $bucket Index + +The following example performs a secondary index lookup on the $bucket +index like in the previous example and pipes this into a MapReduce that +counts the number of records in the `people` bucket. In order to +improve efficiency, the batch size has been increased from the default +size of 20. + +```curl +curl -XPOST localhost:8098/mapred\ + -H "Content-Type: application/json" \ + -d @-<<EOF +{ + "inputs": { + "bucket": "people", + "index": "\$bucket", + "key":"people" + }, + "query": [ + { + "reduce": { + "language": "erlang", + "module": "riak_kv_mapreduce", + "function": "reduce_count_inputs", + "arg": { + "reduce_phase_batch_size":1000 + } + } + } + ] +} +EOF +``` diff --git a/content/riak/kv/2.2.6/developing/usage/security.md b/content/riak/kv/2.2.6/developing/usage/security.md new file mode 100644 index 0000000000..5ed9cb0058 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/usage/security.md @@ -0,0 +1,99 @@ +--- +title: "Client Security" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Security" + identifier: "usage_security" + weight: 114 + parent: "developing_usage" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/advanced/client-security + - /riak-docs/riak/kv/2.2.6/dev/advanced/client-security +--- + +Versions of Riak 2.0 and later come equipped with a [security subsystem]({{<baseurl>}}riak/kv/2.2.6/using/security/basics) that enables you to choose + +* which Riak users/clients are authorized to perform a wide variety of + Riak operations, and +* how those users/clients are required to authenticate themselves. + +The following four authentication mechanisms, aka [security sources]({{<baseurl>}}riak/kv/2.2.6/using/security/managing-sources/) are available: + +* [Trust]({{<baseurl>}}riak/kv/2.2.6/using/security/managing-sources/#trust-based-authentication)-based + authentication enables you to specify trusted + [CIDR](http://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing)s + from which all clients will be authenticated by default +* [Password]({{<baseurl>}}riak/kv/2.2.6/using/security/managing-sources/#password-based-authentication)-based authentication requires + that clients provide a username and password +* [Certificate]({{<baseurl>}}riak/kv/2.2.6/using/security/managing-sources/#certificate-based-authentication)-based authentication + requires that clients +* [Pluggable authentication module (PAM)]({{<baseurl>}}riak/kv/2.2.6/using/security/managing-sources/#pam-based-authentication)-based authentication requires + clients to authenticate using the PAM service specified using the + [`riak-admin security`]({{<baseurl>}}riak/kv/2.2.6/using/security/managing-sources/#managing-sources) + command line interface + +Riak's approach to security is highly flexible. If you choose to use +Riak's security feature, you do not need to require that all clients +authenticate via the same means. Instead, you can specify authentication +sources on a client-by-client, i.e. user-by-user, basis. This means that +you can require clients performing, say, [MapReduce]({{<baseurl>}}riak/kv/2.2.6/developing/usage/mapreduce/) +operations to use certificate auth, while clients performing [K/V Operations]({{<baseurl>}}riak/kv/2.2.6/developing/usage) have to use username and password. The approach +that you adopt will depend on your security needs. + +This document provides a general overview of how that works. For +managing security in Riak itself, see the following documents: + +* [Authentication and Authorization]({{<baseurl>}}riak/kv/2.2.6/using/security/basics) +* [Managing Security Sources]({{<baseurl>}}riak/kv/2.2.6/using/security/managing-sources/) + +We also provide client-library-specific guides for the following +officially supported clients: + +* [Java]({{<baseurl>}}riak/kv/2.2.6/developing/usage/security/java) +* [Ruby]({{<baseurl>}}riak/kv/2.2.6/developing/usage/security/ruby) +* [PHP]({{<baseurl>}}riak/kv/2.2.6/developing/usage/security/php) +* [Python]({{<baseurl>}}riak/kv/2.2.6/developing/usage/security/python) +* [Erlang]({{<baseurl>}}riak/kv/2.2.6/developing/usage/security/erlang) + +## Certificates, Keys, and Authorities + +If Riak security is enabled, all client operations, regardless of the +security source you choose for those clients, must be over a secure SSL +connection. If you are using a self-generated Certificate Authority +(CA), Riak and connecting clients will need to share that CA. + +To use certificate-based auth, you will need to create a Public Key +Infrastructure (PKI) based on +[x.509](http://en.wikipedia.org/wiki/X.509) certificates. The central +foundation of your PKI should be a Certificate Authority (CA), created +inside of a secure environment, that can be used to sign certificates. +In addition to a CA, your client will need to have access to a private +key shared only by the client and Riak as well as a CA-generated +certificate. + +To prevent so-called [Man-in-the-Middle +attacks](http://en.wikipedia.org/wiki/Man-in-the-middle_attack), private +keys should never be shared beyond Riak and connecting clients. + +> **HTTP not supported** +> +> Certificate-based authentication is available only through Riak's +[Protocol Buffers]({{<baseurl>}}riak/kv/2.2.6/developing/api/protocol-buffers/) interface. It is not available through the +[HTTP API]({{<baseurl>}}riak/kv/2.2.6/developing/api/http). + +### Default Names + +In Riak's [configuration files]({{<baseurl>}}riak/kv/2.2.6/configuring/reference/#security), the +default certificate file names are as follows: + +Cert | Filename +:----|:------- +Certificate authority (CA) | `cacertfile.pem` +Private key | `key.pem` +CA-generated cert | `cert.pem` + +These filenames will be used in the client-library-specific tutorials. diff --git a/content/riak/kv/2.2.6/developing/usage/security/erlang.md b/content/riak/kv/2.2.6/developing/usage/security/erlang.md new file mode 100644 index 0000000000..c2cfac3ab3 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/usage/security/erlang.md @@ -0,0 +1,114 @@ +--- +title_supertext: "Client Security:" +title: "Erlang" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Erlang" + identifier: "usage_security_erlang" + weight: 103 + parent: "usage_security" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/advanced/client-security/erlang + - /riak-docs/riak/kv/2.2.6/dev/advanced/client-security/erlang +--- + +This tutorial shows you how to set up a Riak Erlang client to +authenticate itself when connecting to Riak. + +If you are using [trust]({{<baseurl>}}riak/kv/2.2.6/using/security/managing-sources/), [PAM-]({{<baseurl>}}riak/kv/2.2.6/using/security/managing-sources/#pam-based-authentication), you can use the security setup described [below](#erlang-client-basics). [Password]({{<baseurl>}}riak/kv/2.2.6/using/security/managing-sources/#password-based-authentication)-based authentication is covered +in a [later section](#password-based-authentication). If you are using +[certificate]({{<baseurl>}}riak/kv/2.2.6/using/security/managing-sources/#certificate-based-authentication)-based authentication, follow +the instructions in the [section below](#certificate-based-authentication). + +{{% note title="Note on certificate generation" %}} +This tutorial does not cover certificate generation. It assumes that all +necessary certificates have already been created and are stored in a directory +called `/ssl_dir`. This directory name is used only for example purposes. +{{% /note %}} + +## Erlang Client Basics + +When connecting to Riak using an Erlang-based client, you typically use +a process identifier to refer to the client connection. The following +example creates a process identifier (we'll call it `Pid`) for a +connection to `localhost` on port 8087: + +```erlang +{ok, Pid} = riakc_pb_socket:start("127.0.0.1", 8087). +``` + +If you are using Riak security, _all_ connecting clients should have +access to the same Certificate Authority (CA) used on the server side, +regardless of which [security source]({{<baseurl>}}riak/kv/2.2.6/using/security/managing-sources/) you +choose. In addition, all clients should provide a username. The example +above created a connection to Riak without specifying a username or CA. +That information is specified as a list of options passed to the +`start` function. We'll specify those options in a list called +`SecurityOptions`. + +```erlang +CertDir = "/ssl_dir", +SecurityOptions = [ + {credentials, "riakuser", ""}, + {cacertfile, filename:join([CertDir, "cacertfile.pem"])} + ], +{ok, Pid} = riakc_pb_socket:start("127.0.0.1", 8087, SecurityOptions). +``` + +Please note that you do not need to specify a password if you are not +using password-based authentication. If you are using a different +security source, Riak will ignore the password. You can enter an empty +string (as in the example above) or anything you'd like. + +This client is not currently set up to use any of the available security +sources, with the exception of trust-based authentication, provided that +the [CIDR](http://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing) +from which the client is connecting has been specified as trusted. More +on specifying trusted CIDRs can be found in [Trust-based Authentication]({{<baseurl>}}riak/kv/2.2.6/using/security/managing-sources/#trust-based-authentication). + +## Password-based Authentication + +To enable our client to use password-based auth, we can use most of the +information from the example above, with the exception that we'll also +specify a password for the client in the `SecurityOptions` list from +above. We'll use the password `rosebud` here and in the rest of the +examples. + +```erlang +CertDir = "/ssl_dir", +SecurityOptions = [ + {credentials, "riakuser", "rosebud"}, + {cacertfile, filename:join([CertDir, "cacertfile.pem"])} + ], +{ok, Pid} = riakc_pb_socket:start("127.0.0.1", 8087, SecurityOptions). +``` + +## PAM-based Authentication + +If you have specified that a specific client be authenticated using +[PAM]({{<baseurl>}}riak/kv/2.2.6/using/security/managing-sources/#pam-based-authentication), you will +need to provide a CA as well as the username and password that you +specified when creating the user in Riak. For more, see our +documentation on [User Management]({{<baseurl>}}riak/kv/2.2.6/using/security/basics/#user-management). + +## Certificate-based Authentication + +Using certificate-based authentication requires us to specify the +location of a general CA (as with all security sources), a username, a +CA-generated cert, and a private key. We'll assume that all certs are +stored in `/ssl_dir`, as in the previous examples. + +```erlang +CertDir = "/ssl_dir", +SecurityOptions = [ + {credentials, "riakuser", "rosebud"}, + {cacertfile, filename:join([CertDir, "cacertfile.pem"])}, + {certfile, filename:join([CertDir, "cert.pem"])}, + {keyfile, filename:join([CertDir, "key.pem"])} + ], +{ok, Pid} = riakc_pb_socket:start("127.0.0.1", 8087, SecurityOptions). +``` diff --git a/content/riak/kv/2.2.6/developing/usage/security/java.md b/content/riak/kv/2.2.6/developing/usage/security/java.md new file mode 100644 index 0000000000..7f5821a9d9 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/usage/security/java.md @@ -0,0 +1,117 @@ +--- +title_supertext: "Client Security:" +title: "Java" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Java" + identifier: "usage_security_java" + weight: 100 + parent: "usage_security" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/advanced/client-security/java + - /riak-docs/riak/kv/2.2.6/dev/advanced/client-security/java +--- + +This tutorial shows you how to set up a Riak Java client to authenticate +itself when connecting to Riak. + +If you are using [trust-]({{<baseurl>}}riak/kv/2.2.6/using/security/managing-sources/#trust-based-authentication) or [PAM]({{<baseurl>}}riak/kv/2.2.6/using/security/managing-sources/#pam-based-authentication)-based authentication, you can use the +security setup described [below](#java-client-basics). [Certificate]({{<baseurl>}}riak/kv/2.2.6/using/security/managing-sources/#certificate-based-authentication)-based authentication is not +yet supported in the Java client. + +{{% note title="Note on certificate generation" %}} +This tutorial does not cover certificate generation. It assumes that all +necessary certificates have already been created and are stored in a directory +called `/ssl_dir`. This directory name is used only for example purposes. +{{% /note %}} + +## Java Client Basics + +When connecting to Riak using a Java-based client, you typically do so +by instantiating separate `RiakNode` objects for each node in your +cluster, a `RiakCluster` object registering those `RiakNode` objects, +and finally a `RiakClient` object that registers the general cluster +configuration. In this document, we will be working with only one node. + +If you are using Riak security, _all_ connecting clients should have +access to the same Certificate Authority (CA) used on the server side, +regardless of which [security source]({{<baseurl>}}riak/kv/2.2.6/using/security/managing-sources/) you +choose. All clients should also provide a username, regardless of +security source. The example below sets up a single node object (we'll +simply call it `node`) that connects to Riak on `localhost` and on port +8087 and specifies `riakuser` as a username. That object will be used to +create a cluster object (we'll call it `cluster`), which will in turn be +used to create a `client` object. The setup below does not specify a CA: + +```java +import com.basho.riak.client.api.RiakClient; +import com.basho.riak.client.api.RiakCluster; +import com.basho.riak.client.api.RiakNode; + +RiakNode node = new RiakNode.Builder() + .withRemoteAddress("127.0.0.1") + .withRemotePort(8087) + // This will specify a username but no password or keystore: + .withAuth("riakuser", null, null) + .build(); + +RiakCluster cluster = new RiakCluster.Builder(node) + .build(); + +RiakClient client = new RiakClient(cluster); +``` + +This client object is not currently set up to use any of the available +security sources. This will change in the sections below. + +## Password-based Authentication + +To enable our client to use password-based auth, we can use most of the +setup from the example above, with the exception that we will specify a +password for the client in the `withAuth` method in the `node` object's +constructor rather than leaving it as `null`. We will also pass a +`KeyStore` object into that method. + +```java +import java.io.FileInputStream; +import java.io.InputStream; +import java.security.KeyStore; +import java.security.cert.CertificateFactory; +import java.security.cert.X509Certificate; + +// Generate an InputStream from the CA cert +InputStream inputStream = new InputStream("/ssl_dir/cacertfile.pem"); + +// Generate an X509Certificate from the InputStream and close the stream +CertificateFactory certFactory = CertificateFactory.getInstance("X.509"); +X509Certificate caCert = (X509Certificate) certFactory.generateCertificate(inputStream); +inputStream.close(); + +// Generate a KeyStore object +KeyStore ks = KeyStore.getInstance(KeyStore.getDefaultType()); +ks.load(null, "password".toCharArray()); +ks.setCertificateEntry("cacert", caCert); + +RiakNode node = new RiakNode.Builder() + .withRemoteAddress("127.0.0.1") + .withRemotePort(8087) + .withAuth("riakuser", "rosebud", ks) + .build(); + +// Construct the cluster and client object in the same fashion as above +``` + +## PAM- and Trust-based Authentication + +If you are using PAM- or trust-based authentication, the only difference +from password-based authentication is that you do not need to specify a +password. + +## Certificate-based Authentication + +Certificate-based authentication is not currently supported in the +official Riak Java client. diff --git a/content/riak/kv/2.2.6/developing/usage/security/php.md b/content/riak/kv/2.2.6/developing/usage/security/php.md new file mode 100644 index 0000000000..5dc4d35d89 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/usage/security/php.md @@ -0,0 +1,118 @@ +--- +title_supertext: "Client Security:" +title: "PHP" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "PHP" + identifier: "usage_security_php" + weight: 104 + parent: "usage_security" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/advanced/client-security/php + - /riak-docs/riak/kv/2.2.6/dev/advanced/client-security/php +--- + +This tutorial shows you how to set up a Riak PHP client to authenticate +itself when connecting to Riak. + +If you are using [trust-]({{<baseurl>}}riak/kv/2.2.6/using/security/managing-sources/#trust-based-authentication) or [PAM]({{<baseurl>}}riak/kv/2.2.6/using/security/managing-sources/#pam-based-authentication)-based authentication, you can use the +security setup described [below](#php-client-basics). [Certificate]({{<baseurl>}}riak/kv/2.2.6/using/security/managing-sources/#certificate-based-authentication)-based authentication is not +yet supported in the PHP client due to limitations of the HTTP interface of Riak. + +## PHP Client Basics + +When connecting to Riak using a PHP-based client, you typically do so +by instantiating separate `\Basho\Riak\Node` objects for each node in your +cluster and passing those `\Basho\Riak\Node` objects as an array to a +`\Basho\Riak` object as a dependency. In this document, we will be working with +only one node. + +If you are using Riak security, _all_ connecting clients should have +access to the same Certificate Authority (CA) used on the server side, +regardless of which [security source]({{<baseurl>}}riak/kv/2.2.6/using/security/managing-sources/) you choose. All clients should also provide a username, regardless of +security source. The example below sets up a single node object (we'll +simply call it `node`) that connects to Riak on `localhost` and on port +8087 and specifies `riakuser` as a username. That object will be used to +create a Riak object. The setup below does not specify a CA and will throw +an `\Basho\Riak\Node\Builder\Exception`: + +```php +use \Basho\Riak; +use \Basho\Riak\Node; + +$node = (new Node\Builder()) + ->atHost('127.0.0.1') + ->onPort('8087') + ->usingPasswordAuthentication('riakuser') + ->build(); + +// since we are using a single node, it needs to be wrapped in array brackets +$riak = new Riak([$node]); +``` + +This client object is not currently set up to use any of the available +security sources. This will change in the sections below. + +## Password-based Authentication + +To enable our client to use password-based auth, we can use most of the +setup from the example above, with the exception that we will specify a +password for the client in the `usingPasswordAuthentication` method in +the `node` object's builder rather than omitting it. We will also +pass the path of the CA file relative to the current working directory into +the `withCertificateAuthorityFile` method. + +```php +use \Basho\Riak; +use \Basho\Riak\Node; + +$node = (new Node\Builder()) + ->atHost('127.0.0.1') + ->onPort('8087') + ->usingPasswordAuthentication('riakuser', 'rosebud') + ->withCertificateAuthorityFile(getcwd() . '/ssl_dir/cacertfile.pem') + ->build(); + +// since we are using a single node, it needs to be wrapped in array brackets +$riak = new Riak([$node]); +``` + +## PAM- and Trust-based Authentication + +If you are using PAM- or trust-based authentication, the only difference +from password-based authentication is that you do not need to specify a +password. There are helper methods that handle this for you, +`usingPamAuthentication` and `usingTrustAuthentication`. + +```php +use \Basho\Riak; +use \Basho\Riak\Node; + +// PAM Example +$node = (new Node\Builder()) + ->atHost('127.0.0.1') + ->onPort('8087') + ->usingPamAuthentication('riakuser') + ->withCertificateAuthorityFile(getcwd() . '/ssl_dir/cacertfile.pem') + ->build(); + +// Trust Example +$node = (new Node\Builder()) + ->atHost('127.0.0.1') + ->onPort('8087') + ->usingTrustAuthentication('riakuser') + ->withCertificateAuthorityFile(getcwd() . '/ssl_dir/cacertfile.pem') + ->build(); + +// since we are using a single node, it needs to be wrapped in array brackets +$riak = new Riak([$node]); +``` + +## Certificate-based Authentication + +Certificate-based authentication is not currently supported in the +official Riak PHP client due to limitations in the HTTP interface. diff --git a/content/riak/kv/2.2.6/developing/usage/security/python.md b/content/riak/kv/2.2.6/developing/usage/security/python.md new file mode 100644 index 0000000000..9b4350585b --- /dev/null +++ b/content/riak/kv/2.2.6/developing/usage/security/python.md @@ -0,0 +1,172 @@ +--- +title_supertext: "Client Security:" +title: "Python" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Python" + identifier: "usage_security_python" + weight: 102 + parent: "usage_security" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/advanced/client-security/python + - /riak-docs/riak/kv/2.2.6/dev/advanced/client-security/python +--- + +This tutorial shows you how to set up a Riak Python client to +authenticate itself when connecting to Riak. + +If you are using [trust-]({{<baseurl>}}riak/kv/2.2.6/using/security/managing-sources/) or [PAM-]({{<baseurl>}}riak/kv/2.2.6/using/security/managing-sources/#pam-based-authentication), you can use the security +setup described [below](#python-client-basics). [Password]({{<baseurl>}}riak/kv/2.2.6/using/security/managing-sources/#password-based-authentication)-based authentication is covered +in a [later section](#password-based-authentication). If you are using +[certificate]({{<baseurl>}}riak/kv/2.2.6/using/security/managing-sources/#certificate-based-authentication)-based authentication, follow +the instructions in the [section below](#certificate-based-authentication). + +{{% note title="Note on certificate generation" %}} +This tutorial does not cover certificate generation. It assumes that all +necessary certificates have already been created and are stored in a directory +called `/ssl_dir`. This directory name is used only for example purposes. +{{% /note %}} + +## OpenSSL Versions + +The Riak Python client requires that you install OpenSSL 1.0.1g or +later. If you have an earlier version installed, you will receive a +warning along the following lines: + +``` +Found OpenSSL 0.9.8za 5 Jun 2014 version, but expected at least OpenSSL 1.0.1g. Security may not support TLS 1.2. +``` + +## Python Client Basics + +When connecting to Riak using a Python-based client, you typically +instantiate an object from the `RiakClient` class that then handles all +interactions with Riak. All authentication-related information that +needs to be used by the client object can be passed to the object upon +instantiation by creating a `SecurityCreds` object. + +If you are using Riak Security, _all_ connecting clients should have +access to the same Certificate Authority (CA) used on the server side, +regardless of which [security source]({{<baseurl>}}riak/kv/2.2.6/using/security/managing-sources/) you +choose. All clients should also provide a username. The example below +sets up a client object (we'll simply call it `client`) that connects to +Riak on `localhost` and on port 8087 without any security credentials: + +```python +from riak import RiakClient + +client = RiakClient(host='127.0.0.1', pb_port=8087) +``` + +To provide security credentials, we'll create an object called `creds` +and specify `riakuser` as the username. We'll also point the client to a +CA stored at `/ssl_dir/cacertfile.pem`. + +```python +creds = SecurityCreds(username='riakuser', + cacert_file='/ssl_dir/cacertfile.pem') +``` + +Now we can specify those credentials when we create our `client` object. + +```python +client = RiakClient(host='127.0.0.1', pb_port=8087, credentials=creds) +``` + +This client object is not currently set up to use any of the +available security sources with the exception of trust-based auth, +provided that the +[CIDR](http://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing) from +which the client is connecting has been specified as trusted. More on +specifying trusted CIDRs can be found in [Trust-based +Authentication]({{<baseurl>}}riak/kv/2.2.6/using/security/managing-sources/#Trust-based-Authentication). + +**Note**: The examples in the following sections specify certs on the +basis of their filepaths, e.g. `/ssl_dir/cacertfile.pem`. In addition to +specifying certs by location, you can also provide OpenSSL objects +instead. You can find out how to do so in [Using OpenSSL Objects](#using-openssl-objects) below. + +## Password-based Authentication + +To enable our client to use password-based auth, we can use most of the +information from the above, with the exception that we'll also specify a +password for the client in the `creds` object from above. We'll use the +password `rosebud` here and in the rest of the examples. + +```python +creds = SecurityCreds(username='riakuser', + cacert_file='/ssl_dir/cacertfile.pem', + password='rosebud') +``` + +## PAM-based Authentication + +If you have specified that a specific client be authenticated using +[PAM]({{<baseurl>}}riak/kv/2.2.6/using/security/managing-sources/#pam-based-authentication), you will +need to provide a CA as well as the username and password that you +specified when creating the user in Riak. For more, see our +documentation on [User Management]({{<baseurl>}}riak/kv/2.2.6/using/security/basics/#user-management). + +## Certificate-based Authentication + +Using certificated-based authentication requires us to specify the +location of a general CA (as with all security sources), a username, a +CA-generated cert, and a private key. We'll assume that all certs are +stored in `/ssl_dir`, as in the previous examples. + +```python +creds = SecurityCreds(username='riakuser', + cacert_file='/ssl_dir/cacertfile.pem', + cert_file='/ssl_dir/cert.pem', + pkey_file='/ssl_dir/key.pem') +``` + +## Specifying a Certificate Revocation List + +If you are using a CA-generated Certificate Revocation List (CRL), you +can specify its filepath using the `crl_file` parameter. + +```python +creds = SecurityCreds(username='riakuser', + # Using the cert information from above + crl_file='/ssl_dir/revocation.crl') +``` + +## Specifying Ciphers + +To specify a list of preferred [security ciphers]({{<baseurl>}}riak/kv/2.2.6/using/security/basics/#security-ciphers), you can pass in a colon-delimited +string to the `ciphers` parameter: + +```python +creds = SecurityCreds(username='riakuser', + # Using the cert information from above + ciphers='X-CIPHER-1:X-CIPHER-2:X-CIPHER-3:ETC') +``` + +## Using OpenSSL Objects + +Whenever you specify certs, you have the option of either passing in +file paths as strings (as in the examples above) or properly created +OpenSSL objects, e.g. objects created using the +[pyOpenSSL](https://pyopenssl.readthedocs.org/en/latest/) library. If +you generate OpenSSL objects this way, you should note that they must +be specified differently when creating a `SecurityCreds` object. The +table below lists the appropriate parameter names for each method, as +well as the pyOpenSSL class to which each cert must belong if you create +OpenSSL objects. + +Cert | File path | OpenSSL object | Class +:----|:----------|:---------------|:----- +Certificate Authority (CA) | `cacert_file` | `cacert` | `OpenSSL.crypto.X509` +Private key | `key_file` | `key` | `OpenSSL.crypto.PKey` +CA-generated cert | `cert` | `cert_file` | `OpenSSL.crypto.X509` +CRL | `crl` | `crl_file` | `OpenSSL.crypto.CRL` + +If you specify filepaths, the appropriate certs will be loaded and +converted into the appropriate OpenSSL object. The functions used for +this are `OpenSSL.crypto.load_privatekey()` for the private key and +`OpenSSL.crypto.load_certificate` for the cert and CA cert. diff --git a/content/riak/kv/2.2.6/developing/usage/security/ruby.md b/content/riak/kv/2.2.6/developing/usage/security/ruby.md new file mode 100644 index 0000000000..2f3de3d16e --- /dev/null +++ b/content/riak/kv/2.2.6/developing/usage/security/ruby.md @@ -0,0 +1,158 @@ +--- +title_supertext: "Client Security:" +title: "Ruby" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Ruby" + identifier: "usage_security_ruby" + weight: 101 + parent: "usage_security" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/advanced/client-security/ruby + - /riak-docs/riak/kv/2.2.6/dev/advanced/client-security/ruby +--- + +This tutorial shows you how to set up a Riak Ruby client to authenticate +itself when connecting to Riak. + +If you are using [trust-]({{<baseurl>}}riak/kv/2.2.6/using/security/managing-sources/) or [PAM]({{<baseurl>}}riak/kv/2.2.6/using/security/managing-sources/#pam-based-authentication)-based authentication, you +can use the security setup described in the [Ruby Client Basics](#ruby-client-basics) section. +[Password]({{<baseurl>}}riak/kv/2.2.6/using/security/managing-sources/#password-based-authentication)-based authentication is covered +in a [later section](#password-based-authentication), while [certificate]({{<baseurl>}}riak/kv/2.2.6/using/security/managing-sources/#certificate-based-authentication)-based authentication +is covered [further down](#certificate-based-authentication). + +{{% note title="Note on certificate generation" %}} +This tutorial does not cover certificate generation. It assumes that all +necessary certificates have already been created and are stored in a directory +called `/ssl_dir`. This directory name is used only for example purposes. +{{% /note %}} + +## Ruby Client Basics + +When connecting to Riak using a Ruby-based client, you must instantiate +an object from the `Riak::Client` class that then handles interactions +with Riak (you may have more than one client object active in an +application if you wish). All authentication-related information that +needs to be used can be passed to the object upon instantiation in an +`authentication` hash. + +If you are using Riak Security, _all_ connecting clients should have +access to the same Certificate Authority (CA) used on the server side, +regardless of which [security source]({{<baseurl>}}riak/kv/2.2.6/using/security/managing-sources/) you choose. All clients should also provide a username. The example below sets up a client object (we'll simply call it `client`) that connects +to Riak on `localhost` and on port 8087, specifies `riakuser` as a +username, and points the client to a CA located at +`/ssl_dir/cacertfile.pem`. + +```ruby +require 'riak' + +client = Riak::Client.new( + host: '127.0.0.1', + pb_port: 8087, + authentication: { + ca_file: '/ssl_dir/cacertfile.pem', + user: 'riakuser' + } +) +``` + +This client object is currently not set up to use any of the available +security sources, except trust-based auth, provided that the CIDR from +which the client is connecting has been specified as trusted. More on +this in [Trust-based Authentication]({{<baseurl>}}riak/kv/2.2.6/using/security/managing-sources/#trust-based-authentication). + +## Password-based Authentication + +To enable our client to use password-based auth, we can use most of the +information from the example above, with the exception that we will +specify a password for the client in the `authentication` hash. We'll +use the password `rosebud` here and in the rest of the examples. + +```ruby +client = Riak::Client.new( + # Using the host and pb_port from above + authentication: { + ca_file: '/ssl_dir/cacertfile.pem', + user: 'riakuser', + password: 'rosebud' + } +) +``` + +## PAM-based Authentication + +If you have specified that a specific client be authenticated using +[PAM]({{<baseurl>}}riak/kv/2.2.6/using/security/managing-sources/#pam-based-authentication), you will +need to provide a CA as well as the username and password that you +specified when creating the user in Riak. For more, see our +documentation on [User Management]({{<baseurl>}}riak/kv/2.2.6/using/security/basics#user-management). + + +## Certificate-based Authentication + +Using certificate-based authentication requires us to specify the +location of a CA (as with all security sources), a username, a +client-specific CA, a CA-generated cert, and a private key. We'll assume +that all certs are stored in `/ssl_dir`, as in the previous examples. + +```ruby +client = Riak::Client.new( + # Using the host and pb_port from above + authentication: { + ca_file: '/path/to/cacertfile.pem', + user: 'riakuser', + client_ca: '/path/to/client_cert.pem', + cert: '/path/to/cert.pem', + key: '/path/to/key.pem' + } +) +``` + +The `client_ca` must be specified if you intend to use a CA that is +different from the CA used by Riak, e.g. if you are integrating with +an existing single sign-on (SSO) system. If the client and server CA are +the same, you don't need to specify `client_ca`. The client cert and +key, however, must always be specified. + +The `client_ca`, `cert`, and `key` fields are all flexible in their +usage. You can use a string specifying a filename (as in the example +above), or you can pass in an appropriate OpenSSL object, e.g. an SSL +object created using the +[OpenSSL](http://ruby-doc.org/stdlib-2.0/libdoc/openssl/rdoc/OpenSSL.html) +gem. If you use specify filenames, those files will be loaded and +converted into the appropriate OpenSSL object. + +## Specifying a Certificate Revocation List + +If you create certificates specifying a CA-signed Certificate Revocation +List (CRL), those certs will be checked against the CRLs specified. You +can specify the location of the list in the `authentication` hash: + +```ruby +client = Riak::Client.new( + # Using the host and pb_port from above + authentication: { + ca_file: '/ssl_dir/cacertfile.pem', + user: 'riakuser', + # Using the cert paths from above + crl_file: '/ssl_dir/revocation.crl' + } +) +``` + +CRL checking can sometimes be a slow process. To disable it, you can set +`crl` to `false` in the `authentication` hash when instantiating your +client object. + +## Online Certificate Status Protocol + +If you create certificates with a specified Online Certificate Status +Protocol +([OCSP](http://en.wikipedia.org/wiki/Online_Certificate_Status_Protocol)), +the OCSP endpoint will automatically be checked. If that endpoint is not +available or if checking is running slowly, you can disable OCSP +checking by setting `ocsp` to `false` in the `authentication` hash. diff --git a/content/riak/kv/2.2.6/developing/usage/updating-objects.md b/content/riak/kv/2.2.6/developing/usage/updating-objects.md new file mode 100644 index 0000000000..3011423145 --- /dev/null +++ b/content/riak/kv/2.2.6/developing/usage/updating-objects.md @@ -0,0 +1,774 @@ +--- +title: "Updating Objects" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Updating Objects" + identifier: "usage_updating_objects" + weight: 102 + parent: "developing_usage" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/using/updates + - /riak-docs/riak/kv/2.2.6/dev/using/updates +--- + +[glossary vnode]: {{<baseurl>}}riak/kv/2.2.6/learn/glossary/#vnode + +## Using Causal Context + +If an object already exists under a certain key and you want to write a +new object to that key, Riak needs to know what to do, especially if +multiple writes are happening at the same time. Which of the objects +being written should be deemed correct? These kinds of scenarios can +arise quite frequently in distributed, [eventually consistent]({{<baseurl>}}riak/kv/2.2.6/learn/concepts/eventual-consistency) systems. + +Riak decides which object to choose in case of conflict using [causal context]({{<baseurl>}}riak/kv/2.2.6/learn/concepts/causal-context). These objects track the causal history of objects. +They are attached to _all_ Riak objects as metadata, and they are not +readable by humans. They may sound complex---and they are fairly complex +behind the scenes---but using them in your application is very simple. + +Using causal context in an update would involve the following steps; + +1. Fetch the object +2. Modify the object's value (without modifying the fetched [context object]({{<baseurl>}}riak/kv/2.2.6/learn/concepts/causal-context) +3. Write the new object to Riak + +Step 2 is the most important here. All of Basho's official Riak clients +enable you to modify an object's value without modifying its [causal context]({{<baseurl>}}riak/kv/2.2.6/learn/concepts/causal-context). Although a more detailed tutorial on context objects and +object updates can be found in [Conflict Resolution]({{<baseurl>}}riak/kv/2.2.6/developing/usage/conflict-resolution), we'll walk you +through a basic example here. + +Let's say that the current NBA champion is the Washington Generals. +We've stored that data in Riak under the key `champion` in the bucket +`nba`, which bears the bucket type `sports`. The value of the object is +a simple text snippet that says `Washington Generals`. + +But one day the Harlem Globetrotters enter the league and dethrone the +hapless Generals (forever, as it turns out). Because we want our Riak +database to reflect this new development in the league, we want to make +a new write to the `champion` key. Let's read the object stored there +and modify the value. + +```java +Location currentChampion = new Location(new Namespace("sports", "nba"), "champion"); +FetchValue fetch = new FetchValue.Builder(currentChampion) + .build(); +FetchValue.Response response = client.execute(fetch); +RiakObject obj = response.getValue(RiakObject.class); +obj.setValue(BinaryValue.create("Harlem Globetrotters")) +``` + +```ruby +bucket = client.bucket_type('sports').bucket('nba') +obj = bucket.get('champion') +obj.raw_data = 'Harlem Globetrotters' +obj.store +``` + +```php +$location = new \Basho\Riak\Location('champion', new \Basho\Riak\Bucket('nba', 'sports')); +$object = (new \Basho\Riak\Command\Builder\FetchObject($riak)) + ->withLocation($location) + ->build() + ->execute() + ->getObject(); + +$object->setData('Harlem Globetrotters'); + +(new \Basho\Riak\Command\Builder\StoreObject($riak)) + ->withLocation($location) + ->withObject($object) + ->build() + ->execute(); +``` + +```python +bucket = client.bucket_type('sports').bucket('nba') +obj = bucket.get('champion') +obj.data = 'Harlem Globetrotters' +``` + +```csharp +var id = new RiakObjectId("sports", "nba", "champion"); +var obj = new RiakObject(id, "Washington Generals", + RiakConstants.ContentTypes.TextPlain); +var rslt = client.Put(obj); + +rslt = client.Get(id); +obj = rslt.Value; +obj.SetObject("Harlem Globetrotters", + RiakConstants.ContentTypes.TextPlain); +rslt = client.Put(obj); +``` + +```javascript +var riakObj = new Riak.Commands.KV.RiakObject(); +riakObj.setContentType('text/plain'); +riakObj.setValue('Washington Generals'); + +var options = { + bucketType: 'sports', bucket: 'nba', key: 'champion', + value: riakObj +}; +client.storeValue(options, function (err, rslt) { + if (err) { + throw new Error(err); + } + delete options.value; + client.fetchValue(options, function (err, rslt) { + if (err) { + throw new Error(err); + } + var fetchedObj = rslt.values.shift(); + fetchedObj.setValue('Harlem Globetrotters'); + options.value = fetchedObj; + options.returnBody = true; + client.storeValue(options, function (err, rslt) { + if (err) { + throw new Error(err); + } + var updatedObj = rslt.values.shift(); + logger.info("champion: %s", updatedObj.value.toString('utf8')); + }); + }); +}); +``` + +```erlang +%% In the Erlang client, you cannot view a context objectdirectly, but it +%% will be included in the output when you fetch an object: + +{ok, Obj} = riakc_pb_socket:get(Pid, + {<<"sports">>, <<"nba">>}, + <<"champion">>), +UpdatedObj = riakc_obj:update_value(Obj, <<"Harlem Globetrotters">>), +{ok, NewestObj} = riakc_pb_socket:put(Pid, UpdatedObj, [return_body]). +``` + +```golang +obj := &riak.Object{ + ContentType: "text/plain", + Charset: "utf-8", + ContentEncoding: "utf-8", + Value: []byte("Washington Generals"), +} + +cmd, err := riak.NewStoreValueCommandBuilder(). + WithBucketType("sports"). + WithBucket("nba"). + WithKey("champion"). + WithContent(obj). + WithReturnBody(true). + Build() + +if err != nil { + fmt.Println(err.Error()) + return +} + +if err := cluster.Execute(cmd); err != nil { + fmt.Println(err.Error()) + return +} + +svc := cmd.(*riak.StoreValueCommand) +rsp := svc.Response +obj = rsp.Values[0] +obj.Value = []byte("Harlem Globetrotters") + +cmd, err = riak.NewStoreValueCommandBuilder(). + WithBucketType("sports"). + WithBucket("nba"). + WithKey("champion"). + WithContent(obj). + WithReturnBody(true). + Build() + +if err != nil { + fmt.Println(err.Error()) + return +} + +if err := cluster.Execute(cmd); err != nil { + fmt.Println(err.Error()) + return +} + +svc = cmd.(*riak.StoreValueCommand) +rsp = svc.Response +obj = rsp.Values[0] +fmt.Printf("champion: %v", string(obj.Value)) +``` + +```curl +# When using curl, the context object is attached to the X-Riak-Vclock header + +curl -i http://localhost:8098/types/sports/buckets/nba/keys/champion + +# In the resulting output, the header will look something like this: + +X-Riak-Vclock: a85hYGBgzGDKBVIcWu/1S4OVPaIymBIZ81gZbskuOMOXBQA= + +# When performing a write to the same key, that same header needs to +# accompany the write for Riak to be able to use the context object +``` + +In the samples above, we didn't need to actually interact with the +context object, as retaining and passing along the context object was +accomplished automatically by the client. If, however, you do need +access to an object's context, the clients enable you to fetch it from +the object: + +```java +// Using the RiakObject obj from above: + +Vclock vClock = obj.getVclock(); +System.out.println(vClock.asString()); + +// The context object will look something like this: +// a85hYGBgzGDKBVIcWu/1S4OVPaIymBIZ81gZbskuOMOXBQA= +``` + +```ruby +# Using the RObject obj from above: + +obj.vclock + +# The context object will look something like this: +# a85hYGBgzGDKBVIcWu/1S4OVPaIymBIZ81gZbskuOMOXBQA= +``` + +```php +# Using the RObject obj from above: + +echo $object->getVclock(); // a85hYGBgzGDKBVIcWu/1S4OVPaIymBIZ81gZbskuOMOXBQA= +``` + +```python +# Using the RiakObject obj from above: + +obj.vclock + +# The context object will look something like this: +# a85hYGBgzGDKBVIcWu/1S4OVPaIymBIZ81gZbskuOMOXBQA= +``` + +```csharp +// Using the RiakObject obj from above: +var vclock = result.Value.VectorClock; +Console.WriteLine(Convert.ToBase64String(vclock)); + +// The output will look something like this: +// a85hYGBgzGDKBVIcWu/1S4OVPaIymBIZ81gZbskuOMOXBQA= +``` + +```javascript +// Using the RiakObject fetchedObj from above: +var fetchedObj = rslt.values.shift(); +logger.info("vclock: %s", fetchedObj.getVClock().toString('base64')); + +// The output will look something like this: +// vclock: a85hYGBgymDKBVIcR4M2cov1HeHKYEpkymNlsE2cfo4PKjXXjuOU+FHdWqAUM1CqECSVBQA= +``` + +```erlang +%% Using the Obj object from above: + +riakc_obj:vclock(Obj). + +%% The context object will look something like this in the Erlang shell: +%% <<107,206,97,96,96,96,204,96,2.2.6,82,28,202,156,255,126, +%% 6,175,157,255,57,131,41,145,49,143,149,225,240,...>> +``` + +```golang +svc := cmd.(*riak.StoreValueCommand) +rsp := svc.Response +fmt.Println(rsp.VClock) + +// Output: +// X3hNXFq3ythUqvvrG9eJEGbUyLS +``` + +## The Object Update Cycle + +If you decide that your application requires mutable data in Riak, we +recommend that you: + +* avoid high-frequency object updates to the same key (i.e. multiple + updates per second for long periods of time), as this will degrade + Riak performance; and that you +* follow a read-modify-write cycle when performing updates. + +That cycle looks something like this: + +1. **Read** the object from Riak. This step is important for updates +because this enables you to fetch the object's [causal context]({{<baseurl>}}riak/kv/2.2.6/learn/concepts/causal-context), which +is the information that Riak uses to make decisions about which object +values are most recent (this is especially useful for objects that are +frequently updated). This context object needs to be passed back to Riak +when you update the object. This step is handled for you by Basho's +client libraries as long as you perform a read prior to an update. In +addition, if you have chosen to allow Riak to generate +[siblings]({{<baseurl>}}riak/kv/2.2.6/developing/usage/conflict-resolution/#siblings) \(which we recommend), you +should **resolve sibling conflicts** upon read if they exist. For more +on this, please see our documentation on [conflict resolution]({{<baseurl>}}riak/kv/2.2.6/developing/usage/conflict-resolution), along +with examples from our official client libraries: + * [Java]({{<baseurl>}}riak/kv/2.2.6/developing/usage/conflict-resolution/java) + * [Ruby]({{<baseurl>}}riak/kv/2.2.6/developing/usage/conflict-resolution/ruby) + * [Python]({{<baseurl>}}riak/kv/2.2.6/developing/usage/conflict-resolution/python) + * [C#]({{<baseurl>}}riak/kv/2.2.6/developing/usage/conflict-resolution/csharp) + * [Go]({{<baseurl>}}riak/kv/2.2.6/developing/usage/conflict-resolution/golang) +2. **Modify the object** on the application side. +3. **Write** the new, modified object to Riak. Because you read the +object first, Riak will receive the object's causal context metadata. +Remember that this happens automatically. + +In general, you should read an object before modifying it. Think of it +as performing a `GET` prior to any `PUT` when interacting with a REST +API. + +> **Note on strong consistency** +> +> If you are using Riak's [strong consistency]({{<baseurl>}}riak/kv/2.2.6/developing/app-guide/strong-consistency/) feature, it is not only desirable but also necessary to use the read/modify/write cycle explained in the section above. If you attempt to update an object without fetching the object first, your update operation will necessarily fail. More information can be found in the +[strong consistency documentation]({{<baseurl>}}riak/kv/2.2.6/developing/app-guide/strong-consistency/#strongly-consistent-writes). + +### Updating Deleted Objects + +You should use the read-modify-write cycle explained above at all times, +_even if you're updating deleted objects_. The reasons for that can be +found in our documentation on [tombstones]({{<baseurl>}}riak/kv/2.2.6/using/reference/object-deletion/#tombstones). + +There are some modifications that you may need to make if you are +updating objects that may have been deleted previously. If you are using +the Java client, an explanation and examples are given in the +[Java-specific section below](#java-client-example). If +you are using the Python or Erlang clients, causal context for deleted +objects will be handled automatically. If you are using the Ruby client, +you will need to explicitly set the `deletedvclock` parameter to `true` +when reading an object, like so: + +```ruby +bucket = client.bucket('fruits') +obj = bucket.get('banana', deletedvclock: true) +``` + +## Example Update + +In this section, we'll provide an update example for Basho's official Ruby, +Python, .NET, Node.js, Erlang and Go clients. Because updates with the official +Java client functions somewhat differently, those examples can be found in the +[section below](#java-client-example). + +For our example, imagine that you are storing information about NFL head +coaches in the bucket `coaches`, which will bear the bucket type +`siblings`, which sets `allow_mult` to `true`. The key for each object +is the name of the team, e.g. `giants`, `broncos`, etc. Each object will +consist of the name of the coach in plain text. Here's an example of +creating and storing such an object: + +```ruby +bucket = client.bucket('coaches') +obj = bucket.get_or_new('seahawks', type: 'siblings') +obj.content_type = 'text/plain' +obj.raw_data = 'Pete Carroll' +obj.store +``` + +```php +$location = new \Basho\Riak\Location('seahawks', new \Basho\Riak\Bucket('coaches', 'siblings')); +$response = (new \Basho\Riak\Command\Builder\FetchObject($riak)) + ->atLocation($location) + ->build() + ->execute(); + +if ($response->isSuccess()) { + $object = $response->getObject(); + $object->setData('Pete Carroll'); +} else { + $object = new \Basho\Riak\Object('Pete Carroll', 'text/plain'); +} + +(new \Basho\Riak\Command\Builder\StoreObject($riak)) + ->withObject($object) + ->atLocation($location) + ->build() + ->execute(); +``` + +```python +bucket = client.bucket_type('siblings').bucket('coaches') +obj = RiakObject(client, bucket, 'seahawks') +obj.content_type = 'text/plain' +obj.data = 'Pete Carroll' +obj.store() +``` + +```csharp +var id = new RiakObjectId("siblings", "coaches", "seahawks"); +var obj = new RiakObject(id, "Pete Carroll", + RiakConstants.ContentTypes.TextPlain); +var rslt = client.Put(obj); +``` + +```javascript +var riakObj = new Riak.Commands.KV.RiakObject(); +riakObj.setContentType('text/plain'); +riakObj.setBucketType('siblings'); +riakObj.setBucket('coaches'); +riakObj.setKey('seahawks'); +riakObj.setValue('Pete Carroll'); +client.storeValue({ value: riakObj }, function (err, rslt) { + if (err) { + throw new Error(err); + } else { + logger.info('Stored Pete Carroll'); + } +}); +``` + +```erlang +Obj = riakc_obj:new({<<"siblings">>, <<"coaches">>}, + <<"seahawks">>, + <<"Pete Carroll">>, + <<"text/plain">>). +riakc_pb_socket:put(Pid, Obj). +``` + +```golang +obj := &riak.Object{ + ContentType: "text/plain", + Charset: "utf-8", + ContentEncoding: "utf-8", + Value: []byte("Pete Carroll"), +} + +cmd, err := riak.NewStoreValueCommandBuilder(). + WithBucketType("siblings"). + WithBucket("coaches"). + WithKey("seahawks"). + WithContent(obj). + Build() + +if err != nil { + fmt.Println(err.Error()) + return +} + +if err := cluster.Execute(cmd); err != nil { + fmt.Println(err.Error()) + return +} + +fmt.Println("Stored Pete Carroll") +``` + +Every once in a while, though, head coaches change in the NFL, which +means that our data would need to be updated. Below is an example +function for updating such objects: + +```ruby +def update_coach(team, new_coach) + bucket = client.bucket('coaches') + # The read phase + obj = bucket.get_or_new(team, type: 'siblings') + # The modify phase + obj.data = new_coach + # The write phase + obj.store +end + +# Example usage +update_coach('packers', 'Vince Lombardi') +``` + +```php +function update_coach($team, $coach) { + $location = new \Basho\Riak\Location('seahawks', new \Basho\Riak\Bucket('coaches', 'siblings')); + $response = (new \Basho\Riak\Command\Builder\FetchObject($riak)) + ->atLocation($location) + ->build() + ->execute(); + + if ($response->isSuccess()) { + $object = $response->getObject(); + $object->setData('Pete Carroll'); + } else { + $object = new \Basho\Riak\Object('Pete Carroll', 'text/plain'); + } + + $response = (new \Basho\Riak\Command\Builder\StoreObject($riak)) + ->withObject($object) + ->atLocation($location) + ->build() + ->execute(); + + return $response->isSuccess(); +} + +echo update_coach('packers', 'Vince Lombardi'); // true +``` + +```python +def update_coach(team, new_coach): + bucket = client.bucket_type('siblings').bucket('coaches') + # The read phase + obj = bucket.get(team) + # The modify phase + obj.data = new_coach + # The write phase + obj.store() + +# Example usage +update_coach('packers', 'Vince Lombardi') +``` + +```csharp +private void UpdateCoach(string team, string newCoach) +{ + var id = new RiakObjectId("siblings", "coaches", team); + var getResult = client.Get(id); + + RiakObject obj = getResult.Value; + obj.SetObject<string>(newCoach, RiakConstants.ContentTypes.TextPlain); + client.Put(obj); +} +``` + +```javascript +function update_coach(team, newCoach) { + client.fetchValue({ + bucketType: 'siblings', bucket: 'coaches', key: team + }, function (err, rslt) { + if (err) { + throw new Error(err); + } + + var riakObj = rslt.values.shift(); + riakObj.setValue(newCoach); + client.storeValue({ value: riakObj }, function (err, rslt) { + if (err) { + throw new Error(err); + } + }); + }); +} +``` + +```erlang +update_coach(team, new_coach) -> + {ok, Obj} = riakc_pb_socket:get(Pid, + {<<"siblings">>, <<"coaches">>}, + <<team>>), + ModifiedObj = riakc_obj:update_value(Obj, <<new_coach>>), + riakc_pb_socket:put(Pid, ModifiedObj). + +%% Example usage +update_coach('packers', 'Vince Lombardi') +``` + +```golang +func updateCoach(cluster *riak.Cluster, team, newCoach string) error { + var cmd riak.Command + var err error + + cmd, err = riak.NewFetchValueCommandBuilder(). + WithBucketType("siblings"). + WithBucket("coaches"). + WithKey(team). + Build() + + if err != nil { + return err + } + + if err := cluster.Execute(cmd); err != nil { + return err + } + + fvc := cmd.(*riak.FetchValueCommand) + obj := fvc.Response.Values[0] + obj.Value = []byte(newCoach) + + cmd, err = riak.NewStoreValueCommandBuilder(). + WithBucketType("siblings"). + WithBucket("coaches"). + WithKey(team). + WithContent(obj). + Build() + + if err != nil { + return err + } + + if err := cluster.Execute(cmd); err != nil { + return err + } + + return nil +} +``` + +In the example above, you can see the three steps in action: first, the +object is read, which automatically fetches the object's causal context; +then the object is modified, i.e. the object's value is set to the name +of the new coach; and finally the object is written back to Riak. + +## Object Update Anti-patterns + +The most important thing to bear in mind when updating objects is this: +you should always read an object prior to updating it _unless_ you are +certain that no object is stored there. If you are storing [sensor data]({{<baseurl>}}riak/kv/2.2.6/developing/data-modeling/#sensor-data) in Riak and using timestamps as keys, for example, then you can be sure that keys are not repeated. In that case, making writes to Riak without first reading the object is fine. If +you're not certain, however, then we recommend always reading the object +first. + +## Java Client Example + +As with the other official clients, object updates using the Java client +will automatically fetch the object's causal context metadata, modify +the object, and then write the modified value back to Riak. You can +update object values by creating your own `UpdateValue` operations that +extend the abstract class `Update<T>`. An `UpdateValue` operation must +have an `apply` method that returns a new `T`. In our case, the data +class that we're dealing with is `User`. First, let's create a very +basic `User` class: + +```java +public class User { + public String username; + public List<String> hobbies; + + public User(String username, List<String> hobbies) { + this.name = username; + this.hobbies = hobbies; + } +} +``` + +In the example below, we'll create an update value operation called +`UpdateUserName`: + +```java +import com.basho.riak.client.api.commands.kv.UpdateValue.Update; + +public class UpdateUserName extends Update<User> { + @Override + public User apply(User original) { + // update logic goes here + } +} +``` + +In the example above, we didn't specify any actual update logic. Let's +change that by creating an `UpdateValue` operation that changes a `User` +object's `name` parameter: + +```java +public class UpdateUserName extends Update<User> { + private String newUsername; + + public UpdateUserName(String newUsername) { + this.newUsername = newUsername; + } + + @Override + public User apply(User original) { + original.username = newUsername; + return original; + } +} +``` + +Now, let's put our `UpdateUserName` operation into effect. In the +example below, we'll change a `User` object's `username` from whatever +it currently is to `cliffhuxtable1986`: + +```java +import com.basho.riak.client.api.commands.kv.FetchValue; + +Location location = new Location(...); +UpdateValue updateOp = new UpdateValue.Builder(location) + .withFetchOption(FetchValue.Option.DELETED_VCLOCK, true) + .withUpdate(new UpdateUserName("cliffhuxtable1986")) + .build(); +client.execute(updateOp); +``` + +You may notice that a fetch option was added to our `UpdateValue` +operation: `FetchValue.Option.DELETED_VCLOCK` was set to `true`. +Remember from the section above that you should always read an object +before modifying and writing it, _even if the object has been deleted_. +Setting this option to `true` ensures that the causal context is fetched +from Riak if the object has been deleted. We recommend always setting +this option to `true` when constructing `UpdateValue` operations. + +### Clobber Updates + +If you'd like to update an object by simply replacing it with an +entirely new value of the same type (unlike in the section above, where +only one property of the object was updated), the Java client provides +you with a "clobber" update that you can use to replace the existing +object with a new object of the same type rather than changing one or +more properties of the object. Imagine that there is a `User` object +stored in the bucket `users` in the key `cliffhuxtable1986`, as in the +example above, and we simply want to replace the object with a brand new +object: + +```java +Location location = new Location(new Namespace("users"), "cliffhuxtable1986"); +User brandNewUser = new User(/* new user info */); +UpdateValue updateOp = new UpdateValue.Builder(Location) + // As before, we set this option to true + .withFetchOption(FetchValue.Option.DELETED_VCLOCK, true) + .withUpdate(Update.clobberUpdate(brandNewUser)) + .build(); +client.execute(updateOp); +``` + +### No-operation Updates in Java + +The Java client also enables you to construct **no-operation updates** +that don't actually modify the object and simply write the original +value back to Riak. What is the use of that, given that it isn't +changing the value of the object at all? No-operation updates can be +useful because they can help Riak resolve [sibling conflicts]({{<baseurl>}}riak/kv/2.2.6/developing/usage/conflict-resolution#siblings). If you have an object---or many objects, for that +matter---with siblings, a no-operation update will fetch the object _and +its causal context_ and write the object back to Riak with the same, +fetched context. This has the effect of telling Riak that you deem this +value to be most current. Riak can then use this information in internal +sibling resolution operations. + +Below is an example: + +```java +Location loc = new Location(...); +UpdateValue updateOp = new UpdateValue.Builder(loc) + .withUpdate(Update.noopUpdate()) + .build(); +client.execute(updateOp); +``` + +The example above would update the object without fetching it. You +could, however, use a no-operation update to _read_ an object as well if +you set `return_body` to `true` in your request: + +```java +// Using the Location object "loc" from above: +UpdateValue updateOp = new UpdateValue.Builder(loc) + .withFetchOption(Option.RETURN_BODY, true) + .withUpdate(Update.noopUpdate()) + .build(); +UpdateValue.Response response = client.execute(updateOp); +RiakObject object = response.getValue(RiakObject.class); + +// Or to continue the User example from above: +User user = response.getValue(User.class); +``` + +In general, you should use no-operation updates only on keys that you +suspect may have accumulated siblings or on keys that are frequently +updated (and thus bear the possibility of accumulating siblings). +Otherwise, you're better off performing normal reads. diff --git a/content/riak/kv/2.2.6/downloads.md b/content/riak/kv/2.2.6/downloads.md new file mode 100644 index 0000000000..359e1990ac --- /dev/null +++ b/content/riak/kv/2.2.6/downloads.md @@ -0,0 +1,22 @@ +--- +title: "Download for Riak KV 2.2.6" +description: "Download some stuff!" +menu: + riak_kv-2.2.6: + name: "Download Riak KV" + identifier: "download_riak_kv" + weight: 101 + pre: download-alt +project: "riak_kv" +project_version: "2.2.6" +toc: false +layout: downloads +listed_projects: + - project: "riak_kv" + version: "2.2.6" + title: "Riak KV" + install_instructions_set: "setup/installing" +aliases: + - /riak-docs/riak/2.2.6/downloads + - /riak-docs/riak/kv/2.2.6/downloads +--- diff --git a/content/riak/kv/2.2.6/index.md b/content/riak/kv/2.2.6/index.md new file mode 100644 index 0000000000..9d23cf8b3e --- /dev/null +++ b/content/riak/kv/2.2.6/index.md @@ -0,0 +1,73 @@ +--- +title: "Riak KV" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Riak KV" + identifier: "index" + weight: 100 + pre: riak +toc: false +aliases: + - /riak-docs/riak/2.2.6/ +--- + +[aboutenterprise]: https://www.tiot.jp/en/about-us/contact-us/ +[config index]: {{<baseurl>}}riak/kv/2.2.6/configuring +[downloads]: {{<baseurl>}}riak/kv/2.2.6/downloads/ +[install index]: {{<baseurl>}}riak/kv/2.2.6/setup/installing/ +[plan index]: {{<baseurl>}}riak/kv/2.2.6/setup/planning +[perf open files]: {{<baseurl>}}riak/kv/2.2.6/using/performance/open-files-limit +[install debian & ubuntu]: {{<baseurl>}}riak/kv/2.2.6/setup/installing/debian-ubuntu +[usage search]: {{<baseurl>}}riak/kv/2.2.6/developing/usage/search +[getting started]: {{<baseurl>}}riak/kv/2.2.6/developing/getting-started +[dev client libraries]: {{<baseurl>}}riak/kv/2.2.6/developing/client-libraries + + + +Riak KV is a distributed NoSQL database designed to deliver maximum data availability by distributing data across multiple servers. As long as your Riak KV client can reach one Riak server, it should be able to write data. + +Riak KV 2.2.6 is the first Open Source only release that includes the features of the former [Riak KV Enterprise][aboutenterprise] product such as multi-datacenter cluster replication, which ensures low-latency and robust business continuity. + +## Supported Operating Systems + +- Amazon Linux 2016.09 (AWS) +- Amazon Linux 2 (AWS) +- CentOS 6 +- CentOS 7 +- Debian 7.0 ("Wheezy") +- Debian 8.0 ("Jessie") +- Debian 9.0 ("Stretch") +- Red Hat Enterprise Linux 6 +- Red Hat Enterprise Linux 7 +- Ubuntu 12.04 ("Precise Pangolin") +- Ubuntu 14.04 ("Trusty Tahr") +- Ubuntu 16.04 ("Xenial Xerus") +- Ubuntu 17.10 ("Artful Ardvark") +- Ubuntu 18.04 ("Bionic Beaver") +- FreeBSD 10.4 +- FreeBSD 11.1 +- Mac OSX 10.8+ (development only) + +## Getting Started + +Are you brand new to Riak KV? Start by [downloading][downloads] Riak KV, and then follow the below pages to get started: + +1. [Install Riak KV][install index] +2. [Plan your Riak KV setup][plan index] +3. [Configure Riak KV for your needs][config index] + +{{% note title="Developing with Riak KV" %}} +If you are looking to integrate Riak KV with your existing tools, check out the [Developing with Riak KV]({{<baseurl>}}riak/kv/2.2.6/developing) docs. They provide instructions and examples for languages such as: Java, Ruby, Python, Go, Haskell, NodeJS, Erlang, and more. +{{% /note %}} + +## Popular Docs + +1. [Open Files Limit][perf open files] +2. [Installing on Debian-Ubuntu][install debian & ubuntu] +3. [Developing with Riak KV: Searching][usage search] +4. [Developing with Riak KV: Getting Started][getting started] +5. [Developing with Riak KV: Client Libraries][dev client libraries] + diff --git a/content/riak/kv/2.2.6/learn.md b/content/riak/kv/2.2.6/learn.md new file mode 100644 index 0000000000..861f9dd460 --- /dev/null +++ b/content/riak/kv/2.2.6/learn.md @@ -0,0 +1,47 @@ +--- +title: "Learn About Riak KV" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Learning" + identifier: "learn" + weight: 400 + pre: beaker +toc: true +--- + +[learn why riak]: ./why-riak-kv/ +[learn use cases]: ./use-cases/ +[learn new nosql]: ./new-to-nosql/ +[glossary]: ./glossary/ +[concepts]: ./concepts/ + +## In This Section + +#### [Why Riak KV?][learn why riak] + +An overview of Riak KV and when to use it. + +[Learn More >>][learn why riak] + +#### [Use Cases][learn use cases] + +Details use cases and applications in which Riak KV excels. + +[Learn More >>][learn use cases] + + + +#### [Glossary][glossary] + +A list of terms relating to Riak used throughout the documentation. + +[Learn More >>][glossary] + +#### [Concepts][concepts] + +Provides definitions for, insight into, and high level information about the various parts of Riak KV + +[Learn More >>][concepts] diff --git a/content/riak/kv/2.2.6/learn/concepts.md b/content/riak/kv/2.2.6/learn/concepts.md new file mode 100644 index 0000000000..797edee7a5 --- /dev/null +++ b/content/riak/kv/2.2.6/learn/concepts.md @@ -0,0 +1,44 @@ +--- +title: "Concepts" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Concepts" + identifier: "learn_concepts" + weight: 104 + parent: "learn" +toc: true +--- + +[concept aae]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/active-anti-entropy +[concept buckets]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/buckets +[concept cap neg]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/capability-negotiation +[concept causal context]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/causal-context +[concept clusters]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/clusters +[concept crdts]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/crdts +[concept eventual consistency]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/eventual-consistency +[concept keys objects]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/keys-and-objects +[concept replication]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/replication +[concept strong consistency]: {{<baseurl>}}riak/kv/2.2.6/using/reference/strong-consistency +[concept vnodes]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/vnodes +[config index]: {{<baseurl>}}riak/kv/2.2.6/configuring +[plan index]: {{<baseurl>}}riak/kv/2.2.6/setup/planning +[use index]: {{<baseurl>}}riak/kv/2.2.6/using/ + + +Riak KV has many great features, functions, and guiding principles that inform how the product works. This section provides definitions for, insight into, and high level information about the various parts of Riak KV you will encounter as you [plan][plan index], [configure][config index], and [use][use index] Riak. + +Learn more about: + +* [Active Anti-Entropy (AAE)][concept aae] +* [Buckets][concept buckets] +* [Capability Negotiation][concept cap neg] +* [Causal Context][concept causal context] +* [Clusters][concept clusters] +* [Convergent Replicated Data Types (CRDTs)][concept crdts] +* [Eventual Consistency][concept eventual consistency] +* [Keys and Objects][concept keys objects] +* [Replication][concept replication] +* [Virtual Nodes (vnodes)][concept vnodes] diff --git a/content/riak/kv/2.2.6/learn/concepts/active-anti-entropy.md b/content/riak/kv/2.2.6/learn/concepts/active-anti-entropy.md new file mode 100644 index 0000000000..e80416dde1 --- /dev/null +++ b/content/riak/kv/2.2.6/learn/concepts/active-anti-entropy.md @@ -0,0 +1,107 @@ +--- +title: "Active Anti-Entropy" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Active Anti-Entropy" + identifier: "learn_concepts_aae" + weight: 100 + parent: "learn_concepts" +toc: true +aliases: + - /riak-docs/riak/2.2.6/theory/concepts/aae + - /riak-docs/riak/kv/2.2.6/theory/concepts/aae +--- + +[cluster ops v3 mdc]: {{<baseurl>}}riak/kv/2.2.6/using/cluster-operations/v3-multi-datacenter +[cluster ops aae]: {{<baseurl>}}riak/kv/2.2.6/using/cluster-operations/active-anti-entropy +[concept clusters]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/clusters +[concept eventual consistency]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/eventual-consistency +[config aae]: {{<baseurl>}}riak/kv/2.2.6/configuring/reference/#active-anti-entropy +[glossary read rep]: {{<baseurl>}}riak/kv/2.2.6/learn/glossary/#read-repair +[glossary vnode]: {{<baseurl>}}riak/kv/2.2.6/learn/glossary/#vnode +[Merkle tree]: http://en.wikipedia.org/wiki/Merkle_tree +[usage search]: {{<baseurl>}}riak/kv/2.2.6/developing/usage/search + + +In a [clustered][concept clusters], [eventually consistent][concept eventual consistency] system like Riak, conflicts between object replicas stored +on different nodes are an expected byproduct of node failure, concurrent +client updates, physical data loss and corruption, and other events that +distributed systems are built to handle. These conflicts occur when +objects are either + +* **missing**, as when one node holds a replica of the object and + another node does not, or +* **divergent**, as when the values of an existing object differ across + nodes. + +Riak KV offers two means of resolving object conflicts: read repair and +active anti-entropy (AAE). Both of these conflict resolution mechanisms +apply both to normal key/value data in Riak as well as to +[search indexes][usage search] + + +## Read Repair vs. Active Anti-Entropy + +In versions of Riak prior to 1.3, replica conflicts were healed via +[read repair][glossary read rep] which is a _passive_ +anti-entropy mechanism that heals object conflicts only when a read +request reaches Riak from a client. Under read repair, if the +[vnode][glossary vnode] coordinating the read request determines +that different nodes hold divergent values for the object, the repair +process will be set in motion. + +One advantage of using read repair alone is that it doesn't require any +kind of background process to take effect, which can cut down on CPU +resource usage. The drawback of the read repair-only approach, however, +is that the healing process only can only ever reach those objects that +are read by clients. Any conflicts in objects that are not read by +clients will go undetected. + +The _active_ anti-entropy (AAE) subsystem was added to Riak in +versions 1.3 and later to enable conflict resolution to run as a +continuous background process, in contrast with read repair, which does +not run continuously. AAE is most useful in clusters containing so- +called "cold data" that may not be read for long periods of time, even +months or years, and is thus not reachable by read repair. + +Although AAE is enabled by default, it can be turned off if necessary. +See our documentation on [managing active anti-entropy][cluster ops aae] +for information on how to enable and disable AAE, as well as on configuring +and monitoring AAE. + +## Active Anti-Entropy and Hash Tree Exchange + +In order to compare object values between replicas without using more +resources than necessary, Riak relies on [Merkle +tree] hash exchanges between +nodes. + +Using this type of exchange enables Riak to compare a balanced tree of +Riak object hashes. Any difference at a higher level in the hierarchy +means that at least one value has changed at a lower level. AAE +recursively compares the tree, level by level, until it pinpoints exact +values with a difference between nodes. The result is that AAE is able +to run repair operations efficiently regardless of how many objects are +stored in a cluster, since it need only repair specific objects instead +of all objects. + +In contrast with related systems, Riak uses persistent, on-disk hash +trees instead of in-memory hash trees. The advantages of this approach +are twofold: + +* Riak can run AAE operations with a minimal impact on memory usage +* Riak nodes can be restarted without needing to rebuild hash trees + +In addition, hash trees are updated in real time as new writes come in, +which reduces the time that it takes to detect and repair missing or +divergent replicas. + +As an additional fallback measure, Riak periodically clears and +regenerates all hash trees from on-disk key/value data, which enables +Riak to detect silent data corruption to on-disk data arising from disk +failure, faulty hardware, and other sources. The default time period for +this regeneration is one week, but this can be adjusted in each node's +[configuration file][config aae]. diff --git a/content/riak/kv/2.2.6/learn/concepts/buckets.md b/content/riak/kv/2.2.6/learn/concepts/buckets.md new file mode 100644 index 0000000000..b8dbd2e7de --- /dev/null +++ b/content/riak/kv/2.2.6/learn/concepts/buckets.md @@ -0,0 +1,213 @@ +--- +title: "Buckets" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Buckets" + identifier: "learn_concepts_buckets" + weight: 101 + parent: "learn_concepts" +toc: true +aliases: + - /riak-docs/riak/2.2.6/theory/concepts/Buckets + - /riak-docs/riak/kv/2.2.6/theory/concepts/Buckets + - /riak-docs/riak/2.2.6/theory/concepts/buckets + - /riak-docs/riak/kv/2.2.6/theory/concepts/buckets +--- + +[apps cluster metadata]: {{<baseurl>}}riak/kv/2.2.6/developing/app-guide/cluster-metadata +[cluster ops bucket types]: {{<baseurl>}}riak/kv/2.2.6/using/cluster-operations/bucket-types +[cluster ops strong consistency]: {{<baseurl>}}riak/kv/2.2.6/using/cluster-operations/strong-consistency +[concept causal context]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/causal-context +[concept causal context sib]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/causal-context/#siblings +[concept replication]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/replication +[concept strong consistency]: {{<baseurl>}}riak/kv/2.2.6/using/reference/strong-consistency +[config basic]: {{<baseurl>}}riak/kv/2.2.6/configuring/basic +[dev api http]: {{<baseurl>}}riak/kv/2.2.6/developing/api/http +[dev data types]: {{<baseurl>}}riak/kv/2.2.6/developing/data-types +[glossary ring]: {{<baseurl>}}riak/kv/2.2.6/learn/glossary/#ring +[plan backend leveldb]: {{<baseurl>}}riak/kv/2.2.6/setup/planning/backend/leveldb +[plan backend bitcask]: {{<baseurl>}}riak/kv/2.2.6/setup/planning/backend/bitcask +[plan backend memory]: {{<baseurl>}}riak/kv/2.2.6/setup/planning/backend/memory +[plan backend multi]: {{<baseurl>}}riak/kv/2.2.6/setup/planning/backend/multi +[usage bucket types]: {{<baseurl>}}riak/kv/2.2.6/developing/usage/bucket-types +[usage commit hooks]: {{<baseurl>}}riak/kv/2.2.6/developing/usage/commit-hooks +[usage conflict resolution]: {{<baseurl>}}riak/kv/2.2.6/developing/usage/conflict-resolution +[usage replication]: {{<baseurl>}}riak/kv/2.2.6/developing/usage/replication + + +Buckets are used to define a virtual keyspace for storing Riak objects. +They enable you to define non-default configurations over that keyspace +concerning [replication properties][concept replication] and [other +parameters][config basic]. + +In certain respects, buckets can be compared to tables in relational +databases or folders in filesystems, respectively. From the standpoint +of performance, buckets with default configurations are essentially +"free," while non-default configurations, defined [using bucket +types][cluster ops bucket types], will be gossiped around [the ring][glossary read rep] using Riak's [cluster metadata][apps cluster metadata] subsystem. + +## Configuration + +Bucket configurations are defined [using bucket types][cluster ops bucket types], which enables +you to create and modify sets of configurations and apply them to as +many buckets as you wish. With bucket types, you can configure the +following bucket-level parameters, overriding the default values if you +wish. + +#### allow_mult + +Determines whether sibling values can be created. See [siblings][concept causal context sib]. The default can be `true` or `false` depending on +the context. See the documentation on [`allow_mult`][usage bucket types] for more +information. + +#### n_val + +Specifies the number of copies of each object to be stored in the +cluster. See the documentation on [replication properties][usage replication]. Default: +`3`. + +#### last_write_wins + +Indicates if an object's timestamp will be used to decide the canonical +write in the case of a conflict. See the documentation on [vector +clocks][concept causal context] and on [conflict resolution][usage conflict resolution] for more information. Default: +`false`. + +#### r, pr, w, dw, pw, rw, notfound_ok, basic_quorum + +See the documentation on [replication properties][usage replication] for more information +on all of these properties. + +#### precommit + +A list of Erlang functions to be executed before writing an object. See +our documentation on [pre-commit hooks][usage commit hooks] for more information. Default: no pre-commit +hooks, i.e. an empty list. + +#### postcommit + +A list of Erlang functions to be executed after writing an object. See +our documentation on [pre-commit hooks][usage commit hooks] for more information. Default: no post-commit +hooks, i.e. an empty list. + +#### old_vclock, young_vclock, small_vclock, big_vclock + +These settings enable you to manage [vector clock pruning][concept causal context]. + +#### backend + +If you are using the [Multi][plan backend multi] backend, this property enables you to +determine which of Riak's available backends---[Bitcask][plan backend bitcask], [LevelDB][plan backend leveldb], or [Memory][plan backend memory]---will be used in buckets of this type. If you are using +LevelDB, Bitcask, or the Memory backend at a cluster-wide level, _all_ +buckets of all types will use the assigned backend. + +#### consistent + +If you are using Riak's experimental [strong consistency][concept strong consistency] feature for buckets +bearing a type, this setting must be set to `true`. The default is +`false`. More information can be found in our documentation on [using +strong consistency][cluster ops strong consistency]. + +#### datatype + +If you are using [Riak data types][dev data types], this setting +determines which data type will be used in +buckets of this bucket type. Possible values: `counter`, `set`, or +`map`. + +#### dvv_enabled + +Whether [dotted version vectors][concept causal context] +will be used instead of traditional vector clocks for [conflict resolution][usage conflict resolution]. Default: `false`. + +#### chash_keyfun, linkfun + +These settings involve features that have been deprecated. You will not +need to adjust these values. + +## Fetching Bucket Properties + +If you'd like to see how a particular bucket has been configured, you +can do so using our official client libraries or through Riak's [HTTP +API][dev api http]. The following would fetch the properties for the bucket +`animals` if that bucket had a default configuration, i.e. the `default` +bucket type: + +```java +Namespace animalsBucket = new Namespace("animals"); +FetchBucketProperties fetchProps = + new FetchBucketProperties.Builder(animalsBucket).build(); +FetchBucketProperties.Response response = client.execute(fetchProps); +BucketProperties props = response.getProperties(); +``` + +```ruby +bucket = client.bucket('animals') +bucket.properties +``` + +```php +$bucketProperties = (new \Basho\Riak\Command\Builder\FetchBucketProperties($riak)) + ->buildBucket('animals') + ->build() + ->execute() + ->getBucket() + ->getProperties(); +``` + +```python +bucket = client.bucket('animals') +bucket.get_properties() +``` + +```erlang +{ok, Props} = riakc_pb_socket:get_bucket(Pid, <<"animals">>). +``` + +```curl +# Assuming that Riak is running on "localhost" and port 8087: + +curl http://localhost:8087/types/default/buckets/animals/props +``` + +If the bucket `animals` had a different type that you had created and +activated, e.g. `my_custom_type`, you could fetch the bucket properties +like so: + +```java +Namespace customTypedBucket = new Namespace("my_custom_type", "animals"); +FetchBucketProperties fetchProps = + new FetchBucketProperties.Builder(customTypedBucket).build(); +FetchBucketProperties.Response response = client.execute(fetchProps); +BucketProperties props = response.getProperties(); +``` + +```ruby +bucket = client.bucket_type('my_custom_type').bucket('animals') +bucket.properties +``` + +```php +$bucketProperties = (new \Basho\Riak\Command\Builder\FetchBucketProperties($riak)) + ->buildBucket('animals', 'my_custom_type') + ->build() + ->execute() + ->getBucket() + ->getProperties(); +``` + +```python +bucket = client.bucket_type('my_custom_type').bucket('animals') +bucket.get_properties() +``` + +```erlang +{ok, Props} = riakc_pb_socket:get_bucket(Pid, {<<"my_custom_type">>, <<"animals">>}). +``` + +```curl +curl http://localhost:8087/types/my_custom_type/buckets/animals/props +``` diff --git a/content/riak/kv/2.2.6/learn/concepts/capability-negotiation.md b/content/riak/kv/2.2.6/learn/concepts/capability-negotiation.md new file mode 100644 index 0000000000..b4c33b154e --- /dev/null +++ b/content/riak/kv/2.2.6/learn/concepts/capability-negotiation.md @@ -0,0 +1,32 @@ +--- +title: "Capability Negotiation" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Capability Negotiation" + identifier: "learn_concepts_cap_negot" + weight: 102 + parent: "learn_concepts" +toc: true +aliases: + - /riak-docs/riak/2.2.6/theory/concepts/capability-negotiation + - /riak-docs/riak/kv/2.2.6/theory/concepts/capability-negotiation +--- + + +[glossary vnode]: {{<baseurl>}}riak/kv/2.2.6/learn/glossary/#vnode +[upgrade cluster]: {{<baseurl>}}riak/kv/2.2.6/setup/upgrading/cluster +[usage mapreduce]: {{<baseurl>}}riak/kv/2.2.6/developing/usage/mapreduce + + +In early versions of Riak KV, [rolling upgrades][upgrade cluster] from an older version to a newer involved (a) disabling all new features associated with the newer version, and then (b) re-enabling those features once all nodes in the cluster were upgraded. + +Rolling upgrades no longer require you to disable and then re-enable features due to the *capability negotiation* subsystem that automatically manages the addition of new features. Using this subsystem, nodes negotiate with each other to automatically determine which versions are supported on which nodes, which allows clusters to maintain normal operations even when divergent versions of Riak KV are present in the cluster. + +{{% note title="Note on Mixed Versions" %}} +The capability negotiation subsystem is used to manage mixed versions of Riak KV within a cluster ONLY during rolling upgrades. We strongly recommend not running mixed versions during normal operations. +{{% /note %}} + + diff --git a/content/riak/kv/2.2.6/learn/concepts/causal-context.md b/content/riak/kv/2.2.6/learn/concepts/causal-context.md new file mode 100644 index 0000000000..973e5b235d --- /dev/null +++ b/content/riak/kv/2.2.6/learn/concepts/causal-context.md @@ -0,0 +1,285 @@ +--- +title: "Causal Context" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Causal Context" + identifier: "learn_concepts_causal_context" + weight: 103 + parent: "learn_concepts" +toc: true +aliases: + - /riak-docs/riak/2.2.6/theory/concepts/context + - /riak-docs/riak/kv/2.2.6/theory/concepts/context +--- + + +[concept aae]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/active-anti-entropy +[concept clusters]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/clusters +[concept eventual consistency]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/eventual-consistency +[CRM]: http://en.wikipedia.org/wiki/Customer_relationship_management +[dev api http]: {{<baseurl>}}riak/kv/2.2.6/developing/api/http +[dev key value]: {{<baseurl>}}riak/kv/2.2.6/developing/key-value-modeling +[glossary read rep]: {{<baseurl>}}riak/kv/2.2.6/learn/glossary/#read-repair +[perf latency reduc]: {{<baseurl>}}riak/kv/2.2.6/using/performance/latency-reduction +[usage bucket types]: {{<baseurl>}}riak/kv/2.2.6/developing/usage/bucket-types +[usage conflict resolution]: {{<baseurl>}}riak/kv/2.2.6/developing/usage/conflict-resolution +[usage protocol buffers]: {{<baseurl>}}riak/kv/2.2.6/developing/api/protocol-buffers +[usage updating objects]: {{<baseurl>}}riak/kv/2.2.6/developing/usage/updating-objects +[Vector Clocks on Wikipedia]: http://en.wikipedia.org/wiki/Vector_clock +[Why Vector Clocks are Easy]: http://basho.com/posts/technical/why-vector-clocks-are-easy/ +[Why Vector Clocks are Hard]: http://basho.com/posts/technical/why-vector-clocks-are-hard/ +[work of Leslie Lamport]: http://portal.acm.org/citation.cfm?id=359563 +[Evaluating Dotted Version Vectors in Riak]: http://asc.di.fct.unl.pt/~nmp/pubs/inforum-2011-2.pdf +[Improving Logical Clocks in Riak with Dotted Version Vectors: A Case Study]: http://paginas.fe.up.pt/~prodei/dsie12/papers/paper_19.pdf +[Dotted Version Vector Sets]: https://github.com/ricardobcl/Dotted-Version-Vectors +[A History of Time in Riak]: https://www.youtube.com/watch?v=3SWSw3mKApM + + +Because Riak is an [eventually consistent][concept eventual consistency], +[clustered][concept clusters] database, [conflicts][usage conflict resolution] between +object replicas stored on different nodes are inevitable, particularly +when multiple clients update an object simultaneously. + +## The Problem of Conflicting Values + +To illustrate this problem, imagine that you're building a +[CRM] +application and storing customer information in Riak. Now imagine that +information about a particular user is being stored in the [key][dev key value] `mariejohnston` in the [bucket][usage bucket types] `customers`. +What happens if Marie has two browser windows open and changes her phone +number to 555-1337 in one window and saves it, and then also changes it +to 555-1212 in another window and saves it? + +This means that two different values are sent into Riak. So what +happens at that point? There are several possible outcomes: + +1. Riak is able to discern that one object is more causally recent than the other (in this case 555-1212) and chooses to store that value as the "correct" value. +2. The two operations hit the database at roughly the same time, i.e. two **concurrent +updates** have been completed, and Riak is unable to determine which +value "wins." In this scenario, one of three things can happen: + + a. The object is a CRDT, so Riak is able to resolve conflicting values by type-specific rules + + b. Riak creates sibling values, aka **siblings**, for the object + + c. Riak resolves the values on the basis of timestamps + +In the case of outcome 1 above, Riak uses **causal context** metadata to +make that decision. This metadata is attached to every object in Riak. +Causal context comes in two forms in Riak: **vector clocks** and +**dotted version vectors**. More information in both can be found in the +sections below. + +In the case of outcome 2, the choice between **a**, **b** and **c** is determined by settings. If you set the `allow_mult` parameter to `true` for a [bucket type]({{<baseurl>}}riak/kv/2.2.6/developing/usage/bucket-types), all non-CRDT writes to that bucket type will create siblings in the case of concurrent writes (and occasionally under other +scenarios, e.g. healed network partitions). + +If, however, `allow_mult` is set to `false`, then Riak will not generate +siblings, instead relying on simple timestamp resolution to decide which value +"wins." In general, we recommend _always_ setting `allow_mult` to +`true`. A more complete discussion can be found in our documentation on +[conflict resolution][usage conflict resolution]. + +## Vector Clocks + +In versions of Riak prior to 1.4, Riak used vector clocks as the sole +means of tracking the history of object updates. In Riak versions 2.0 +and later, we recommend using [dotted version vectors](#dotted-version-vectors) instead, for reasons that are explained +in that section. + +Like dotted version vectors, vector clocks are a means of tracking +events in distributed systems. Unlike normal clocks, vector clocks have +no sense of chronological time, i.e. they don't care if something +happened at 6 pm today or back in 1972. They care only about sequences +of events. More specifically, they keep track of who---i.e. which actor +in the system---has modified an object and how many times they've done +so. + +In a distributed system like Riak, multiple replicas of each object are +active in the cluster all the time. Because it's inevitable that objects +will have conflicting values due to events like concurrent updates and +healed network partitions, Riak needs a mechanism to keep track of which +replica of an object is more current than another. In versions of Riak +prior to 2.0, vector clocks were the means employed by Riak to do +precisely that. + +A number of important aspects of the relationship between object +replicas can be determined using vector clocks: + + * Whether one object is a direct descendant of the other + * Whether the objects are direct descendants of a common parent + * Whether the objects are unrelated in recent heritage + +Behind the scenes, Riak uses vector clocks as an essential element of +its [active anti-entropy][concept aae] subsystem and of its automatic read +repair capabilities. + + +Vector clocks are non-human-readable metadata attached to all Riak +objects. They look something like this: + +``` +a85hYGBgzGDKBVIcR4M2cgczH7HPYEpkzGNlsP/VfYYvCwA= +``` + +While vector clocks quite often resolve object conflicts without +trouble, there are times when they can't, i.e. when it's unclear which +value of an object is most current. When that happens, Riak, if +configured to do so, will create **siblings**. + +## More Information on Vector Clocks + +Additional information on vector clocks: + +* [Conflict Resolution][usage conflict resolution] in Riak KV +* [Vector Clocks on Wikipedia] +* [Why Vector Clocks are Easy] +* [Why Vector Clocks are Hard] +* The vector clocks used in Riak are based on the [work of Leslie Lamport]. + +## Siblings + +It is possible, though not recommendable, to [configure Riak][usage conflict resolution] to ensure that only one copy of an object ever exists in a +specific location. This will ensure that _at most_ one value is returned +when a read is performed on a bucket type/bucket/key location (and no +value if Riak returns `not found`). + +It's also possible, however, to configure Riak to store multiple objects +in a single key if necessary, i.e. for an object to have different +values on different nodes. Objects stored this way have what are called +sibling values. You can instruct Riak to allow for sibling creation by +setting the the `allow_mult` bucket property to `true` for a specific +bucket, preferably [using bucket types][usage bucket types]. + +From the standpoint of application development, the difficulty with +siblings is that they _by definition_ conflict with one another. When an +application attempts to read an object that has siblings, multiple +replicas will be stored in the location where the application is +looking. This means that the application will need to develop a +strategy for [conflict resolution][usage conflict resolution], i.e. the application will need to +decide which value is more correct depending on the use case. + +## Dotted Version Vectors + +In versions of Riak prior to 2.0, all causality-based conflict +resolution, whether on the client side or in Riak, was achieved using +[vector clocks][concept causal context]. In version 2.0, +Riak added the option of using **dotted version vectors** (DVVs) +instead. + +Like vector clocks, dotted version vectors are a mechanism for tracking +object update causality in terms of **logical time** rather than +chronological time (as with timestamps), enabling Riak to make decisions +about which objects are more current than others in cases of conflict. + +>**Note: DVVs Recommended Over Vector Clocks** +> +>If you are using Riak version 2.0 or later, we strongly recommend using +dotted version vectors instead of vector clocks, as DVVs are far better +at limiting the number of siblings produced in a cluster, which can +prevent a wide variety of potential issues. + + +## DVVs Versus Vector Clocks + +The role that DVVs play in Riak is directly analogous to that of +vector clocks, as both are used +to resolve object conflicts, whether during background operations like +[active anti-entropy][concept aae] or [read repair][glossary read rep], or +when applications engage in client-side [conflict resolution][usage conflict resolution]. The +crucial difference between them, however, lies in the way that they +handle concurrent updates. + +Vector clocks can detect concurrent updates to the same object but they +can't identify which value was associated with each update. If an object +stored in the bucket `frequent_updates` with the key `update_me` is +updated by five different clients concurrently and tagged with the same +vector clock, then five values should be created as siblings. However, +depending on the order of delivery of those updates to the different +replicas, sibling values may be duplicated, which can in turn lead to +[sibling explosion](#siblings) and thus undue +[latency][perf latency reduc]. + +DVVs, on the other hand, identify each value with the update that +created it. If five clients concurrently update the object above (in the +bucket `frequent_updates`, with the key `update_me`), each of these +updates will be marked with a _dot_ (a minimal vector clock) that indicates the specific event that introduced it. This +means that duplicate values can always be identified and removed, +reducing the likelihood of sibling explosion. Rather than being potentially unbounded, the +number of sibling values will be proportional to the number of +concurrent updates. + +In terms of performance, the difference between vector clocks and DVVs +should be minimal in most cases. Because DVVs de-duplicate updates, they +should generally be smaller than objects that use vector clocks. + +## Usage + +From an application's perspective, vector clocks and DVVs function in +exactly the same fashion. Object updates using DVVs involve the same +sequence in interacting with Riak: + +* fetch an object from Riak, +* fetch the object's metadata, which will contain an opaque context + object (e.g. `a85hYGBgzGDKBVIcWu/1S4Pjin9lMCWy5bEycN1/cYYvCwA=`) for + the vector clock or DVV attached to that version of the object, and + finally +* pass that opaque context object back to Riak when you update the + object. + +You will not need to modify your application code when switching from +vector clocks to DVVs, even if you choose to switch all Riak objects in +your cluster to DVVs. You should make sure, however, that the right +bucket types and buckets are being targeted by your application after +the `dvv_enabled` parameter has been changed. + +For compatibility's sake, DVVs contained in Riak objects' metadata are +still labeled `X-Riak-Vclock` if you're using the [HTTP API][dev api http] and +`vclock` if using the [Protocol Buffers interface][usage protocol buffers]. + +More on using vector clocks and DVVs on the application side can be +found in our documentation on [conflict resolution][usage conflict resolution]. + +>**Note on DVVs and bucket types** +> +>The choice between vector clocks and DVVs can be made at the bucket +level, [using bucket types][usage bucket types]. This enables you to employ a mixed +conflict resolution strategy in your Riak cluster, using DVVs in some +buckets and vector clocks in others if you wish. DVVs can be enabled by +setting the `dvv_enabled` bucket property to +`true` for one or more bucket types. +> +>Vector clocks remain the default if you are not using bucket types. +However, any bucket type that you create and activate will have +`dvv_enabled` set to `true`. And so if you wish to +create a bucket type that uses traditional vector clocks, you will need +to explicitly set `dvv_enabled` to `false` for +that bucket type. + + +## Sibling Explosion + +Sibling explosion occurs when an object rapidly collects siblings that +are not reconciled. This can lead to a variety of problems, including +degraded performance, especially if many objects in a cluster suffer +from siblings explosion. At the extreme, having an enormous object in a +node can cause reads of that object to crash the entire node. Other +issues include [undue latency][perf latency reduc] and +out-of-memory errors. + +To prevent sibling explosion, we recommend the following: + +1. Use [dotted version vectors](#dotted-version-vectors) +instead of vector clocks for causal +context. +2. Always update mutable objects within a read/modify/write cycle. More +information can be found in the [Object Updates][usage updating objects] doc. + +## Resources + +* [Evaluating Dotted Version Vectors in Riak] +* [Improving Logical Clocks in Riak with Dotted Version Vectors: A Case Study] +* [Dotted Version Vector Sets] +* [A History of Time in Riak] diff --git a/content/riak/kv/2.2.6/learn/concepts/clusters.md b/content/riak/kv/2.2.6/learn/concepts/clusters.md new file mode 100644 index 0000000000..ac2ce18612 --- /dev/null +++ b/content/riak/kv/2.2.6/learn/concepts/clusters.md @@ -0,0 +1,113 @@ +--- +title: "Clusters" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Clusters" + identifier: "learn_concepts_clusters" + weight: 103 + parent: "learn_concepts" +toc: true +aliases: + - /riak-docs/riak/2.2.6/theory/concepts/Clusters + - /riak-docs/riak/kv/2.2.6/theory/concepts/Clusters + - /riak-docs/riak/2.2.6/theory/concepts/clusters + - /riak-docs/riak/kv/2.2.6/theory/concepts/clusters +--- + + +[concept buckets]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/buckets +[concept keys objects]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/keys-and-objects +[concept replication]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/replication +[glossary node]: {{<baseurl>}}riak/kv/2.2.6/learn/glossary/#node +[glossary vnode]: {{<baseurl>}}riak/kv/2.2.6/learn/glossary/#vnode +[learn dynamo]: {{<baseurl>}}riak/kv/2.2.6/learn/dynamo +[usage bucket types]: {{<baseurl>}}riak/kv/2.2.6/developing/usage/bucket-types +[usage conflict resolution]: {{<baseurl>}}riak/kv/2.2.6/developing/usage/conflict-resolution +[usage replication]: {{<baseurl>}}riak/kv/2.2.6/developing/usage/replication + + +Riak's default mode of operation is to work as a cluster consisting of +multiple [nodes][glossary node], i.e. multiple well-connected data +hosts. + +Each host in the cluster runs a single instance of Riak, referred to as +a Riak node. Each Riak node manages a set of virtual nodes, or +[vnodes][glossary vnode], that are responsible for storing a +separate portion of the keys stored in the cluster. + +In contrast to some high-availability systems, Riak nodes are _not_ +clones of one another, and they do not all participate in fulfilling +every request. Instead, you can configure, at runtime or at request +time, the number of nodes on which data is to be replicated, as well as +when [replication][concept replication] occurs and which [merge strategy][usage conflict resolution] and failure model are to be followed. + +## The Ring + +Though much of this section is discussed in our annotated discussion of +the Amazon [Dynamo paper][learn dynamo], it nonetheless provides a summary of +how Riak implements the distribution of data throughout a cluster. + +Any client interface to Riak interacts with objects in terms of the +[bucket][concept buckets] and [key][concept keys objects] in which a value is +stored, as well as the [bucket type][usage bucket types] that is used +to set the bucket's properties. + +Internally, Riak computes a 160-bit binary hash of each bucket/key pair +and maps this value to a position on an ordered **ring** of all such +values. This ring is divided into partitions, with each Riak vnode +responsible for one of these partitions (we say that each vnode +_claims_ that partition). + +Below is a visual representation of a Riak ring: + +![A Riak Ring]({{<baseurl>}}images/riak-ring.png) + +The nodes of a Riak cluster each attempt to run a roughly equal number +of vnodes at any given time. In the general case, this means that each +node in the cluster is responsible for 1/(number of nodes) of the ring, +or (number of partitions)/(number of nodes) vnodes. + +If two nodes define a 16-partition cluster, for example, then each node +will run 8 vnodes. Nodes attempt to claim their partitions at intervals +around the ring such that there is an even distribution amongst the +member nodes and that no node is responsible for more than one replica +of a key. + +## Intelligent Replication + +When an object is being stored in the cluster, any node may participate +as the **coordinating node** for the request. The coordinating node +consults the ring state to determine which vnode owns the partition in +which the value's key belongs, then sends the write request to that +vnode as well as to the vnodes responsible for the next N-1 partitions +in the ring (where N is a [configurable parameter][usage replication] that describes how many copies of the value to store). The +write request may also specify that at least W (=< N) of those vnodes +reply with success, and that DW (=< W) reply with success only after +durably storing the value. + +A read, or GET, request operates similarly, sending requests to the +vnode that "claims" the partition in which the key resides, as well as +to the next N-1 partitions. The request also specifies R (=< N), the +number of vnodes that must reply before a response is returned. + +Here is an illustration of this process: + +![A Riak Ring]({{<baseurl>}}images/riak-data-distribution.png) + +When N is set to 3, the value `REM` is stored in the key `artist`. That +key is assigned to 3 partitions out of 32 available partitions. When a +read request is made to Riak, the ring state will be used to determine +which partitions are responsible. From there, a variety of +[configurable parameters][usage replication] determine how Riak +will behave in case the value is not immediately found. + +## Gossiping + +The ring state is shared around the cluster by means of a "gossip +protocol." Whenever a node changes its claim on the ring, it announces, +i.e. "gossips," this change to other nodes so that the other nodes can +respond appropriately. Nodes also periodically re-announce what they +know about ring in case any nodes happened to miss previous updates. diff --git a/content/riak/kv/2.2.6/learn/concepts/crdts.md b/content/riak/kv/2.2.6/learn/concepts/crdts.md new file mode 100644 index 0000000000..7cf609960d --- /dev/null +++ b/content/riak/kv/2.2.6/learn/concepts/crdts.md @@ -0,0 +1,248 @@ +--- +title_supertext: "Concept" +title: "Data Types" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Data Types" + identifier: "learn_concepts_data_types" + weight: 104 + parent: "learn_concepts" +toc: true +aliases: + - /riak-docs/riak/2.2.6/theory/concepts/crdts + - /riak-docs/riak/kv/2.2.6/theory/concepts/crdts +--- + +[crdts pdf]: http://hal.upmc.fr/docs/00/55/55/88/PDF/techreport.pdf +[data types converg]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/crdts/#convergence +[crdts reading list]: http://christophermeiklejohn.com/crdt/2014/07/22/readings-in-crdts.html +[data types impl]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/crdts/#implementation +[concept causal context dvv]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/causal-context/#dotted-version-vectors +[concept causal context sib]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/causal-context/#siblings +[concept causal context vc]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/causal-context/#vector-clocks +[concept eventual consistency]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/eventual-consistency +[concept strong consistency]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/strong-consistency +[dev data types]: {{<baseurl>}}riak/kv/2.2.6/developing/data-types +[riak_dt]: https://github.com/basho/riak_dt +[dev data types context]: {{<baseurl>}}riak/kv/2.1.4/developing/data-types/#data-types-and-context +[glossary node]: {{<baseurl>}}riak/kv/2.2.6/learn/glossary/#node +[glossary vnode]: {{<baseurl>}}riak/kv/2.2.6/learn/glossary/#vnode +[usage conflict resolution]: {{<baseurl>}}riak/kv/2.2.6/developing/usage/conflict-resolution + +Riak Data Types are convergent replicated data types (CRDTs), inspired by the work of [Marc Shapiro, Nuno Preguiça, Carlos Baquero, and Marek Zawirski][crdts pdf]. Riak KV supports the following eventually-convergent data types, described in later sections: + +- Counters +- Flags +- HyperLogLogs +- Maps +- Registers +- Sets + +The difference between Riak Data Types and typical key/value data stored in Riak KV is that Riak Data Types are operations-based from the standpoint of Riak KV clients. + +Instead of the usual create, read, update, and delete (CRUD) operations +performed on key/value pairs, data types enable you to perform +operations such as removing a register from a map, telling a counter to +increment itself by 5, or enabling a flag that was previously disabled. + +It's important to note that Riak Data Types are operations-based from the standpoint of connecting clients. Like CRDTs, the [convergence logic][data types converg] is state-based behind the scenes. + +Riak Data Types enable applications to use CRDTs through a simple interface, without being exposed to the complex state-based logic underneath. More on Data Types and state can be found in the section on [implementation][data types impl] below. + +For more articles on CRDTs, check out this [reading list][crdts reading list]. + + +## Counters + +Counters are a bucket-level Riak data type that can be used by themselves, associated with a bucket/key pair, or used within a map. A counter’s value can only be a positive integer, negative integer, or zero. + +Counters are useful when a count is needed, for example: + +- Counting the number of people following someone on Twitter +- Counting the amount of likes on a Facebook post +- Counting the points scored by a player in a game + +If you require unique, ordered IDs counters should not be used because uniqueness cannot be guaranteed. + +### Operations + +Counters are subject to two operations: increment and decrement. + + +## Flags + +Flags are similar to Boolean values, but instead of `true` or +`false` flags are the value `enable` or `disable`. Flags can only be stored within maps; they cannot be stored in a bucket/key on their own. + +Some examples of using flags: + +- Showing if a tweet has been retweeted +- Showing if a user has signed up for a specific pricing plan + +### Operations + +Flags support only two operations: `enable` and `disable`. Flags can be +added to or removed from a map, but those operations are performed on +the map and not on the flag directly. + + +## HyperLogLogs + +HyperLogLogs (HLLs) are a data type used to count unique elements within a data set or stream. + +For example, hyperloglogs can be used for: + +- Counting the number of unique visitors to your website +- Counting the number of unique searches users performed + +### Operations + +HyperLogLogs support two operations: adding elements and retrieving the count. + + +## Maps + +Maps are the most versatile of the Riak data types because all other data types can be embedded within them, _including maps themselves_. This enables the creation of complex, custom data types from a few basic building blocks. + +Maps are best suited for complex, multi-faceted data. The following +JSON-inspired pseudocode shows how a tweet might be structured as a map: + +``` +Map tweet { + Counter: numberOfRetweets, + Register: username, + Register: tweetContent, + Flag: favorited?, + Map: userInfo +} +``` + +### Operations + +You can perform two types of operations on maps: + +1. Operations performed directly on the map itself, which includes + adding fields to and removing fields from the map (e.g. adding a flag + or removing a counter). +2. Operations performed on the Data Types nested in the map, e.g. + incrementing a counter in the map or setting a flag to `enable`. + Those operations behave just like the operations specific to that + Data Type. + + +## Registers + +Registers are essentially named binaries (like strings). Any binary +value can act as the value of a register. Like flags, registers cannot +be used on their own and must be embedded in maps. + +Some examples of using registers: + +- Storing the name `Cassius` in the register `first_name` in a map called `user14325_info` +- Storing the title of a blog post in a map called `2010-03-01_blog_post` + +### Operations + +Registers can only have the binaries stored within them changed. They can be added to and removed from maps, but those operations take place on the map in which the register is nested, and not on the register itself. + + +## Sets + +Sets are collections of unique binary values, such as strings. All of +the values in a set are unique. For example, if you attempt to add the +element `shovel` to a set that already contains `shovel`, the operation +will be ignored by Riak KV. Sets can be used either on their own or +embedded in a map. + +Some examples of using sets: + +- Storing the UUIDs of a user's friends in a social network application +- Storing items in an e-commerce shopping cart + +### Operations + +Sets are subject to four basic operations: add an element, remove an +element, add multiple elements, or remove multiple elements. + + +## Advantages and Disadvantages of Data Types + +[Conflict resolution][usage conflict resolution] in Riak KV can be difficult because it involves reasoning about concurrency, [eventual consistency][concept eventual consistency], [siblings][concept causal context sib], and other issues that many other databases don't require you to consider. + +One of the core purposes behind data types is to relieve developers +using Riak KV of the burden of producing data convergence at the +application level by absorbing a great deal of that complexity into Riak KV +itself. Riak KV manages this complexity by building eventual consistency +into the data types themselves instead of requiring clients to do so. + +You can still build applications with Riak KV that treat it as a highly +available key/value store, and you will always have this choice. What +Riak Data Types provide is additional flexibility and a broader choice +palette. + +The trade-off that data types necessarily present is that they don't +allow you to produce your own convergence logic. If your use case +demands that you be able to create your own deterministic merge +functions, then Riak Data Types might not be a good fit. + + +## Implementation + +Conflicts between replicas are inevitable in a distributed system like +Riak KV. + +For example, if a map is stored in the key `my_map`, it is always +possible that the value of `my_map` will be different in nodes A and B. + +Without using data types, that conflict must be resolved using +timestamps, [vector clocks][concept causal context vc], [dotted version vectors][concept causal context dvv], or some other means. With data types, conflicts are resolved by Riak KV itself, using a subsystem called [`riak_dt`][riak_dt]. + + +## Convergence + +The benefit of data types is that Riak KV knows how to resolve value +conflicts by applying data type-specific rules. + +Riak KV does this by remembering the history of a value and broadcasting that +history along with the current value in the form of a [context object][dev data types context] that is similar to a [vector clock][concept causal context vc] or [dotted version vectors][concept causal context dvv]. Riak KV uses the history of each data type to make deterministic judgments about which value should be deemed correct. + +### Example + +Imagine a set stored in the key `fruits`. On one [node][glossary node] the set `fruits` has two elements, `apple` and `orange`. While on another node the set has only one element, `apple`. + +What happens when the two nodes communicate and note the divergence? + +In this case Riak KV would declare the set with two elements the winner. +At that point, the node with the incorrect set would be told: "The set +`fruits` should have elements `apple` and `orange`." + +In general, convergence involves the following stages: + +1. Check for divergence. If the data types have the same value, Riak KV + does nothing. But if divergence is noted... +2. Riak KV applies data type-specific merge rules, like in the `fruits` + set example above, which will result in a "correct" value. +3. After the merge logic is applied and the correct value is determined, + the relevant [vnodes][glossary vnode] are notified and act to + correct the divergence. + +## Convergence Rules + +Convergence means that data type conflicts are weighted in a certain direction. Riak's Data Types have their own internal weights that dictate what happens in case of conflict: + +Data Type | Convergence rule +:--------|:------------ +Flags | `enable` wins over `disable` +Registers | The most chronologically recent value wins, based on timestamps +Counters | Implemented as a PN-Counter ([paper][crdts pdf]), so all increments and decrements by all actors are eventually applied. Every actor wins. +Sets | If an element is concurrently added and removed, the add will win +Maps | If a field is concurrently added or updated and removed, the add/update will win + +In a production Riak KV cluster being hit by lots and lots of concurrent +writes, value conflicts are inevitable. Riak Data Types are not perfect, particularly because they do not guarantee [strong consistency][concept strong consistency] and you cannot specify the rules yourself. But the +rules that dictate the convergence logic behind the Riak Data Types +were carefully chosen to minimize the potential downsides associated +with value conflicts. diff --git a/content/riak/kv/2.2.6/learn/concepts/eventual-consistency.md b/content/riak/kv/2.2.6/learn/concepts/eventual-consistency.md new file mode 100644 index 0000000000..7f82ee0d89 --- /dev/null +++ b/content/riak/kv/2.2.6/learn/concepts/eventual-consistency.md @@ -0,0 +1,198 @@ +--- +title: "Eventual Consistency" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Eventual Consistency" + identifier: "learn_concepts_eventual_cons" + weight: 105 + parent: "learn_concepts" +toc: true +aliases: + - /riak-docs/riak/2.2.6/theory/concepts/Eventual-Consistency + - /riak-docs/riak/kv/2.2.6/theory/concepts/Eventual-Consistency + - /riak-docs/riak/2.2.6/theory/concepts/eventual-consistency + - /riak-docs/riak/kv/2.2.6/theory/concepts/eventual-consistency +--- + + +[concept buckets]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/buckets +[concept causal context vc]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/causal-context/#vector-clocks +[concept clusters]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/clusters +[concept replication]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/replication +[glossary node]: {{<baseurl>}}riak/kv/2.2.6/learn/glossary/#node +[glossary read rep]: {{<baseurl>}}riak/kv/2.2.6/learn/glossary/#read-repair +[usage bucket types]: {{<baseurl>}}riak/kv/2.2.6/developing/usage/bucket-types +[usage conflict resolution]: {{<baseurl>}}riak/kv/2.2.6/developing/usage/conflict-resolution + + +In a distributed and fault-tolerant system like Riak, server and network +failures are expected. Riak is designed to respond to requests even when +[nodes][glossary node] are offline or the cluster is experiencing +a network partition. + +Riak handles this problem by enabling conflicting copies of data stored +in the same location, as specified by [bucket type][concept buckets], bucket, and key, to exist at the same time in the cluster. This +gives rise to the problem of **data inconsistency**. + +## Data Inconsistency + +Conflicts between replicas of an object are inevitable in +highly-available, [clustered][concept clusters] systems like Riak because there +is nothing in those systems to guarantee so-called [ACID +transactions](http://en.wikipedia.org/wiki/ACID). Because of this, these +systems need to rely on some form of conflict-resolution mechanism. + +One of the things that makes Riak's eventual consistency model powerful +is that Riak does not dictate how data resolution takes place. While +Riak does ship with a set of defaults regarding how data is +[replicated](#replication-properties-and-request-tuning) and how +[conflicts are resolved][usage conflict resolution], you can override these +defaults if you want to employ a different strategy. + +Among those strategies, you can enable Riak to resolve object conflicts +automatically, whether via internal [vector clocks][concept causal context vc], timestamps, or +special eventually consistent [Data Types]({{<baseurl>}}riak/kv/2.2.6/developing/data-types/), or you can resolve those +conflicts on the application side by employing a use case-specific logic +of your choosing. More information on this can be found in our guide to +[conflict resolution][usage conflict resolution]. + +This variety of options enables you to manage Riak's eventually +consistent behavior in accordance with your application's [data model +or models]({{<baseurl>}}riak/kv/2.2.6/developing/data-modeling/). + +## Replication Properties and Request Tuning + +In addition to providing you different means of resolving conflicts, +Riak also enables you to fine-tune **replication properties**, which +determine things like the number of nodes on which data should be stored +and the number of nodes that are required to respond to read, write, and +other requests. + +An in-depth discussion of these behaviors and how they can be +implemented on the application side can be found in our guides to +[replication properties][concept replication] and [conflict resolution][usage conflict resolution]. + +In addition to our official documentation, we also recommend checking +out the [Understanding Riak's Configurable +Behaviors](http://basho.com/understanding-riaks-configurable-behaviors-part-1/) +series from [the Basho blog](http://basho.com/blog/). + +## A Simple Example of Eventual Consistency + +Let's assume for the moment that a sports news application is storing +all of its data in Riak. One thing that the application always needs to +be able to report to users is the identity of the current manager of +Manchester United, which is stored in the key `manchester-manager` in +the bucket `premier-league-managers`. This bucket has `allow_mult` set +to `false`, which means that Riak will resolve all conflicts by itself. + +Now let's say that a node in this cluster has recently recovered from +failure and has an old copy of the key `manchester-manager` stored in +it, with the value `Alex Ferguson`. The problem is that Sir Ferguson +stepped down in 2013 and is no longer the manager. Fortunately, the +other nodes in the cluster hold the value `David Moyes`, which is +correct. + +Shortly after the recovered node comes back online, other cluster +members recognize that it is available. Then, a read request for +`manchester-manager` arrives from the application. Regardless of which +order the responses arrive to the node that is coordinating this +request, `David Moyes` will be returned as the value to the client, +because `Alex Ferguson` is recognized as an older value. + +Why is this? How does Riak make this decision? Behind the scenes, after +`David Moyes` is sent to the client, a [read repair][glossary read rep] mechanism will occur on the cluster to fix the +older value on the node that just came back online. Because Riak tags +all objects with versioning information, it can make these kinds of +decisions on its own, if you wish. + +### R=1 + +Let's say that you keep the above scenario the same, except you tweak +the request and set R to 1, perhaps because you want faster responses to +the client. In this case, it _is_ possible that the client will receive +the outdated value `Alex Ferguson` because it is only waiting for a +response from one node. + +However, the read repair mechanism will kick in and fix the value, so +the next time someone asks for the value of `manchester-manager`, `David +Moyes` will indeed be the answer. + +### R=1, sloppy quorum + +Let's take the scenario back in time to the point at which our unlucky +node originally failed. At that point, all 3 nodes had `Alex Ferguson` +as the value for `manchester-manager`. + +When a node fails, Riak's *sloppy quorum* feature kicks in and another +node takes responsibility for serving its requests. + +The first time we issue a read request after the failure, if `R` is set +to 1, we run a significant risk of receiving a `not found` response from +Riak. The node that has assumed responsibility for that data won't have +a copy of `manchester-manager` yet, and it's much faster to verify a +missing key than to pull a copy of the value from disk, so that node +will likely respond fastest. + +If `R` is left to its default value of 2, there wouldn't be a problem +because 1 of the nodes that still had a copy of `Alex Ferguson` would +also respond before the client got its result. In either case, read +repair will step in after the request has been completed and make +certain that the value is propagated to all the nodes that need it. + +### PR, PW, sloppy quorum + +Thus far, we've discussed settings that permit sloppy quorums in the +interest of allowing Riak to maintain as high a level of availability as +possible in the presence of node or network failure. + +It is possible to configure requests to ignore sloppy quorums in order +to limit the possibility of older data being returned to a client. The +tradeoff, of course, is that there is an increased risk of request +failures if failover nodes are not permitted to serve requests. + +In the scenario we've been discussing, for example, the possibility of a +node for the `manchester-manager` key having failed, but to be more +precise, we've been talking about a *primary* node, one that when the +cluster is perfectly healthy would bear responsibility for that key. + +When that node failed, using `R=2` as we've discussed or even `R=3` for +a read request would still work properly: a failover node (sloppy quorum +again) would be tasked to take responsibility for that key, and when it +receives a request for it, it would reply that it doesn't have any such +key, but the two surviving primary nodes still know who the +`manchester-manager` is. + +However, if the PR (primary read) value is specified, only the two +surviving primary nodes are considered valid sources for that data. + +So, setting PR to 2 works fine, because there are still 2 such nodes, +but a read request with PR=3 would fail because the 3rd primary node is +offline, and no failover node can take its place *as a primary*. + +The same is true of writes: W=2 or W=3 will work fine with the primary +node offline, as will PW=2 (primary write), but PW=3 will result in an +error. + +>**Note: Errors and Failures** +> +>It is important to understand the difference between an error and a +failure. +> +>The `PW=3` request in this scenario will result in an error, +but the value will still be written to the two surviving primary +nodes. +> +>By specifying `PW=3` the client indicated that 3 primary +nodes must respond for the operation to be considered successful, which +it wasn't, but there's no way to tell without performing another read +whether the operation truly failed. + + +## Further Reading + +* [Understanding Riak's Configurable Behaviors blog series](http://basho.com/understanding-riaks-configurable-behaviors-part-1/) +* Werner Vogels, et. al.: [Eventually Consistent - Revisited](http://www.allthingsdistributed.com/2008/12/eventually_consistent.html) diff --git a/content/riak/kv/2.2.6/learn/concepts/keys-and-objects.md b/content/riak/kv/2.2.6/learn/concepts/keys-and-objects.md new file mode 100644 index 0000000000..e7789fa3d6 --- /dev/null +++ b/content/riak/kv/2.2.6/learn/concepts/keys-and-objects.md @@ -0,0 +1,49 @@ +--- +title: "Keys and Objects" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Keys and Objects" + identifier: "learn_concepts_keys_objects" + weight: 106 + parent: "learn_concepts" +toc: true +aliases: + - /riak-docs/riak/2.2.6/theory/concepts/keys-and-values + - /riak-docs/riak/kv/2.2.6/theory/concepts/keys-and-values +--- + +[concept buckets]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/buckets +[concept causal context vc]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/causal-context/#vector-clocks + +In an RDBMS, data is organized by tables that are individually +identifiable entities. Within those tables exist rows of a data +organized into columns. It is possible to retrieve or update entire +tables, individual rows, or a group of columns within a set of +rows. In contrast, Riak has a simpler data model in which the Object +(explained below) is both the largest and smallest data element. When +performing any fetch or update operation in Riak, the entire Riak +Object must be retrieved or modified; there are no partial fetches or +updates. + +## Keys + +Keys in Riak are simply binary values (or strings) used to identify +Objects. From the perspective of a client interacting with Riak, +each bucket appears to represent a separate keyspace. It is important +to understand that Riak treats the bucket-key pair as a single entity +when performing fetch and store operations (see: [Buckets][concept buckets]). + +## Objects + +Objects are the only unit of data storage in Riak. Riak Objects are +essentially structs identified by bucket and key and composed of the +following parts: a bucket, key, vector clock, and a list of +metadata-value pairs. Normally, objects have only one metadata-value +pair, but when there are more than one, the object is said to have +"siblings". These siblings may occur both within a single node and +across multiple nodes, and do occur when either more than one actor +updates an object, a network partition occurs, or a stale vector clock +is submitted when updating an object (see: [Vector Clocks][concept causal context vc]). diff --git a/content/riak/kv/2.2.6/learn/concepts/replication.md b/content/riak/kv/2.2.6/learn/concepts/replication.md new file mode 100644 index 0000000000..9909a66f42 --- /dev/null +++ b/content/riak/kv/2.2.6/learn/concepts/replication.md @@ -0,0 +1,319 @@ +--- +title: "Replication" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Replication" + identifier: "learn_concepts_replication" + weight: 108 + parent: "learn_concepts" +toc: true +aliases: + - /riak-docs/riak/2.2.6/theory/concepts/Replication + - /riak-docs/riak/kv/2.2.6/theory/concepts/Replication + - /riak-docs/riak/2.2.6/theory/concepts/replication + - /riak-docs/riak/kv/2.2.6/theory/concepts/replication +--- + + +[cluster ops v3 mdc]: {{<baseurl>}}riak/kv/2.2.6/using/cluster-operations/v3-multi-datacenter +[concept aae]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/active-anti-entropy +[concept causal context vc]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/causal-context/#vector-clocks +[concept clusters]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/clusters +[concept vnodes]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/vnodes +[glossary node]: {{<baseurl>}}riak/kv/2.2.6/learn/glossary/#node +[glossary ring]: {{<baseurl>}}riak/kv/2.2.6/learn/glossary/#ring +[usage replication]: {{<baseurl>}}riak/kv/2.2.6/developing/usage/replication + + +Data replication is a core feature of Riak's basic architecture. Riak +was designed to operate as a [clustered][concept clusters] system containing +multiple Riak [nodes][glossary node], which allows data to live +on multiple machines at once in case a node in the cluster goes down. + +Replication is fundamental and automatic in Riak, providing security +that your data will still be there if a node in your Riak cluster goes +down. All data stored in Riak will be replicated to a number of nodes in +the cluster according to the N value (`n_val`) property set in a +bucket's [bucket type]({{<baseurl>}}riak/kv/2.2.6/developing/usage/bucket-types). + +>**Note: Replication across clusters** +> +>If you're interested in replication not just within a cluster but across +multiple clusters, we recommend checking out our documentation on Riak's +[Multi-Datacenter Replications]({{<baseurl>}}riak/kv/2.2.6/setup/planning/backend/multi) capabilities. + +## Selecting an N value (`n_val`) + +By default, Riak chooses an `n_val` of 3 default. This means that data +stored in any bucket will be replicated to 3 different nodes. For this +to be effective, you need at least 3 nodes in your cluster. + +The ideal value for N depends largely on your application and the shape +of your data. If your data is highly transient and can be reconstructed +easily by the application, choosing a lower N value will provide greater +performance. However, if you need high assurance that data is available +even after node failure, increasing the N value will help protect +against loss. How many nodes do you expect will fail at any one time? +Choose an N value larger than that and your data will still be +accessible when they go down. + +The N value also affects the behavior of read (GET) and write (PUT) +requests. The tunable parameters you can submit with requests are bound +by the N value. For example, if N=3, the maximum read quorum (known as +"R") you can request is also 3. If some nodes containing the data you +are requesting are down, an R value larger than the number of available +nodes with the data will cause the read to fail. + +## Setting the N value (`n_val`) + +To change the N value for a bucket, you need to create a [bucket +type]({{<baseurl>}}riak/kv/2.2.6/developing/usage/bucket-types) with `n_val` set to your desired value and +then make sure that the bucket bears that type. + +In this example, we'll set N to 2. First, we'll create the bucket type +and call it `n_val_of_2` and then activate that type: + +```bash +riak-admin bucket-type create n_val_of_2 '{"props":{"n_val":2}}' +riak-admin bucket-type activate n_val_of_2 +``` + +Now, any bucket that bears the type `n_val_of_2` will propagate objects +to 2 nodes. + +>**Note on changing the value of N** +> +>Changing the N value after a bucket has data in it is *not +recommended*. If you do change the value, especially if you +increase it, you might need to force read repair (more on that below). +Overwritten objects and newly stored objects will automatically be +replicated to the correct number of nodes. + +## Changing the N value (`n_val`) + +While raising the value of N for a bucket or object shouldn't cause +problems, it's important that you never lower N. If you do so, you can +wind up with dead, i.e. unreachable data. This can happen because +objects' preflists, i.e. lists of [vnodes][concept vnodes] responsible for the object, +can end up + +Unreachable data is a problem because it can negatively impact coverage +queries, e.g. [secondary index]({{<baseurl>}}riak/kv/2.2.6/developing/usage/secondary-indexes/) and +[MapReduce]({{<baseurl>}}riak/kv/2.2.6/developing/usage/mapreduce/) queries. Lowering an object or bucket's +`n_val` will likely mean that objects that you would expect to +be returned from those queries will no longer be returned. + +## Active Anti-Entropy + +Riak's active anti-entropy (AAE) subsystem is a continuous background +process that compares and repairs any divergent or missing object +replicas. For more information on AAE, see the following documents: + +* [Active Anti-Entropy][concept aae] +* [Managing Active Anti-Entropy][cluster ops v3 mdc] + + +## Read Repair + +Read repair occurs when a successful read occurs---i.e. when the target +number of nodes have responded, as determined by R---but not all +replicas of the object agree on the value. There are two possibilities +here for the errant nodes: + +1. The node responded with a `not found` for the object, meaning that + it doesn't have a copy. +2. The node responded with a [vector clock][concept causal context vc] that is an + ancestor of the vector clock of the successful read. + +When this situation occurs, Riak will force the errant nodes to update +the object's value based on the value of the successful read. + +### Forcing Read Repair + +When you increase the `n_val` of a bucket, you may start to see failed +read operations, especially if the R value you use is larger than the +number of replicas that originally stored the object. Forcing read +repair will solve this issue. Or if you have [active +anti-entropy][usage replication] enabled, your values will +eventually replicate as a background task. + +For each object that fails read (or the whole bucket, if you like), read +the object using an R value less than or equal to the original number of +replicas. For example, if your original `n_val` was 3 and you increased +it to 5, perform your read operations with R=3 or less. This will cause +the nodes that do not have the object(s) yet to respond with `not +found`, invoking read repair. + +## So what does N=3 really mean? + +N=3 simply means that three copies of each piece of data will be stored +in the cluster. That is, three different partitions/vnodes will receive +copies of the data. **There are no guarantees that the three replicas +will go to three separate physical nodes**; however, the built-in +functions for determining where replicas go attempts to distribute the +data evenly. + +As nodes are added and removed from the cluster, the ownership of +partitions changes and may result in an uneven distribution of the data. +On some rare occasions, Riak will also aggressively reshuffle ownership +of the partitions to achieve a more even balance. + +For cases where the number of nodes is less than the N value, data will +likely be duplicated on some nodes. For example, with N=3 and 2 nodes in +the cluster, one node will likely have one replica, and the other node +will have two replicas. + +## Understanding replication by example + +To better understand how data is replicated in Riak let's take a look at +a put request for the bucket/key pair `my_bucket`/`my_key`. Specifically +we'll focus on two parts of the request: routing an object to a set of +partitions and storing an object on a partition. + +### Routing an object to a set of partitions + + * Assume we have 3 nodes + * Assume we store 3 replicas per object (N=3) + * Assume we have 8 partitions in our [ring][glossary ring] \(ring_creation_size=8) + +**Note**: It is not recommended that you use such a small ring size. +This is for demonstration purposes only. + +With only 8 partitions our ring will look approximately as follows +(response from `riak_core_ring_manager:get_my_ring/0` truncated for +clarity): + +```erlang +(dev1@127.0.0.1)3> {ok,Ring} = riak_core_ring_manager:get_my_ring(). +[{0,'dev1@127.0.0.1'}, +{182687704666362864775460604089535377456991567872, 'dev2@127.0.0.1'}, +{365375409332725729550921208179070754913983135744, 'dev3@127.0.0.1'}, +{548063113999088594326381812268606132370974703616, 'dev1@127.0.0.1'}, +{730750818665451459101842416358141509827966271488, 'dev2@127.0.0.1'}, +{913438523331814323877303020447676887284957839360, 'dev3@127.0.0.1'}, +{1096126227998177188652763624537212264741949407232, 'dev1@127.0.0.1'}, +{1278813932664540053428224228626747642198940975104, 'dev2@127.0.0.1'}] +``` + +The node handling this request hashes the bucket/key combination: + +```erlang +(dev1@127.0.0.1)4> DocIdx = riak_core_util:chash_key({<<"my_bucket">>, <<"my_key">>}). +<<183,28,67,173,80,128,26,94,190,198,65,15,27,243,135,127,121,101,255,96>> +``` + +The DocIdx hash is a 160-bit integer: + +```erlang +(dev1@127.0.0.1)5> <<I:160/integer>> = DocIdx. +<<183,28,67,173,80,128,26,94,190,198,65,15,27,243,135,127,121,101,255,96>> +(dev1@127.0.0.1)6> I. +1045375627425331784151332358177649483819648417632 +``` + +The node looks up the hashed key in the ring, which returns a list of +_preferred_ partitions for the given key. + +```erlang +(node1@127.0.0.1)> Preflist = riak_core_ring:preflist(DocIdx, Ring). +[{1096126227998177188652763624537212264741949407232, 'dev1@127.0.0.1'}, +{1278813932664540053428224228626747642198940975104, 'dev2@127.0.0.1'}, +{0, 'dev1@127.0.0.1'}, +{182687704666362864775460604089535377456991567872, 'dev2@127.0.0.1'}, +{365375409332725729550921208179070754913983135744, 'dev3@127.0.0.1'}, +{548063113999088594326381812268606132370974703616, 'dev1@127.0.0.1'}, +{730750818665451459101842416358141509827966271488, 'dev2@127.0.0.1'}, +{913438523331814323877303020447676887284957839360, 'dev3@127.0.0.1'}] +``` + +The node chooses the first N partitions from the list. The remaining +partitions of the "preferred" list are retained as fallbacks to use if +any of the target partitions are unavailable. + +```erlang +(dev1@127.0.0.1)9> {Targets, Fallbacks} = lists:split(N, Preflist). +{[{1096126227998177188652763624537212264741949407232, 'dev1@127.0.0.1'}, +{1278813932664540053428224228626747642198940975104, 'dev2@127.0.0.1'}, +{0,'dev1@127.0.0.1'}], +[{182687704666362864775460604089535377456991567872, 'dev2@127.0.0.1'}, +{365375409332725729550921208179070754913983135744, 'dev3@127.0.0.1'}, +{548063113999088594326381812268606132370974703616, 'dev1@127.0.0.1'}, +{730750818665451459101842416358141509827966271488, 'dev2@127.0.0.1'}, +{913438523331814323877303020447676887284957839360, 'dev3@127.0.0.1'}]} +``` + +The partition information returned from the ring contains a partition +identifier and the parent node of that partition: + +```erlang +{1096126227998177188652763624537212264741949407232, 'dev1@127.0.0.1'} +``` + +The requesting node sends a message to each parent node with the object +and partition identifier (pseudocode for clarity): + +```erlang +'dev1@127.0.0.1' ! {put, Object, 1096126227998177188652763624537212264741949407232} +'dev2@127.0.0.1' ! {put, Object, 1278813932664540053428224228626747642198940975104} +'dev1@127.0.0.1' ! {put, Object, 0} +``` + +If any of the target partitions fail, the node sends the object to one +of the fallbacks. When the message is sent to the fallback node, the +message references the object and original partition identifier. For +example, if `dev2@127.0.0.1` were unavailable, the requesting node would +then try each of the fallbacks. The fallbacks in this example are: + +```erlang +{182687704666362864775460604089535377456991567872, 'dev2@127.0.0.1'} +{365375409332725729550921208179070754913983135744, 'dev3@127.0.0.1'} +{548063113999088594326381812268606132370974703616, 'dev1@127.0.0.1'} +``` + +The next available fallback node would be `dev3@127.0.0.1`. The +requesting node would send a message to the fallback node with the +object and original partition identifier: + +```erlang +'dev3@127.0.0.1' ! {put, Object, 1278813932664540053428224228626747642198940975104} +``` + +Note that the partition identifier in the message is the same that was +originally sent to `dev2@127.0.0.1` only this time it is being sent to +`dev3@127.0.0.1`. Even though `dev3@127.0.0.1` is not the parent node of +that partition, it is smart enough to hold on to the object until +`dev2@127.0.0.1` returns to the cluster. + +## Processing partition requests + +Processing requests per partition is fairly simple. Each node runs a +single process (`riak_kv_vnode_master`) that distributes requests to +individual partition processes (`riak_kv_vnode`). The +`riak_kv_vnode_master` process maintains a list of partition identifiers +and corresponding partition processes. If a process does not exist for a +given partition identifier a new process is spawned to manage that +partition. + +The `riak_kv_vnode_master` process treats all requests the same and +spawns partition processes as needed even when nodes receive requests +for partitions they do not own. When a partition's parent node is +unavailable, requests are sent to fallback nodes (handoff). The +`riak_kv_vnode_master` process on the fallback node spawns a process to +manage the partition even though the partition does not belong to the +fallback node. + +The individual partition processes perform hometests throughout the life +of the process. The hometest checks if the current node (`node/0`) +matches the parent node of the partition as defined in the ring. If the +process determines that the partition it is managing belongs on another +node (the parent node), it will attempt to contact that node. If that +parent node responds, the process will hand off any objects it has +processed for that partition and shut down. If that parent node does not +respond, the process will continue to manage that partition and check +the parent node again after a delay. The hometest is also run by +partition processes to account for changes in the ring, such as the +addition or removal of nodes to the cluster. + diff --git a/content/riak/kv/2.2.6/learn/concepts/strong-consistency.md b/content/riak/kv/2.2.6/learn/concepts/strong-consistency.md new file mode 100644 index 0000000000..2c23eee27c --- /dev/null +++ b/content/riak/kv/2.2.6/learn/concepts/strong-consistency.md @@ -0,0 +1,101 @@ +--- +title: "Strong Consistency" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Strong Consistency" + identifier: "learn_concepts_strong_consistency" + weight: 109 + parent: "learn_concepts" +toc: true +aliases: + - /riak-docs/riak/2.2.6/theory/concepts/strong-consistency + - /riak-docs/riak/kv/2.2.6/theory/concepts/strong-consistency +--- + +[usage bucket types]: {{<baseurl>}}riak/kv/2.2.6/developing/usage/bucket-types +[concept eventual consistency]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/eventual-consistency + +{{% note title="Please Note:" %}} +Riak KV's strong consistency is an experimental feature and may be removed +from the product in the future. Strong consistency is not commercially +supported or production-ready. Strong consistency is incompatible with +Multi-Datacenter Replication, Riak Search, Bitcask Expiration, LevelDB +Secondary Indexes, Riak Data Types and Commit Hooks. We do not recommend its +usage in any production environment. +{{% /note %}} + +Riak was originally designed as an [eventually consistent]({{<baseurl>}}riak/kv/2.2.6/learn/concepts/eventual-consistency) system, fundamentally geared toward providing partition +(i.e. fault) tolerance and high read and write availability. + +While this focus on high availability is a great fit for many data +storage needs, there are also many use cases for which strong data +consistency is more important than availability. Basho introduced a new +strong consistency option in version 2.0 to address these use cases. +In Riak, strong consistency is applied [using bucket types][usage bucket types], which +enables developers to apply strong consistency guarantees on a per-key +basis. + +Elsewhere in the documentation there are instructions for [enabling and using]({{<baseurl>}}riak/kv/2.2.6/developing/app-guide/strong-consistency/) strong consistency, as well as a [guide for operators]({{<baseurl>}}riak/kv/2.2.6/configuring/strong-consistency) looking to manage, +configure, and monitor strong consistency. + +## Strong vs. Eventual Consistency + +If you successfully write a value to a key in a strongly consistent +system, the next successful read of that key is guaranteed to show that +write. A client will never see out-of-date values. The drawback is that +some operations may fail if an insufficient number of object replicas +are available. More on this in the section on [trade-offs](#trade-offs). + +In an eventually consistent system, on the other hand, a read may return +an out-of-date value, particularly during system or network failures. +The advantage of this approach is that reads and writes can succeed even +when a cluster is experiencing significant service degradation. + +### Example + +Building on the example presented in the [eventual consistency][concept eventual consistency] doc, +imagine that information about who manages Manchester United is stored +in Riak, in the key `manchester-manager`. In the eventual consistency +example, the value associated with this key was originally +`David Moyes`, meaning that that was the first successful write to that +key. But then `Louis van Gaal` became Man U's manager, and a write was +executed to change the value of `manchester-manager`. + +Now imagine that this write failed on one node in a multi-node cluster. +Thus, all nodes report that the value of `manchester-manager` is `Louis +van Gaal` except for one. On the errant node, the value of the +`manchester-manager` key is still `David Moyes`. An eventually +consistent system is one in which a get request will most likely return +`Louis van Gaal` but could return the outdated value `David Moyes`. + +In a strongly consistent system, conversely, any successful read on +`manchester-manager` will return `Louis van Gaal` and never `David Moyes`. +Reads will return `Louis van Gaal` every single time until Man U gets a new +manager and someone performs a successful write to `manchester-manager` +to change its value. + +It might also be useful to imagine it a bit more abstractly. The +following causal sequence would characterize a strongly consistent +system: + +1. The value of the key `k` is set to `v` +2. All successful reads on `k` return `v` +3. The value of `k` is changed to `v2` +4. All successful reads on `k` return `v2` +5. And so forth + +At no point in time does this system return an out-of-date value. + +The following sequence could characterize an eventually consistent +system: + +1. A write is made that sets the value of the key `k` to `v` +2. Nearly all reads to `k` return `v`, but a small percentage return + `not found` +3. A write to `k` changes the value to `v2` +4. Nearly all reads to `k` now return `v2`, but a small number return + the outdated `v` (or even `not found`) because the newer value hasn't + yet been replicated to all nodes diff --git a/content/riak/kv/2.2.6/learn/concepts/vnodes.md b/content/riak/kv/2.2.6/learn/concepts/vnodes.md new file mode 100644 index 0000000000..45aa4ef381 --- /dev/null +++ b/content/riak/kv/2.2.6/learn/concepts/vnodes.md @@ -0,0 +1,156 @@ +--- +title: "Vnodes" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Vnodes" + identifier: "learn_concepts_vnodes" + weight: 109 + parent: "learn_concepts" +toc: true +aliases: + - /riak-docs/riak/2.2.6/theory/concepts/vnodes + - /riak-docs/riak/kv/2.2.6/theory/concepts/vnodes +--- + + +[concept causal context]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/causal-context +[concept clusters ring]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/clusters/#the-ring +[concept replication]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/replication +[concept strong consistency]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/strong-consistency +[glossary node]: {{<baseurl>}}riak/kv/2.2.6/learn/glossary/#node +[glossary ring]: {{<baseurl>}}riak/kv/2.2.6/learn/glossary/#ring +[plan backend]: {{<baseurl>}}riak/kv/2.2.6/setup/planning/backend +[plan cluster capacity]: {{<baseurl>}}riak/kv/2.2.6/setup/planning/cluster-capacity +[use admin riak cli]: {{<baseurl>}}riak/kv/2.2.6/using/admin/riak-cli + + +Virtual nodes, more commonly referred to as **vnodes**, are processes +that manage partitions in the Riak [ring][glossary ring]. Each data +partition in a Riak cluster has a vnode that **claims** that partition. +Vnodes perform a wide variety of operations, from K/V storage operations +to guaranteeing [strong consistency][concept strong consistency] if you choose to use that +feature. + +## The Number of Vnodes in a Cluster + +The term [node][glossary node] refers to a full instance of Riak, +be it on its own physical machine or alongside others on a single +machine, as in a development cluster on your laptop. Each Riak node +contains multiple vnodes. The number per node is the [ring +size][concept clusters ring] divided by the number of nodes in the cluster. + +This means that in some clusters different nodes will have different +numbers of data partitions (and hence a different number of vnodes), +because (ring size / number of nodes) will not produce an even integer. +If the ring size of your cluster is 64 and you are running three nodes, +two of your nodes will have 21 vnodes, while the third node holds 22 +vnodes. + +The output of the [`riak-admin member-status`][use admin riak cli] +command shows this: + +``` +================================= Membership ================================== +Status Ring Pending Node +------------------------------------------------------------------------------- +valid 34.4% -- 'dev1@127.0.0.1' +valid 32.8% -- 'dev2@127.0.0.1' +valid 32.8% -- 'dev3@127.0.0.1' +------------------------------------------------------------------------------- +Valid: 3 / Leaving:0 / Exiting:0 / Joining:0 / Down:0 +``` + +In this cluster, one node accounts for 34.4% of the ring, i.e. 22 out of +64 partitions, while the other two nodes account for 32.8%, i.e. 21 out +of 64 partitions. This is normal and expected behavior in Riak. + +We strongly recommend setting the appropriate ring size, and by +extension the number of vnodes, prior to building a cluster. A full +guide can be found in our [cluster planning][plan cluster capacity] documentation. + +## The Role of Vnodes + +Vnodes essentially watch over a designated subset of a cluster's key +space. Riak computes a 160-bit binary hash of each bucket/key pair and +maps this value to a position on an ordered [ring][concept clusters ring] +of all such values. The illustration below provides a visual +representation of the Riak ring: + +![The Riak +Ring](http://docs.basho.com/shared/2.0.2/images/riak-ring.png?1416296175) + +You can think of vnodes as managers, responsible for handling incoming +requests from other nodes/vnodes, storing objects in the appropriate +storage backend, fetching objects from backends, interpreting [causal +context][concept causal context] metadata for objects, acting as [strong consistency +ensembles][concept strong consistency] and much +more. At the system level, vnodes are Erlang processes build on top of +the [`gen_fsm`](http://www.erlang.org/doc/design_principles/fsm.html) +abstraction in Erlang, i.e. you can think of vnodes as **finite state +machines** that are constantly at work ensuring that Riak's key +goals---high availability, fault tolerance, etc.---are guaranteed for +their allotted portion of the cluster's key space. Whereas nodes are +essentially a passive container for a wide variety of Riak processes, +vnodes are the true workhorses of Riak. + +While each vnode has a main Erlang process undergirding it, vnodes may +also spawn new worker processes (i.e. new Erlang actors) to perform +asynchronous tasks on behalf of the vnode. + +If you're navigating through the file system of a Riak node, you'll +notice that each node's `/data` directory holds a variety of +subdirectories. If you're using, say, [Bitcask]({{<baseurl>}}riak/kv/2.2.6/setup/planning/backend/bitcask) as a backend, navigate +into the `/bitcask` directory (you'll also see a `/ring` directory and +several others). If you open up the `/bitcask` directory, you'll see a +wide assortment of directories with numbers as names, e.g. `0` or +`1004782375664995756265033322492444576013453623296`. These directories +each house the data from a particular partition. + +## Vnodes and Replication Properties + +In our documentation on [replication properties][concept replication], we make frequent +mention of users' ability to choose how many nodes store copies of +data, how many nodes must respond for a read request to succeed, and so +on. This is slightly misleading, as the fundamental units of replication +are not nodes but rather vnodes. + +This can be illustrated by way of a potential user error. If you store +an object and set N=5, this means that you want the object to be stored +on 5 different nodes. But imagine that your cluster only has 3 nodes. +Setting N=5 on a 3-node cluster is actually just fine. The data will be +managed by 5 vnodes, but some of that data may end up being stored more +than once on different nodes. A likely scenario is that two nodes will +store two copies of the data a piece, while the third node will store +only one. Absent such an error, however, nodes will not contain multiple +vnodes responsible for the same partition. + +## Vnode Status + +You can check the current status of all vnodes in your cluster using the +[`riak-admin vnode-status`][use admin riak cli] +command. When you run that command, you will see a series of reports on +each of the vnodes active on the local node. The output of this command +consists of a series of reports on each active vnode. The report for a +specific vnode should look something like this: + +``` +VNode: 1278813932664540053428224228626747642198940975104 +Backend: riak_kv_bitcask_backend +Status: +[{key_count, 275}, + {status,[{"./data/bitcask/1278813932664540053428224228626747642198940975104/2.bitcask.data", + 0,0,335}]}] +Status: +{vnodeid,<<"ÅR±\vi80\f">>} +``` + +The meaning of each field is given in the table below. + +Field | Description +:-----|:----------- +`VNode` | The ID of the vnode in question +`Backend` | The storage [backend][plan backend] utilized by the vnode +`Status` | The number of keys managed by the vnode and the file where the vnode stores its data. The other information can be ignored. diff --git a/content/riak/kv/2.2.6/learn/dynamo.md b/content/riak/kv/2.2.6/learn/dynamo.md new file mode 100644 index 0000000000..71f209811c --- /dev/null +++ b/content/riak/kv/2.2.6/learn/dynamo.md @@ -0,0 +1,1924 @@ +--- +title: "Dynamo: Amazon’s Highly Available Key-value Store" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Dynamo" + identifier: "learn_dynamo" + weight: 110 + parent: "learn" +toc: false +aliases: + - /riak-docs/riak/2.2.6/theory/dynamo + - /riak-docs/riak/kv/2.2.6/theory/dynamo +--- + +<div style="text-align:center;font-style:italic"> + Giuseppe DeCandia, Deniz Hastorun, Madan Jampani, Gunavardhan Kakulapati, + Avinash Lakshman, Alex Pilchin, Swaminathan Sivasubramanian, Peter Vosshall + and Werner Vogels + <br> + Amazon.com +</div> + +<br> + +> *Dynamo: Amazon's Highly Available Key-value Store* is reprinted here in its +> entirety, images and all. +> +> Throughout the paper you will find notes containing Riak KV-specifics that +> relate to a given section of the paper; anything from links to the docs, to +> code references, to explanations of why and how we did what we did. + +<!-- Random comment to add some padding between blockquotes --> + +> This paper was first released in 2007 and was popularized on the blog of +> Werner Vogels. Since then, several Dynamo-inspired databases have appeared +> (either entirely or partially) by this paper. In addition to Riak KV, +> Cassandra and Voldemort come to mind. You may also remember Dynomite (which +> predates all of these). There are probably more. +> +> Also note that this paper has little to do with Amazon's DynamoDB service. +> They have not published the inner workings of that implementation. + + +## Abstract + +Reliability at massive scale is one of the biggest challenges we face at +Amazon.com, one of the largest e-commerce operations in the world; even the +slightest outage has significant financial consequences and impacts customer +trust. The Amazon.com platform, which provides services for many web sites +worldwide, is implemented on top of an infrastructure of tens of thousands of +servers and network components located in many datacenters around the world. At +this scale, small and large components fail continuously and the way persistent +state is managed in the face of these failures drives the reliability and +scalability of the software systems. + +This paper presents the design and implementation of Dynamo, a highly available +key-value storage system that some of Amazon’s core services use to provide an +“always-on” experience. To achieve this level of availability, Dynamo sacrifices +consistency under certain failure scenarios. It makes extensive use of object +versioning and application-assisted conflict resolution in a manner that +provides a novel interface for developers to use. + +Categories and Subject Descriptors + +* D.4.2 [Operating Systems]: Storage Management; +* D.4.5 [Operating Systems]: Reliability; +* D.4.2 [Operating Systems]: Performance; + +General Terms + +Algorithms, Management, Measurement, Performance, Design, Reliability. + +## 1. Introduction + +Amazon runs a world-wide e-commerce platform that serves tens of millions +customers at peak times using tens of thousands of servers located in many data +centers around the world. There are strict operational requirements on Amazon’s +platform in terms of performance, reliability and efficiency, and to support +continuous growth the platform needs to be highly scalable. Reliability is one +of the most important requirements because even the slightest outage has +significant financial consequences and impacts customer trust. In addition, to +support continuous growth, the platform needs to be highly scalable. + +One of the lessons our organization has learned from operating Amazon’s platform +is that the reliability and scalability of a system is dependent on how its +application state is managed. Amazon uses a highly decentralized, loosely +coupled, service oriented architecture consisting of hundreds of services. In +this environment there is a particular need for storage technologies that are +always available. For example, customers should be able to view and add items to +their shopping cart even if disks are failing, network routes are flapping, or +data centers are being destroyed by tornados. Therefore, the service responsible +for managing shopping carts requires that it can always write to and read from +its data store, and that its data needs to be available across multiple data +centers. + +Dealing with failures in an infrastructure comprised of millions of components +is our standard mode of operation; there are always a small but significant +number of server and network components that are failing at any given time. As +such Amazon’s software systems need to be constructed in a manner that treats +failure handling as the normal case without impacting availability or +performance. + +To meet the reliability and scaling needs, Amazon has developed a number of +storage technologies, of which the Amazon Simple Storage Service (also available +outside of Amazon and known as Amazon S3), is probably the best known. This +paper presents the design and implementation of Dynamo, another highly available +and scalable distributed data store built for Amazon’s platform. Dynamo is used +to manage the state of services that have very high reliability requirements and +need tight control over the tradeoffs between availability, consistency, cost- +effectiveness and performance. Amazon’s platform has a very diverse set of +applications with different storage requirements. A select set of applications +requires a storage technology that is flexible enough to let application +designers configure their data store appropriately based on these tradeoffs to +achieve high availability and guaranteed performance in the most cost effective +manner. + +There are many services on Amazon’s platform that only need primary-key access +to a data store. For many services, such as those that provide best seller +lists, shopping carts, customer preferences, session management, sales rank, and +product catalog, the common pattern of using a relational database would lead to +inefficiencies and limit scale and availability. Dynamo provides a simple +primary-key only interface to meet the requirements of these applications. + +Dynamo uses a synthesis of well known techniques to achieve scalability and +availability: Data is partitioned and replicated using consistent hashing [10], +and consistency is facilitated by object versioning [12]. The consistency among +replicas during updates is maintained by a quorum-like technique and a +decentralized replica synchronization protocol. Dynamo employs a gossip based +distributed failure detection and membership protocol. Dynamo is a completely +decentralized system with minimal need for manual administration. Storage nodes +can be added and removed from Dynamo without requiring any manual partitioning +or redistribution. + +> Like Dynamo, Riak KV employs consistent hashing to partition and replicate +> data around the ring. For the consistent hashing that takes place in +> riak_core, Basho chose the SHA1 hash. See [Consistent Hashing] in our docs. +> +> Riak KV uses vector clocks for object versioning. Scroll down to section 4.4 +> to read up on this in depth. +> +> Riak KV makes use of gossiping in the same way that Dynamo does: to +> communicate ring state and node membership. See [Gossip Protocol] in our docs. +> +> And, nodes can be added and removed from your Riak cluster as needed. + +[Consistent Hashing]: {{<baseurl>}}riak/kv/2.2.6/learn/glossary/#consistent-hashing +[Gossip Protocol]: {{<baseurl>}}riak/kv/2.2.6/learn/glossary/#gossiping + +In the past year, Dynamo has been the underlying storage technology for a number +of the core services in Amazon’s e-commerce platform. It was able to scale to +extreme peak loads efficiently without any downtime during the busy holiday +shopping season. For example, the service that maintains shopping cart (Shopping +Cart Service) served tens of millions requests that resulted in well over 3 +million checkouts in a single day and the service that manages session state +handled hundreds of thousands of concurrently active sessions. + +The main contribution of this work for the research community is the evaluation +of how different techniques can be combined to provide a single highly-available +system. It demonstrates that an eventually-consistent storage system can be used +in production with demanding applications. It also provides insight into the +tuning of these techniques to meet the requirements of production systems with +very strict performance demands. + +The paper is structured as follows. Section 2 presents the background and +Section 3 presents the related work. Section 4 presents the system design and +Section 5 describes the implementation. Section 6 details the experiences and +insights gained by running Dynamo in production and Section 7 concludes the +paper. There are a number of places in this paper where additional information +may have been appropriate but where protecting Amazon’s business interests +require us to reduce some level of detail. For this reason, the intra- and +inter-datacenter latencies in section 6, the absolute request rates in section +6.2 and outage lengths and workloads in section 6.3 are provided through +aggregate measures instead of absolute details. + + +## 2. Background + +Amazon’s e-commerce platform is composed of hundreds of services that work in +concert to deliver functionality ranging from recommendations to order +fulfillment to fraud detection. Each service is exposed through a well defined +interface and is accessible over the network. These services are hosted in an +infrastructure that consists of tens of thousands of servers located across many +data centers world-wide. Some of these services are stateless (i.e., services +which aggregate responses from other services) and some are stateful (i.e., a +service that generates its response by executing business logic on its state +stored in persistent store). + +> **Brief Background on Riak KV** +> +> Basho Technologies started to develop Riak KV back in 2007 to solve an +> internal problem. We were, at the time, builing a web application that would +> require a database layer that afforded higher availability and scale out +> properties than any technology we knew of. So, we rolled our own. +> +> After using Riak KV in production for several successful applications that +> generated revenue, we decided to open source it and share our creation with +> the world. + +Traditionally production systems store their state in relational databases. For +many of the more common usage patterns of state persistence, however, a +relational database is a solution that is far from ideal. Most of these services +only store and retrieve data by primary key and do not require the complex +querying and management functionality offered by an RDBMS. This excess +functionality requires expensive hardware and highly skilled personnel for its +operation, making it a very inefficient solution. In addition, the available +replication technologies are limited and typically choose consistency over +availability. Although many advances have been made in the recent years, it is +still not easy to scale-out databases or use smart partitioning schemes for load +balancing. + +This paper describes Dynamo, a highly available data storage technology that +addresses the needs of these important classes of services. Dynamo has a simple +key/value interface, is highly available with a clearly defined consistency +window, is efficient in its resource usage, and has a simple scale out scheme to +address growth in data set size or request rates. Each service that uses Dynamo +runs its own Dynamo instances. + +> Riak KV is a highly available, scalable, open source key/value database. These +> notes describe where Riak KV's design decisions emulated and diverged from +> Dynamo's (as described in this paper). +> +> Riak KV offers several query methods in addition to the standard key/value +> interface, is made to be highly-available, is efficient in its resource uses, +> and has a simple scale out story to accompany data and traffic growth. + + +### 2.1 System Assumptions and Requirements + +The storage system for this class of services has the following requirements: + + +* Query Model: simple read and write operations to a data item that is uniquely +identified by a key. State is stored as binary objects (i.e., blobs) identified +by unique keys. No operations span multiple data items and there is no need for +relational schema. This requirement is based on the observation that a +significant portion of Amazon’s services can work with this simple query model +and do not need any relational schema. Dynamo targets applications that need to +store objects that are relatively small (usually less than 1 MB). + +> **Riak KV's Query Model** +> +> We've extended Dynamo's proposed query model in several ways. Currently Riak +> KV offers: +> +> 1. Standard key/value access (GET, PUT, DELETE) +> 2. MapReduce querying +> 3. Secondary Indexing +> 4. Full-text Search +> +> Riak KV's realistic object size limit is around 5MB. + +* ACID Properties: ACID (Atomicity, Consistency, Isolation, Durability) is a set +of properties that guarantee that database transactions are processed reliably. +In the context of databases, a single logical operation on the data is called a +transaction. Experience at Amazon has shown that data stores that provide ACID +guarantees tend to have poor availability. This has been widely acknowledged by +both the industry and academia [5]. Dynamo targets applications that operate +with weaker consistency (the “C” in ACID) if this results in high availability. +Dynamo does not provide any isolation guarantees and permits only single key +updates. + +> **ACID?** +> +> Riak KV offers no traditional "ACID" semantics around transactions. Instead, +> it's built to be "eventually consistent." We did this because we were of the +> opinion (and our users proved this out)that most applications don't require +> heavy transactions. (Even ATMs are eventually consistent.) + +* Efficiency: The system needs to function on a commodity hardware +infrastructure. In Amazon’s platform, services have stringent latency +requirements which are in general measured at the 99.9th percentile of the +distribution. Given that state access plays a crucial role in service operation +the storage system must be capable of meeting such stringent SLAs (see Section +2.2 below). Services must be able to configure Dynamo such that they +consistently achieve their latency and throughput requirements. The tradeoffs +are in performance, cost efficiency, availability, and durability guarantees. + +> **Efficiency** +> +> Agreed. Riak KV is made to (and will!) scale linearly on commodity hardware +> (often called "pizza boxes"). + +* Other Assumptions: Dynamo is used only by Amazon’s internal services. Its +operation environment is assumed to be non-hostile and there are no security +related requirements such as authentication and authorization. Moreover, since +each service uses its distinct instance of Dynamo, its initial design targets a +scale of up to hundreds of storage hosts. We will discuss the scalability +limitations of Dynamo and possible scalability related extensions in later +sections. + + +### 2.2 Service Level Agreements (SLA) + +To guarantee that the application can deliver its functionality in a bounded +time, each and every dependency in the platform needs to deliver its +functionality with even tighter bounds. Clients and services engage in a Service +Level Agreement (SLA), a formally negotiated contract where a client and a +service agree on several system-related characteristics, which most prominently +include the client’s expected request rate distribution for a particular API and +the expected service latency under those conditions. An example of a simple SLA +is a service guaranteeing that it will provide a response within 300ms for 99.9% +of its requests for a peak client load of 500 requests per second. + +In Amazon’s decentralized service oriented infrastructure, SLAs play an +important role. For example a page request to one of the e-commerce sites +typically requires the rendering engine to construct its response by sending +requests to over 150 services. These services often have multiple dependencies, +which frequently are other services, and as such it is not uncommon for the call +graph of an application to have more than one level. To ensure that the page +rendering engine can maintain a clear bound on page delivery each service within +the call chain must obey its performance contract. + +> **Riak KV Loves SLAs** +> +> Much like Amazon built Dynamo to guarantee their applications were always +> available to retail shoppers, the design decisions in Riak KV were taken to +> ensure that developers could sleep well knowing that their database would +> always be available to serve requests. +> +> Many of our clients and open source users have explicit uptime agreements +> related to their applications and services built on Riak KV. This was not an +> accident. + + +<a href="#figure-1">Figure 1</a> shows an abstract view of the architecture of +Amazon’s platform, where dynamic web content is generated by page rendering +components which in turn query many other services. A service can use different +data stores to manage its state and these data stores are only accessible within +its service boundaries. Some services act as aggregators by using several other +services to produce a composite response. Typically, the aggregator services are +stateless, although they use extensive caching. + +**<figure id="figure-1" style="text-align:center;"> + <img src="/riak-docs/images/dynamo/figure1.png"> + <figcaption> + Figure 1: Service-oriented architecture of Amazon’s platform. + </figcaption> +</figure>** + +A common approach in the industry for forming a performance oriented SLA is to +describe it using average, median and expected variance. At Amazon we have found +that these metrics are not good enough if the goal is to build a system where +all customers have a good experience, rather than just the majority. For example +if extensive personalization techniques are used then customers with longer +histories require more processing which impacts performance at the high-end of +the distribution. An SLA stated in terms of mean or median response times will +not address the performance of this important customer segment. To address this +issue, at Amazon, SLAs are expressed and measured at the 99.9th percentile of +the distribution. The choice for 99.9% over an even higher percentile has been +made based on a cost-benefit analysis which demonstrated a significant increase +in cost to improve performance that much. Experiences with Amazon’s production +systems have shown that this approach provides a better overall experience +compared to those systems that meet SLAs defined based on the mean or median. + +In this paper there are many references to this 99.9th percentile of +distributions, which reflects Amazon engineers’ relentless focus on performance +from the perspective of the customers’ experience. Many papers report on +averages, so these are included where it makes sense for comparison purposes. +Nevertheless, Amazon’s engineering and optimization efforts are not focused on +averages. Several techniques, such as the load balanced selection of write +coordinators, are purely targeted at controlling performance at the 99.9th +percentile. + +Storage systems often play an important role in establishing a service’s SLA, +especially if the business logic is relatively lightweight, as is the case for +many Amazon services. State management then becomes the main component of a +service’s SLA. One of the main design considerations for Dynamo is to give +services control over their system properties, such as durability and +consistency, and to let services make their own tradeoffs between functionality, +performance and cost-effectiveness. + + +### 2.3 Design Considerations + +Data replication algorithms used in commercial systems traditionally perform +synchronous replica coordination in order to provide a strongly consistent data +access interface. To achieve this level of consistency, these algorithms are +forced to tradeoff the availability of the data under certain failure scenarios. +For instance, rather than dealing with the uncertainty of the correctness of an +answer, the data is made unavailable until it is absolutely certain that it is +correct. From the very early replicated database works, it is well known that +when dealing with the possibility of network failures, strong consistency and +high data availability cannot be achieved simultaneously [2, 11]. As such +systems and applications need to be aware which properties can be achieved under +which conditions. + +> **Riak KV's Design Considerations** +> +> Availability under any circumstances was something we stressed when designing +> Riak KV, too. Most databases didn't enable developers to do this in a simple +> way so we set out to change this. + +For systems prone to server and network failures, availability can be increased +by using optimistic replication techniques, where changes are allowed to +propagate to replicas in the background, and concurrent, disconnected work is +tolerated. The challenge with this approach is that it can lead to conflicting +changes which must be detected and resolved. This process of conflict resolution +introduces two problems: when to resolve them and who resolves them. Dynamo is +designed to be an eventually consistent data store; that is all updates reach +all replicas eventually. + +> Remember Eventual Consistency? We followed Dynamo's lead here and made sure +> that Riak KV could withstand network, server and other failures by sacrificing +> absolute consistency and building in mechanisms to rectify object conflicts. + +An important design consideration is to decide when to perform the process of +resolving update conflicts, i.e., whether conflicts should be resolved during +reads or writes. Many traditional data stores execute conflict resolution during +writes and keep the read complexity simple [7]. In such systems, writes may be +rejected if the data store cannot reach all (or a majority of) the replicas at a +given time. On the other hand, Dynamo targets the design space of an “always +writeable” data store (i.e., a data store that is highly available for writes). +For a number of Amazon services, rejecting customer updates could result in a +poor customer experience. For instance, the shopping cart service must allow +customers to add and remove items from their shopping cart even amidst network +and server failures. This requirement forces us to push the complexity of +conflict resolution to the reads in order to ensure that writes are never +rejected. + +> Ditto! + +The next design choice is who performs the process of conflict resolution. This +can be done by the data store or the application. If conflict resolution is done +by the data store, its choices are rather limited. In such cases, the data store +can only use simple policies, such as “last write wins” [22], to resolve +conflicting updates. On the other hand, since the application is aware of the +data schema it can decide on the conflict resolution method that is best suited +for its client’s experience. For instance, the application that maintains +customer shopping carts can choose to “merge” the conflicting versions and +return a single unified shopping cart. Despite this flexibility, some +application developers may not want to write their own conflict resolution +mechanisms and choose to push it down to the data store, which in turn chooses a +simple policy such as “last write wins”. + +> No conflict here (pun intended). Riak KV also follows this approach to +> conflict resolution. + +Other key principles embraced in the design are: + +Incremental scalability: Dynamo should be able to scale out one storage host +(henceforth, referred to as “node”) at a time, with minimal impact on both +operators of the system and the system itself. + +> We refer to hosts as "nodes", too. Riak KV provides a simple set of commands +> to start and join nodes to a running cluster. With proper capacity planning, +> this process should be painless for the ops team and devs, and imperceivable +> by the client. + +Symmetry: Every node in Dynamo should have the same set of responsibilities as +its peers; there should be no distinguished node or nodes that take special +roles or extra set of responsibilities. In our experience, symmetry simplifies +the process of system provisioning and maintenance. + +> Again, we agree. Each storage node is the same at its neighbor. Any node can +> coordinate a request and, in the event that a node goes does, its neighbors +> can cover for it until it's restarted or decommissioned. + +Decentralization: An extension of symmetry, the design should favor +decentralized peer-to-peer techniques over centralized control. In the past, +centralized control has resulted in outages and the goal is to avoid it as much +as possible. This leads to a simpler, more scalable, and more available system. + +> A Riak cluster is completely decentralized. No single node is special and this +> leads to no single points of failure. + +Heterogeneity: The system needs to be able to exploit heterogeneity in the +infrastructure it runs on. e.g. the work distribution must be proportional to +the capabilities of the individual servers. This is essential in adding new +nodes with higher capacity without having to upgrade all hosts at once. + +> Riak KV agrees. + + +## 3. Related Work + +> This section is not strictly necessary to read for an understanding of how a +> Dynamo distributed database functions, especially Riak KV. It's still an +> excellent study of other distributed systems, in some cases ones that helped +> inspire Dynamo. When you have time, we highly recommend you read this section. + + +### 3.1 Peer to Peer Systems + +There are several peer-to-peer (P2P) systems that have looked at the problem of +data storage and distribution. The first generation of P2P systems, such as +Freenet and Gnutella, were predominantly used as file sharing systems. These +were examples of unstructured P2P networks where the overlay links between peers +were established arbitrarily. In these networks, a search query is usually +flooded through the network to find as many peers as possible that share the +data. P2P systems evolved to the next generation into what is widely known as +structured P2P networks. These networks employ a globally consistent protocol to +ensure that any node can efficiently route a search query to some peer that has +the desired data. Systems like Pastry [16] and Chord [20] use routing mechanisms +to ensure that queries can be answered within a bounded number of hops. + +To reduce the additional latency introduced by multi-hop routing, some P2P +systems (e.g., [14]) employ O(1) routing where each peer maintains enough +routing information locally so that it can route requests (to access a data +item) to the appropriate peer within a constant number of hops. + +> Riak KV's gossip protocol communicates between nodes with O(1) routing, and +> maintains local routing information. + +Various storage systems, such as Oceanstore [9] and PAST [17] were built on top +of these routing overlays. Oceanstore provides a global, transactional, +persistent storage service that supports serialized updates on widely replicated +data. To allow for concurrent updates while avoiding many of the problems +inherent with wide-area locking, it uses an update model based on conflict +resolution. Conflict resolution was introduced in [21] to reduce the number of +transaction aborts. Oceanstore resolves conflicts by processing a series of +updates, choosing a total order among them, and then applying them atomically in +that order. It is built for an environment where the data is replicated on an +untrusted infrastructure. By comparison, PAST provides a simple abstraction +layer on top of Pastry for persistent and immutable objects. It assumes that the +application can build the necessary storage semantics (such as mutable files) on +top of it. + +### 3.2 Distributed File Systems and Databases + +Distributing data for performance, availability and durability has been widely +studied in the file system and database systems community. Compared to P2P +storage systems that only support flat namespaces, distributed file systems +typically support hierarchical namespaces. Systems like Ficus [15] and Coda [19] +replicate files for high availability at the expense of consistency. Update +conflicts are typically managed using specialized conflict resolution +procedures. The Farsite system [1] is a distributed file system that does not +use any centralized server like NFS. Farsite achieves high availability and +scalability using replication. The Google File System [6] is another distributed +file system built for hosting the state of Google’s internal applications. GFS +uses a simple design with a single master server for hosting the entire metadata +and where the data is split into chunks and stored in chunkservers. Bayou is a +distributed relational database system that allows disconnected operations and +provides eventual data consistency [21]. + +Among these systems, Bayou, Coda and Ficus allow disconnected operations and are +resilient to issues such as network partitions and outages. These systems differ +on their conflict resolution procedures. For instance, Coda and Ficus perform +system level conflict resolution and Bayou allows application level resolution. +All of them, however, guarantee eventual consistency. + +Similar to these systems, Dynamo allows read and write operations to continue +even during network partitions and resolves updated conflicts using different +conflict resolution mechanisms. Distributed block storage systems like FAB [18] +split large size objects into smaller blocks and stores each block in a highly +available manner. In comparison to these systems, a key-value store is more +suitable in this case because: (a) it is intended to store relatively small +objects (size < 1M) and (b) key-value stores are easier to configure on a per- +application basis. Antiquity is a wide-area distributed storage system designed +to handle multiple server failures [23]. It uses a secure log to preserve data +integrity, replicates each log on multiple servers for durability, and uses +Byzantine fault tolerance protocols to ensure data consistency. In contrast to +Antiquity, Dynamo does not focus on the problem of data integrity and security +and is built for a trusted environment. Bigtable is a distributed storage system +for managing structured data. It maintains a sparse, multi-dimensional sorted +map and allows applications to access their data using multiple attributes [2]. +Compared to Bigtable, Dynamo targets applications that require only key/value +access with primary focus on high availability where updates are not rejected +even in the wake of network partitions or server failures. + +> This all applies to Riak KV, as well. + +Traditional replicated relational database systems focus on the problem of +guaranteeing strong consistency to replicated data. Although strong consistency +provides the application writer a convenient programming model, these systems +are limited in scalability and availability [7]. These systems are not capable +of handling network partitions because they typically provide strong consistency +guarantees. + +### 3.3 Discussion + +Dynamo differs from the aforementioned decentralized storage systems in terms of +its target requirements. First, Dynamo is targeted mainly at applications that +need an “always writeable” data store where no updates are rejected due to +failures or concurrent writes. This is a crucial requirement for many Amazon +applications. Second, as noted earlier, Dynamo is built for an infrastructure +within a single administrative domain where all nodes are assumed to be trusted. +Third, applications that use Dynamo do not require support for hierarchical +namespaces (a norm in many file systems) or complex relational schema (supported +by traditional databases). Fourth, Dynamo is built for latency sensitive +applications that require at least 99.9% of read and write operations to be +performed within a few hundred milliseconds. To meet these stringent latency +requirements, it was imperative for us to avoid routing requests through +multiple nodes (which is the typical design adopted by several distributed hash +table systems such as Chord and Pastry). This is because multi-hop routing +increases variability in response times, thereby increasing the latency at +higher percentiles. Dynamo can be characterized as a zero-hop DHT, where each +node maintains enough routing information locally to route a request to the +appropriate node directly. + + +## 4.System Architecture + +> This is truly the meat of the Dynamo paper. Stick around. It gets good. + +The architecture of a storage system that needs to operate in a production +setting is complex. In addition to the actual data persistence component, the +system needs to have scalable and robust solutions for load balancing, +membership and failure detection, failure recovery, replica synchronization, +overload handling, state transfer, concurrency and job scheduling, request +marshalling, request routing, system monitoring and alarming, and configuration +management. Describing the details of each of the solutions is not possible, so +this paper focuses on the core distributed systems techniques used in Dynamo: +partitioning, replication, versioning, membership, failure handling and scaling. +<a href="#table-1">Table 1</a> presents a summary of the list of techniques +Dynamo uses and their respective advantages. + +<table id="table-1"> + <caption> + Table 1: Summary of techniques used in Dynamo and their advantages. + </caption> + <tr> + <th>Problem</th> + <th>Technique</th> + <th>Advantage</th> + </tr> + <tr> + <td>Partitioning</td> + <td>Consistent Hashing</td> + <td>Incremental Scalability</td> + </tr> + <tr> + <td>High Availability for writes</td> + <td>Vector clocks with reconciliation during reads</td> + <td>Version size is decoupled from update rates.</td> + </tr> + <tr> + <td>Handling temporary failures</td> + <td>Sloppy Quorum and hinted handoff</td> + <td>Provides high availability and durability guarantee when some of the + replicas are not available.</td> + </tr> + <tr> + <td>Recovering from permanent failures</td> + <td>Anti-entropy using Merkle trees</td> + <td>Synchronizes divergent replicas in the background.</td> + </tr> + <tr> + <td>Membership and failure detection</td> + <td>Gossip-based membership protocol and failure detection.</td> + <td>Preserves symmetry and avoids having a centralized registry for storing + membership and node liveness information.</td> + </tr> +</table> + +### 4.1 System Interface + +Dynamo stores objects associated with a key through a simple interface; it +exposes two operations: get() and put(). The get(key) operation locates the +object replicas associated with the key in the storage system and returns a +single object or a list of objects with conflicting versions along with a +context. The put(key, context, object) operation determines where the replicas +of the object should be placed based on the associated key, and writes the +replicas to disk. The context encodes system metadata about the object that is +opaque to the caller and includes information such as the version of the object. +The context information is stored along with the object so that the system can +verify the validity of the context object supplied in the put request. + +> Whereas Dynamo only has the concept of keys, we added a higher level of +> organization called a "bucket." Keys are stored in buckets and buckets are the +> level at which several Riak KV properties can be configured (primarily the "N" +> value, or the replication value.) In addition to the bucket+key identifier and +> value, Riak KV will also return the associated metadata for a given object +> with each get or put. +> +> Riak KV has two APIs: an [HTTP API] and a [Protocol Buffers API]. + +[HTTP API]: {{<baseurl>}}riak/kv/2.2.6/developing/api/http/ +[Protocol Buffers API]: {{<baseurl>}}riak/kv/2.2.6/developing/api/protocol-buffers/ + +Dynamo treats both the key and the object supplied by the caller as an opaque +array of bytes. It applies a MD5 hash on the key to generate a 128-bit +identifier, which is used to determine the storage nodes that are responsible +for serving the key. + +> Riak KV concatenates the bucket with the key and runs it through the SHA1 hash +> to generate a 160 bit identifier which is then used to determine where in the +> database each datum is stored. Riak KV treats data as an opaque binary, thus +> enabling users to store virtually anything. + + +### 4.2 Partitioning Algorithm + +One of the key design requirements for Dynamo is that it must scale +incrementally. This requires a mechanism to dynamically partition the data over +the set of nodes (i.e., storage hosts) in the system. Dynamo’s partitioning +scheme relies on consistent hashing to distribute the load across multiple +storage hosts. In consistent hashing [10], the output range of a hash function +is treated as a fixed circular space or “ring” (i.e. the largest hash value +wraps around to the smallest hash value). Each node in the system is assigned a +random value within this space which represents its “position” on the ring. Each +data item identified by a key is assigned to a node by hashing the data item’s +key to yield its position on the ring, and then walking the ring clockwise to +find the first node with a position larger than the item’s position. Thus, each +node becomes responsible for the region in the ring between it and its +predecessor node on the ring. The principle advantage of consistent hashing is +that departure or arrival of a node only affects its immediate neighbors and +other nodes remain unaffected. + +> **Partitioning in Riak KV** +> +> As mentioned above, Riak KV uses consistent hashing to distribute data around +> ring to partitions responsible for storing data. The ring has a maximum key +> space of 2^160. Each bucket+key (and its associated value) is hashed to a +> location on the ring. +> +> Riak KV also breaks the ring into a set number of partitions. This number is +> configured when a cluster is first built. Each node will be responsible for +> storing the data hashed to a set number of partitions. Each storage node will +> optimistically handle an equal number of partitions. + +The basic consistent hashing algorithm presents some challenges. First, the +random position assignment of each node on the ring leads to non-uniform data +and load distribution. Second, the basic algorithm is oblivious to the +heterogeneity in the performance of nodes. To address these issues, Dynamo uses +a variant of consistent hashing (similar to the one used in [10, 20]): instead +of mapping a node to a single point in the circle, each node gets assigned to +multiple points in the ring. To this end, Dynamo uses the concept of “virtual +nodes”. A virtual node looks like a single node in the system, but each node can +be responsible for more than one virtual node. Effectively, when a new node is +added to the system, it is assigned multiple positions (henceforth, “tokens”) in +the ring. The process of fine-tuning Dynamo’s partitioning scheme is discussed +in Section 6. + +> Riak KV also has the concept of virtual nodes and they are used to the same +> end as they are in Dynamo. Physical storage nodes are responsible for +> partitions, and each partition a vnode. + +Using virtual nodes has the following advantages: + +If a node becomes unavailable (due to failures or routine maintenance), the load +handled by this node is evenly dispersed across the remaining available nodes. + +When a node becomes available again, or a new node is added to the system, the +newly available node accepts a roughly equivalent amount of load from each of +the other available nodes. + +> All of these properties for vnodes in Dynamo hold true for Riak KV, too. + +The number of virtual nodes that a node is responsible can decided based on its +capacity, accounting for heterogeneity in the physical infrastructure. + +> [Further Reading on Partitioning in Riak KV] and [All about the Riak KV Ring]. + +[Further Reading on Partitioning in Riak KV]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/clusters/ +[All about the Riak KV Ring]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/clusters/#the-ring + +### 4.3 Replication + +To achieve high availability and durability, Dynamo replicates its data on +multiple hosts. Each data item is replicated at N hosts, where N is a parameter +configured “per-instance”. Each key, k, is assigned to a coordinator node +(described in the previous section). The coordinator is in charge of the +replication of the data items that fall within its range. In addition to locally +storing each key within its range, the coordinator replicates these keys at the +N-1 clockwise successor nodes in the ring. This results in a system where each +node is responsible for the region of the ring between it and its Nth +predecessor. In <a href="#figure-2">Figure 2</a>, node B replicates the key k at +nodes C and D in addition to storing it locally. Node D will store the keys that +fall in the ranges (A, B], (B, C], and (C, D]. + +**<figure id="figure-2" style="text-align:center;"> + <img src="/riak-docs/images/dynamo/figure2.png"> + <figcaption> + Figure 2: Partitioning and replication of keys in Dynamo ring. + </figcaption> +</figure>** + +> Replication in Riak KV, like in Dynamo, is fundamental and automatic. Remember +> the concept of a bucket we covered above? In Riak KV, the replication +> parameter, "N" (also called "n_val"), is configurable at the bucket level. +> The default n_val in Riak KV is 3, meaning that out of the box Riak KV will +> store three replicas of your data on three different partitions on the ring. +> +> The diagram is applicable to Riak KV and the manner in which it replicates +> data. The preference list is present in Riak KV, too, and is the reason why +> any node in the ring can coordinate a request. The node receives a request, +> consults the preference list, and routes the request accordingly. + +The list of nodes that is responsible for storing a particular key is called the +preference list. The system is designed, as will be explained in Section 4.8, so +that every node in the system can determine which nodes should be in this list +for any particular key. To account for node failures, preference list contains +more than N nodes. Note that with the use of virtual nodes, it is possible that +the first N successor positions for a particular key may be owned by less than N +distinct physical nodes (i.e. a node may hold more than one of the first N +positions). To address this, the preference list for a key is constructed by +skipping positions in the ring to ensure that the list contains only distinct +physical nodes. + + +### 4.4 Data Versioning + +Dynamo provides eventual consistency, which allows for updates to be propagated +to all replicas asynchronously. A put() call may return to its caller before the +update has been applied at all the replicas, which can result in scenarios where +a subsequent get() operation may return an object that does not have the latest +updates.. If there are no failures then there is a bound on the update +propagation times. However, under certain failure scenarios (e.g., server +outages or network partitions), updates may not arrive at all replicas for an +extended period of time. + +> Riak KV is an "eventually consistent" database. All replication is done +> asynchronously, as you would expect, could result in a datum being returned to +> the client that is out of date. But don't worry. We built in some mechanisms +> to address this. + +There is a category of applications in Amazon’s platform that can tolerate such +inconsistencies and can be constructed to operate under these conditions. For +example, the shopping cart application requires that an “Add to Cart” operation +can never be forgotten or rejected. If the most recent state of the cart is +unavailable, and a user makes changes to an older version of the cart, that +change is still meaningful and should be preserved. But at the same time it +shouldn’t supersede the currently unavailable state of the cart, which itself +may contain changes that should be preserved. Note that both “add to cart” and +“delete item from cart” operations are translated into put requests to Dynamo. +When a customer wants to add an item to (or remove from) a shopping cart and the +latest version is not available, the item is added to (or removed from) the +older version and the divergent versions are reconciled later. + +> Much like Dynamo was suited to the design of the shopping cart, Riak KV, and +> its tradeoffs, are appropriate for a certain set of use cases. We happen to +> feel that _most_ use cases can tolerate some level of eventual consistency. + +In order to provide this kind of guarantee, Dynamo treats the result of each +modification as a new and immutable version of the data. It allows for multiple +versions of an object to be present in the system at the same time. Most of the +time, new versions subsume the previous version(s), and the system itself can +determine the authoritative version (syntactic reconciliation). However, version +branching may happen, in the presence of failures combined with concurrent +updates, resulting in conflicting versions of an object. In these cases, the +system cannot reconcile the multiple versions of the same object and the client +must perform the reconciliation in order to collapse multiple branches of data +evolution back into one (semantic reconciliation). A typical example of a +collapse operation is “merging” different versions of a customer’s shopping +cart. Using this reconciliation mechanism, an “add to cart” operation is never +lost. However, deleted items can resurface. + +> The same holds true for Riak KV. If, by way of some failure and concurrent +> update (rare but quite possible), there come to exist multiple versions of the +> same object, Riak KV will push this decision down to the client (who are we to +> tell you which is the authoritative object?). All that said, if your +> application doesn't need this level of version control, we enable you to turn +> the usage of vector clocks on and off at the bucket level. + +It is important to understand that certain failure modes can potentially result +in the system having not just two but several versions of the same data. Updates +in the presence of network partitions and node failures can potentially result +in an object having distinct version sub-histories, which the system will need +to reconcile in the future. This requires us to design applications that +explicitly acknowledge the possibility of multiple versions of the same data (in +order to never lose any updates). + +> Ditto. + +Dynamo uses vector clocks [12] in order to capture causality between different +versions of the same object. A vector clock is effectively a list of (node, +counter) pairs. One vector clock is associated with every version of every +object. One can determine whether two versions of an object are on parallel +branches or have a causal ordering, by examine their vector clocks. If the +counters on the first object’s clock are less-than-or-equal to all of the nodes +in the second clock, then the first is an ancestor of the second and can be +forgotten. Otherwise, the two changes are considered to be in conflict and +require reconciliation. + +> As you may have already figured out, Riak KV uses vector clocks for object +> versioning, too. Here are a whole host of resources to keep you busy for a while: +> +> [Vector Clock on Riak KV Glossary]({{<baseurl>}}riak/kv/2.2.6/learn/glossary/#vector-clock) +> +> [Why Vector Clocks are Easy](http://basho.com/posts/technical/why-vector-clocks-are-easy/) +> | +> [Why Vector Clocks are Hard](http://basho.com/posts/technical/why-vector-clocks-are-hard/) +> +> [Vector Clocks Revisited](http://basho.com/posts/technical/vector-clocks-revisited/) +> +> [Vector Clocks on Wikipedia](https://en.wikipedia.org/wiki/Vector_clock) + +In Dynamo, when a client wishes to update an object, it must specify which +version it is updating. This is done by passing the context it obtained from an +earlier read operation, which contains the vector clock information. Upon +processing a read request, if Dynamo has access to multiple branches that cannot +be syntactically reconciled, it will return all the objects at the leaves, with +the corresponding version information in the context. An update using this +context is considered to have reconciled the divergent versions and the branches +are collapsed into a single new version. + +**<figure id="figure-3" style="text-align:center;"> + <img src="/riak-docs/images/dynamo/figure3.png"> + <figcaption> + Figure 3: Version evolution of an object over time. + </figcaption> +</figure>** + +To illustrate the use of vector clocks, let us consider the example shown in +<a href="#figure-3">Figure 3</a>. A client writes a new object. The node (say +Sx) that handles the write for this key increases its sequence number and uses +it to create the data's vector clock. The system now has the object D1 and its +associated clock [(Sx, 1)]. The client updates the object. Assume the same node +handles this request as well. The system now also has object D2 and its +associated clock [(Sx, 2)]. D2 descends from D1 and therefore over-writes D1, +however there may be replicas of D1 lingering at nodes that have not yet seen +D2. Let us assume that the same client updates the object again and a different +server (say Sy) handles the request. The system now has data D3 and its +associated clock [(Sx, 2), (Sy, 1)]. + +Next assume a different client reads D2 and then tries to update it, and another +node (say Sz) does the write. The system now has D4 (descendant of D2) whose +version clock is [(Sx, 2), (Sz, 1)]. A node that is aware of D1 or D2 could +determine, upon receiving D4 and its clock, that D1 and D2 are overwritten by +the new data and can be garbage collected. A node that is aware of D3 and +receives D4 will find that there is no causal relation between them. In other +words, there are changes in D3 and D4 that are not reflected in each other. Both +versions of the data must be kept and presented to a client (upon a read) for +semantic reconciliation. + +Now assume some client reads both D3 and D4 (the context will reflect that both +values were found by the read). The read's context is a summary of the clocks of +D3 and D4, namely [(Sx, 2), (Sy, 1), (Sz, 1)]. If the client performs the +reconciliation and node Sx coordinates the write, Sx will update its sequence +number in the clock. The new data D5 will have the following clock: [(Sx, 3), +(Sy, 1), (Sz, 1)]. + +A possible issue with vector clocks is that the size of vector clocks may grow +if many servers coordinate the writes to an object. In practice, this is not +likely because the writes are usually handled by one of the top N nodes in the +preference list. In case of network partitions or multiple server failures, +write requests may be handled by nodes that are not in the top N nodes in the +preference list causing the size of vector clock to grow. In these scenarios, it +is desirable to limit the size of vector clock. To this end, Dynamo employs the +following clock truncation scheme: Along with each (node, counter) pair, Dynamo +stores a timestamp that indicates the last time the node updated the data item. +When the number of (node, counter) pairs in the vector clock reaches a threshold +(say 10), the oldest pair is removed from the clock. Clearly, this truncation +scheme can lead to inefficiencies in reconciliation as the descendant +relationships cannot be derived accurately. However, this problem has not +surfaced in production and therefore this issue has not been thoroughly +investigated. + +> Riak KV does a certain amount of vector clock pruning to ensure their growth +> is kept under control. + + +### 4.5 Execution of get () and put () operations + +Any storage node in Dynamo is eligible to receive client get and put operations +for any key. In this section, for sake of simplicity, we describe how these +operations are performed in a failure-free environment and in the subsequent +section we describe how read and write operations are executed during failures. + +> Any node in the Riak KV ring can coordinate a request. The Riak KV information +> in this section applies to a failure-free environment. + +Both get and put operations are invoked using Amazon’s infrastructure-specific +request processing framework over HTTP. There are two strategies that a client +can use to select a node: (1) route its request through a generic load balancer +that will select a node based on load information, or (2) use a partition-aware +client library that routes requests directly to the appropriate coordinator +nodes. The advantage of the first approach is that the client does not have to +link any code specific to Dynamo in its application, whereas the second strategy +can achieve lower latency because it skips a potential forwarding step. + +A node handling a read or write operation is known as the coordinator. +Typically, this is the first among the top N nodes in the preference list. If +the requests are received through a load balancer, requests to access a key may +be routed to any random node in the ring. In this scenario, the node that +receives the request will not coordinate it if the node is not in the top N of +the requested key’s preference list. Instead, that node will forward the request +to the first among the top N nodes in the preference list. + +Read and write operations involve the first N healthy nodes in the preference +list, skipping over those that are down or inaccessible. When all nodes are +healthy, the top N nodes in a key’s preference list are accessed. When there are +node failures or network partitions, nodes that are lower ranked in the +preference list are accessed. + +To maintain consistency among its replicas, Dynamo uses a consistency protocol +similar to those used in quorum systems. This protocol has two key configurable +values: R and W. R is the minimum number of nodes that must participate in a +successful read operation. W is the minimum number of nodes that must +participate in a successful write operation. Setting R and W such that R + W > N +yields a quorum-like system. In this model, the latency of a get (or put) +operation is dictated by the slowest of the R (or W) replicas. For this reason, +R and W are usually configured to be less than N, to provide better latency. + +> Riak KV makes use of the same values. But, thanks to our concept of buckets, +> we made it a bit more customizable. The default R and W values are set at the +> bucket level but can be configured at the request level if the developer deems +> it necessary for certain data. "Quorum" as described in Dynamo is the default +> setting in Riak KV. +> +>Some more resources on R and W: +> +>[REST API]({{<baseurl>}}riak/kv/2.2.6/developing/api/http/) +> +>[Writing Data]({{<baseurl>}}riak/kv/2.2.6/developing/usage/creating-objects/) +> +>[Reading Data]({{<baseurl>}}riak/kv/2.2.6/developing/usage/reading-objects/) + +Upon receiving a put() request for a key, the coordinator generates the vector +clock for the new version and writes the new version locally. The coordinator +then sends the new version (along with the new vector clock) to the N highest- +ranked reachable nodes. If at least W-1 nodes respond then the write is +considered successful. + +> In Riak KV a write is considered successful when the total number of +> responding writes equals W. This need not be a durable write, which is a +> separate value in Riak KV labeled DW. + +Similarly, for a get() request, the coordinator requests all existing versions +of data for that key from the N highest-ranked reachable nodes in the preference +list for that key, and then waits for R responses before returning the result to +the client. If the coordinator ends up gathering multiple versions of the data, +it returns all the versions it deems to be causally unrelated. The divergent +versions are then reconciled and the reconciled version superseding the current +versions is written back. + +> Same for Riak KV. Reconciling divergent versions in Riak KV is called +> [Read Repair]({{<baseurl>}}riak/kv/2.2.6/learn/concepts/replication/#read-repair). + + +### 4.6 Handling Failures: Hinted Handoff + +If Dynamo used a traditional quorum approach it would be unavailable during +server failures and network partitions, and would have reduced durability even +under the simplest of failure conditions. To remedy this it does not enforce +strict quorum membership and instead it uses a “sloppy quorum”; all read and +write operations are performed on the first N healthy nodes from the preference +list, which may not always be the first N nodes encountered while walking the +consistent hashing ring. + +> [Hinted handoff] is built into Riak KV's core. +> +> You can glimpse at Riak KV's preference list (or *preflist*) calculation in +> the [Replication] walkthrough. + +[Hinted handoff]: {{<baseurl>}}riak/kv/2.2.6/learn/glossary/#hinted-handoff +[Replication]: {{<baseurl>}}riak/kv/2.2.6/developing/usage/replication/ + +Consider the example of Dynamo configuration given in <a href="#figure-2">Figure +2</a> with N=3. In this example, if node A is temporarily down or unreachable +during a write operation then a replica that would normally have lived on A will +now be sent to node D. This is done to maintain the desired availability and +durability guarantees. The replica sent to D will have a hint in its metadata +that suggests which node was the intended recipient of the replica (in this case +A). Nodes that receive hinted replicas will keep them in a separate local +database that is scanned periodically. Upon detecting that A has recovered, D +will attempt to deliver the replica to A. Once the transfer succeeds, D may +delete the object from its local store without decreasing the total number of +replicas in the system. + +Using hinted handoff, Dynamo ensures that the read and write operations are not +failed due to temporary node or network failures. Applications that need the +highest level of availability can set W to 1, which ensures that a write is +accepted as long as a single node in the system has durably written the key it +to its local store. Thus, the write request is only rejected if all nodes in the +system are unavailable. However, in practice, most Amazon services in production +set a higher W to meet the desired level of durability. A more detailed +discussion of configuring N, R and W follows in section 6. + +> As mentioned previously, Riak KV does not require that a write be durable, +> only that a vnode responds in the affirmative. If you require a durable write +> in the way mentioned here, use DW. + +It is imperative that a highly available storage system be capable of handling +the failure of an entire data center(s). Data center failures happen due to +power outages, cooling failures, network failures, and natural disasters. Dynamo +is configured such that each object is replicated across multiple data centers. +In essence, the preference list of a key is constructed such that the storage +nodes are spread across multiple data centers. These datacenters are connected +through high speed network links. This scheme of replicating across multiple +datacenters allows us to handle entire data center failures without a data +outage. + +> [Multi Datacenter Replication] was previously only implemented in the commercial extension to +> Riak KV, called [Riak KV Enterprise Edition]. Now it is available in all versions from Riak KV 2.2.6 onwards. + +[Multi Datacenter Replication]: {{<baseurl>}}riak/kv/2.2.6/using/reference/v3-multi-datacenter/architecture/ +[Riak KV Enterprise Edition]: http://basho.com/products/riak-kv/ + + +### 4.7 Handling permanent failures: Replica synchronization + +Hinted handoff works best if the system membership churn is low and node +failures are transient. There are scenarios under which hinted replicas become +unavailable before they can be returned to the original replica node. To handle +this and other threats to durability, Dynamo implements an anti-entropy (replica +synchronization) protocol to keep the replicas synchronized. + +> Read repair, mentioned above, is the simplest form of anti-entropy. But it is +> passive, not active as this section describes. + +To detect the inconsistencies between replicas faster and to minimize the amount +of transferred data, Dynamo uses Merkle trees [13]. A Merkle tree is a hash tree +where leaves are hashes of the values of individual keys. Parent nodes higher in +the tree are hashes of their respective children. The principal advantage of +Merkle tree is that each branch of the tree can be checked independently without +requiring nodes to download the entire tree or the entire data set. Moreover, +Merkle trees help in reducing the amount of data that needs to be transferred +while checking for inconsistencies among replicas. For instance, if the hash +values of the root of two trees are equal, then the values of the leaf nodes in +the tree are equal and the nodes require no synchronization. If not, it implies +that the values of some replicas are different. In such cases, the nodes may +exchange the hash values of children and the process continues until it reaches +the leaves of the trees, at which point the hosts can identify the keys that are +“out of sync”. Merkle trees minimize the amount of data that needs to be +transferred for synchronization and reduce the number of disk reads performed +during the anti-entropy process. + +> Riak KV implements a Merkel-Tree based Active Anti-Entropy (*AAE*). + +Dynamo uses Merkle trees for anti-entropy as follows: Each node maintains a +separate Merkle tree for each key range (the set of keys covered by a virtual +node) it hosts. This allows nodes to compare whether the keys within a key range +are up-to-date. In this scheme, two nodes exchange the root of the Merkle tree +corresponding to the key ranges that they host in common. Subsequently, using +the tree traversal scheme described above the nodes determine if they have any +differences and perform the appropriate synchronization action. The disadvantage +with this scheme is that many key ranges change when a node joins or leaves the +system thereby requiring the tree(s) to be recalculated. This issue is +addressed, however, by the refined partitioning scheme described in Section 6.2. + + +### 4.8 Membership and Failure Detection + +> This section is well expressed in [Adding and Removing Nodes] and +> [Failure Scenarios]. + +[Adding and Removing Nodes]: {{<baseurl>}}riak/kv/2.2.6/using/cluster-operations/adding-removing-nodes/ +[Failure Scenarios]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/eventual-consistency/ + +#### 4.8.1 Ring Membership + +> Riak KV operators can trigger node management via the +> [riak-admin command-line tool]. + +[riak-admin command-line tool]: {{<baseurl>}}riak/kv/2.2.6/using/admin/riak-admin/ + +In Amazon’s environment node outages (due to failures and maintenance tasks) are +often transient but may last for extended intervals. A node outage rarely +signifies a permanent departure and therefore should not result in rebalancing +of the partition assignment or repair of the unreachable replicas. Similarly, +manual error could result in the unintentional startup of new Dynamo nodes. For +these reasons, it was deemed appropriate to use an explicit mechanism to +initiate the addition and removal of nodes from a Dynamo ring. An administrator +uses a command line tool or a browser to connect to a Dynamo node and issue a +membership change to join a node to a ring or remove a node from a ring. The +node that serves the request writes the membership change and its time of issue +to persistent store. The membership changes form a history because nodes can be +removed and added back multiple times. + +> Nodes are manually added using the `riak-admin cluster join`. +> +> When a node permanently departs, rebalancing is triggered using the +> `riak-admin cluster leave` command. + +A gossip-based protocol propagates membership changes and maintains an +eventually consistent view of membership. Each node contacts a peer chosen at +random every second and the two nodes efficiently reconcile their persisted +membership change histories. + +> Riak KV's ring state holds membership information, and is propgated via +> [gossiping], including random reconciliation, defaulting to once a minute. + +[gossiping]: {{<baseurl>}}riak/kv/2.2.6/learn/glossary/#gossiping + +When a node starts for the first time, it chooses its set of tokens (virtual +nodes in the consistent hash space) and maps nodes to their respective token +sets. The mapping is persisted on disk and initially contains only the local +node and token set. The mappings stored at different Dynamo nodes are reconciled +during the same communication exchange that reconciles the membership change +histories. Therefore, partitioning and placement information also propagates via +the gossip-based protocol and each storage node is aware of the token ranges +handled by its peers. This allows each node to forward a key’s read/write +operations to the right set of nodes directly. + +> These tokens are vnodes (virtual nodes) in Riak KV. + + +#### 4.8.2 External Discovery + +The mechanism described above could temporarily result in a logically +partitioned Dynamo ring. For example, the administrator could contact node A to +join A to the ring, then contact node B to join B to the ring. In this scenario, +nodes A and B would each consider itself a member of the ring, yet neither would +be immediately aware of the other. To prevent logical partitions, some Dynamo +nodes play the role of seeds. Seeds are nodes that are discovered via an +external mechanism and are known to all nodes. Because all nodes eventually +reconcile their membership with a seed, logical partitions are highly unlikely. +Seeds can be obtained either from static configuration or from a configuration +service. Typically seeds are fully functional nodes in the Dynamo ring. + +> To rectify these sorts of logical partitions, multiple Riak cluster changes +> are configured as one batch. Any changes must first be viewed `riak-admin +> cluster plan`, then the changes are committed with `riak-admin cluster +> commit`. The new ring state is gossiped. +> +> See _[The Node Join Process]_ for more. + +[The Node Join Process]: {{<baseurl>}}riak/kv/2.2.6/using/cluster-operations/adding-removing-nodes/#joining-nodes-to-form-a-cluster + + +#### 4.8.3 Failure Detection + +Failure detection in Dynamo is used to avoid attempts to communicate with +unreachable peers during get() and put() operations and when transferring +partitions and hinted replicas. For the purpose of avoiding failed attempts at +communication, a purely local notion of failure detection is entirely +sufficient: node A may consider node B failed if node B does not respond to node +A’s messages (even if B is responsive to node C*s messages). In the presence of +a steady rate of client requests generating inter-node communication in the +Dynamo ring, a node A quickly discovers that a node B is unresponsive when B +fails to respond to a message; Node A then uses alternate nodes to service +requests that map to B's partitions; A periodically retries B to check for the +latter's recovery. In the absence of client requests to drive traffic between +two nodes, neither node really needs to know whether the other is reachable and +responsive. + +Decentralized failure detection protocols use a simple gossip-style protocol +that enable each node in the system to learn about the arrival (or departure) of +other nodes. For detailed information on decentralized failure detectors and the +parameters affecting their accuracy, the interested reader is referred to [8]. +Early designs of Dynamo used a decentralized failure detector to maintain a +globally consistent view of failure state. Later it was determined that the +explicit node join and leave methods obviates the need for a global view of +failure state. This is because nodes are notified of permanent node additions +and removals by the explicit node join and leave methods and temporary node +failures are detected by the individual nodes when they fail to communicate with +others (while forwarding requests). + +> Riak KV follows the same mechanism, by manually triggering permanent ring +> state changes, and gossiping the new state. + + +### 4.9 Adding/Removing Storage Nodes + +When a new node (say X) is added into the system, it gets assigned a number of +tokens that are randomly scattered on the ring. For every key range that is +assigned to node X, there may be a number of nodes (less than or equal to N) +that are currently in charge of handling keys that fall within its token range. +Due to the allocation of key ranges to X, some existing nodes no longer have to +some of their keys and these nodes transfer those keys to X. Let us consider a +simple bootstrapping scenario where node X is added to the ring shown in +<a href="#figure-2">Figure 2</a> between A and B. When X is added to the system, +it is in charge of storing keys in the ranges (F, G], (G, A] and (A, X]. As a +consequence, nodes B, C and D no longer have to store the keys in these +respective ranges. Therefore, nodes B, C, and D will offer to and upon +confirmation from X transfer the appropriate set of keys. When a node is removed +from the system, the reallocation of keys happens in a reverse process. + +> Riak KV does not randomly assign vnodes, but rather, iterates through the list +> of partitions, assigning them to nodes in a round-robin style. + +Operational experience has shown that this approach distributes the load of key +distribution uniformly across the storage nodes, which is important to meet the +latency requirements and to ensure fast bootstrapping. Finally, by adding a +confirmation round between the source and the destination, it is made sure that +the destination node does not receive any duplicate transfers for a given key +range. + + +## 5.Implementation + +In Dynamo, each storage node has three main software components: request +coordination, membership and failure detection, and a local persistence engine. +All these components are implemented in Java. + +> Riak KV is implemented in Erlang. Request coordination and membership behavior +> is defined by [riak_core] and implemented by [Riak KV]. + +[riak_core]: http://github.com/basho/riak_core +[Riak KV]: http://github.com/basho/riak_kv + +Dynamo’s local persistence component allows for different storage engines to be +plugged in. Engines that are in use are Berkeley Database (BDB) Transactional +Data Store, BDB Java Edition, MySQL, and an in-memory buffer with persistent +backing store. The main reason for designing a pluggable persistence component +is to choose the storage engine best suited for an application’s access +patterns. For instance, BDB can handle objects typically in the order of tens of +kilobytes whereas MySQL can handle objects of larger sizes. Applications choose +Dynamo’s local persistence engine based on their object size distribution. The +majority of Dynamo’s production instances use BDB Transactional Data Store. + +> Riak KV ships with various [backend options]. [Bitcask] is the default, but +> [LevelDB] and Main [Memory] are also used heavily in production (in that +> order). You can also use more than one backend in production via the [[Multi]] +> backend configuration. +> +> Bitcask is a fast and reliable choice, but does have some limitations at very +> large scales. For larger clusters, you may want to choose LevelDB (which also +> supports [secondary indexes]). The Memory backend is an excellent choice when +> speed is important and durability is not. It also has TTL support. + +[backend options]: {{<baseurl>}}riak/kv/2.2.6/setup/planning/backend/ +[Bitcask]: {{<baseurl>}}riak/kv/2.2.6/setup/planning/backend/bitcask/ +[LevelDB]: {{<baseurl>}}riak/kv/2.2.6/setup/planning/backend/leveldb/ +[Memory]: {{<baseurl>}}riak/kv/2.2.6/setup/planning/backend/memory/ +[secondary indexes]: {{<baseurl>}}riak/kv/2.2.6/developing/usage/secondary-indexes/ + +The request coordination component is built on top of an event-driven messaging +substrate where the message processing pipeline is split into multiple stages +similar to the SEDA architecture [24]. All communications are implemented using +Java NIO channels. The coordinator executes the read and write requests on +behalf of clients by collecting data from one or more nodes (in the case of +reads) or storing data at one or more nodes (for writes). Each client request +results in the creation of a state machine on the node that received the client +request. The state machine contains all the logic for identifying the nodes +responsible for a key, sending the requests, waiting for responses, potentially +doing retries, processing the replies and packaging the response to the client. +Each state machine instance handles exactly one client request. For instance, a +read operation implements the following state machine: (i) send read requests to +the nodes, (ii) wait for minimum number of required responses, (iii) if too few +replies were received within a given time bound, fail the request, (iv) +otherwise gather all the data versions and determine the ones to be returned and +(v) if versioning is enabled, perform syntactic reconciliation and generate an +opaque write context that contains the vector clock that subsumes all the +remaining versions. For the sake of brevity the failure handling and retry +states are left out. + +> Request coordination in Riak KV uses Erlang message passing, but follows a +> similar state machine. + +After the read response has been returned to the caller the state machine waits +for a small period of time to receive any outstanding responses. If stale +versions were returned in any of the responses, the coordinator updates those +nodes with the latest version. This process is called read repair because it +repairs replicas that have missed a recent update at an opportunistic time and +relieves the anti-entropy protocol from having to do it. + +> Riak KV implements [Read Repair]. + +[Read Repair]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/replication/#read-repair + +As noted earlier, write requests are coordinated by one of the top N nodes in +the preference list. Although it is desirable always to have the first node +among the top N to coordinate the writes thereby serializing all writes at a +single location, this approach has led to uneven load distribution resulting in +SLA violations. This is because the request load is not uniformly distributed +across objects. To counter this, any of the top N nodes in the preference list +is allowed to coordinate the writes. In particular, since each write usually +follows a read operation, the coordinator for a write is chosen to be the node +that replied fastest to the previous read operation which is stored in the +context information of the request. This optimization enables us to pick the +node that has the data that was read by the preceding read operation thereby +increasing the chances of getting “read-your-writes” consistency. It also +reduces variability in the performance of the request handling which improves +the performance at the 99.9 percentile. + + +## 6. Experiences & Lessons Learned + +> Much of this section relates to benchmarks run against Dynamo. You can run +> [Basho Bench] against your own Riak cluster to discover your own +> optimal values. + +[Basho Bench]: {{<baseurl>}}riak/kv/2.2.6/using/performance/benchmarking/ + +Dynamo is used by several services with different configurations. These +instances differ by their version reconciliation logic, and read/write quorum +characteristics. The following are the main patterns in which Dynamo is used: + +* Business logic specific reconciliation: This is a popular use case for Dynamo. +Each data object is replicated across multiple nodes. In case of divergent +versions, the client application performs its own reconciliation logic. The +shopping cart service discussed earlier is a prime example of this category. Its +business logic reconciles objects by merging different versions of a customer’s +shopping cart. + +> Riak KV currently supports simple conflict resolution by way of read-repair, +> remanding more complex reconciliation to the client. There are several tools +> to help simplify this task, such as [Statebox]. +> +> Riak KV supports a simple reconciliation strategy, called [CRDTs] (Commutative +> Replicated Data Types), for reconciling common data types like sets and +> counters. + +[Statebox]: https://github.com/mochi/statebox_riak +[CRDTs]: {{<baseurl>}}riak/kv/2.2.6/developing/data-types/ + + +* Timestamp based reconciliation: This case differs from the previous one only +in the reconciliation mechanism. In case of divergent versions, Dynamo performs +simple timestamp based reconciliation logic of “last write wins”; i.e., the +object with the largest physical timestamp value is chosen as the correct +version. The service that maintains customer’s session information is a good +example of a service that uses this mode. + +> Riak also supports this for high-performance cases where accuracy is less +> important than speed. + +* High performance read engine: While Dynamo is built to be an “always +writeable” data store, a few services are tuning its quorum characteristics and +using it as a high performance read engine. Typically, these services have a +high read request rate and only a small number of updates. In this +configuration, typically R is set to be 1 and W to be N. For these services, +Dynamo provides the ability to partition and replicate their data across +multiple nodes thereby offering incremental scalability. Some of these instances +function as the authoritative persistence cache for data stored in more heavy +weight backing stores. Services that maintain product catalog and promotional +items fit in this category. + +> Riak can be used in this manner. + +The main advantage of Dynamo is that its client applications can tune the values +of N, R and W to achieve their desired levels of performance, availability and +durability. For instance, the value of N determines the durability of each +object. A typical value of N used by Dynamo’s users is 3. + +The values of W and R impact object availability, durability and consistency. +For instance, if W is set to 1, then the system will never reject a write +request as long as there is at least one node in the system that can +successfully process a write request. However, low values of W and R can +increase the risk of inconsistency as write requests are deemed successful and +returned to the clients even if they are not processed by a majority of the +replicas. This also introduces a vulnerability window for durability when a +write request is successfully returned to the client even though it has been +persisted at only a small number of nodes. + +Traditional wisdom holds that durability and availability go hand-in-hand. +However, this is not necessarily true here. For instance, the vulnerability +window for durability can be decreased by increasing W. This may increase the +probability of rejecting requests (thereby decreasing availability) because more +storage hosts need to be alive to process a write request. + +The common (N,R,W) configuration used by several instances of Dynamo is (3,2,2). +These values are chosen to meet the necessary levels of performance, durability, +consistency, and availability SLAs. + +All the measurements presented in this section were taken on a live system +operating with a configuration of (3,2,2) and running a couple hundred nodes +with homogenous hardware configurations. As mentioned earlier, each instance of +Dynamo contains nodes that are located in multiple datacenters. These +datacenters are typically connected through high speed network links. Recall +that to generate a successful get (or put) response R (or W) nodes need to +respond to the coordinator. Clearly, the network latencies between datacenters +affect the response time and the nodes (and their datacenter locations) are +chosen such that the applications target SLAs are met. + +> Ditto for Riak. + +### 6.1 Balancing Performance and Durability + +While Dynamo’s principle design goal is to build a highly available data store, +performance is an equally important criterion in Amazon’s platform. As noted +earlier, to provide a consistent customer experience, Amazon’s services set +their performance targets at higher percentiles (such as the 99.9th or 99.99th +percentiles). A typical SLA required of services that use Dynamo is that 99.9% +of the read and write requests execute within 300ms. + +Since Dynamo is run on standard commodity hardware components that have far less +I/O throughput than high-end enterprise servers, providing consistently high +performance for read and write operations is a non-trivial task. The involvement +of multiple storage nodes in read and write operations makes it even more +challenging, since the performance of these operations is limited by the slowest +of the R or W replicas. <a href="#figure-4">Figure 4</a> shows the average and +99.9th percentile latencies of Dynamo’s read and write operations during a +period of 30 days. As seen in the figure, the latencies exhibit a clear diurnal +pattern which is a result of the diurnal pattern in the incoming request rate +(i.e., there is a significant difference in request rate between the daytime and +night). Moreover, the write latencies are higher than read latencies obviously +because write operations always results in disk access. Also, the 99.9th +percentile latencies are around 200 ms and are an order of magnitude higher than +the averages. This is because the 99.9th percentile latencies are affected by +several factors such as variability in request load, object sizes, and locality +patterns. + +**<figure id="figure-4" style="text-align:center;"> + <img src="/riak-docs/images/dynamo/figure4.png"> + <figcaption> + Figure 4: Average and 99.9 percentiles of latencies for read and write + requests during our peak request season of December 2006. The intervals + between consecutive ticks in the x-axis correspond to 12 hours. Latencies + follow a diurnal pattern similar to the request rate and 99.9 percentile + latencies are an order of magnitude higher than averages. + </figcaption> +</figure>** + +While this level of performance is acceptable for a number of services, a few +customer-facing services required higher levels of performance. For these +services, Dynamo provides the ability to trade-off durability guarantees for +performance. In the optimization each storage node maintains an object buffer in +its main memory. Each write operation is stored in the buffer and gets +periodically written to storage by a writer thread. In this scheme, read +operations first check if the requested key is present in the buffer. If so, the +object is read from the buffer instead of the storage engine. + +> This is more similar to Riak's W value, since only DW requires a durable write +> to respond as a success. + +This optimization has resulted in lowering the 99.9th percentile latency by a +factor of 5 during peak traffic even for a very small buffer of a thousand +objects (see <a href="#figure-5">Figure 5</a>). Also, as seen in the figure, +write buffering smoothes out higher percentile latencies. Obviously, this scheme +trades durability for performance. In this scheme, a server crash can result in +missing writes that were queued up in the buffer. To reduce the durability risk, +the write operation is refined to have the coordinator choose one out of the N +replicas to perform a “durable write”. Since the coordinator waits only for W +responses, the performance of the write operation is not affected by the +performance of the durable write operation performed by a single replica. + +**<figure id="figure-5" style="text-align:center;"> + <img src="/riak-docs/images/dynamo/figure5.png"> + <figcaption> + Figure 5: Comparison of performance of 99.9th percentile latencies for + buffered vs. non-buffered writes over a period of 24 hours. The intervals + between consecutive ticks in the x-axis correspond to one hour. + </figcaption> +</figure>** + +> Setting DW=1 will replicate this behavior. + + +### 6.2 Ensuring Uniform Load distribution + +Dynamo uses consistent hashing to partition its key space across its replicas +and to ensure uniform load distribution. A uniform key distribution can help us +achieve uniform load distribution assuming the access distribution of keys is +not highly skewed. In particular, Dynamo’s design assumes that even where there +is a significant skew in the access distribution there are enough keys in the +popular end of the distribution so that the load of handling popular keys can be +spread across the nodes uniformly through partitioning. This section discusses +the load imbalance seen in Dynamo and the impact of different partitioning +strategies on load distribution. + +> Riak follows a SHA1 based consistent hashing for [partitioning]. + +[partitioning]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/replication/#understanding-replication-by-example + +To study the load imbalance and its correlation with request load, the total +number of requests received by each node was measured for a period of 24 hours - +broken down into intervals of 30 minutes. In a given time window, a node is +considered to be “in-balance”, if the node’s request load deviates from the +average load by a value a less than a certain threshold (here 15%). Otherwise +the node was deemed “out-of-balance”. <a href="#figure-6">Figure 6</a> presents +the fraction of nodes that are “out-of-balance” (henceforth, “imbalance ratio”) +during this time period. For reference, the corresponding request load received +by the entire system during this time period is also plotted. As seen in the +figure, the imbalance ratio decreases with increasing load. For instance, during +low loads the imbalance ratio is as high as 20% and during high loads it is +close to 10%. Intuitively, this can be explained by the fact that under high +loads, a large number of popular keys are accessed and due to uniform +distribution of keys the load is evenly distributed. However, during low loads +(where load is 1/8th of the measured peak load), fewer popular keys are +accessed, resulting in a higher load imbalance. + +**<figure id="figure-6" style="text-align:center;"> + <img src="/riak-docs/images/dynamo/figure6.png"> + <figcaption> + Figure 6: Fraction of nodes that are out-of-balance (i.e., nodes whose + request load is above a certain threshold from the average system load) and + their corresponding request load. The interval between ticks in x-axis + corresponds to a time period of 30 minutes. + </figcaption> +</figure>** + +<i>This section discusses how Dynamo’s partitioning scheme has evolved over time +and its implications on load distribution.</i> + +<strong>Strategy 1:</strong> T random tokens per node and partition by token +value: This was the initial strategy deployed in production (and described in +Section 4.2). In this scheme, each node is assigned T tokens (chosen uniformly +at random from the hash space). The tokens of all nodes are ordered according to +their values in the hash space. Every two consecutive tokens define a range. The +last token and the first token form a range that "wraps" around from the highest +value to the lowest value in the hash space. Because the tokens are chosen +randomly, the ranges vary in size. As nodes join and leave the system, the token +set changes and consequently the ranges change. Note that the space needed to +maintain the membership at each node increases linearly with the number of nodes +in the system. + +> Riak uses equal sized partitions with a round-robin distribution--not a +> variably-sized partitions that are randomly distributed. + +While using this strategy, the following problems were encountered. First, when +a new node joins the system, it needs to “steal” its key ranges from other +nodes. However, the nodes handing the key ranges off to the new node have to +scan their local persistence store to retrieve the appropriate set of data +items. Note that performing such a scan operation on a production node is tricky +as scans are highly resource intensive operations and they need to be executed +in the background without affecting the customer performance. This requires us +to run the bootstrapping task at the lowest priority. However, this +significantly slows the bootstrapping process and during busy shopping season, +when the nodes are handling millions of requests a day, the bootstrapping has +taken almost a day to complete. Second, when a node joins/leaves the system, the +key ranges handled by many nodes change and the Merkle trees for the new ranges +need to be recalculated, which is a non-trivial operation to perform on a +production system. Finally, there was no easy way to take a snapshot of the +entire key space due to the randomness in key ranges, and this made the process +of archival complicated. In this scheme, archiving the entire key space requires +us to retrieve the keys from each node separately, which is highly inefficient. + +The fundamental issue with this strategy is that the schemes for data +partitioning and data placement are intertwined. For instance, in some cases, it +is preferred to add more nodes to the system in order to handle an increase in +request load. However, in this scenario, it is not possible to add nodes without +affecting data partitioning. Ideally, it is desirable to use independent schemes +for partitioning and placement. To this end, following strategies were +evaluated: + +<strong>Strategy 2:</strong> T random tokens per node and equal sized +partitions: In this strategy, the hash space is divided into Q equally sized +partitions/ranges and each node is assigned T random tokens. Q is usually set +such that Q >> N and Q >> S*T, where S is the number of nodes in the system. In +this strategy, the tokens are only used to build the function that maps values +in the hash space to the ordered lists of nodes and not to decide the +partitioning. A partition is placed on the first N unique nodes that are +encountered while walking the consistent hashing ring clockwise from the end of +the partition. <a href="#figure-7">Figure 7</a> illustrates this strategy for +N=3. In this example, nodes A, B, C are encountered while walking the ring from +the end of the partition that contains key k1. The primary advantages of this +strategy are: (i) decoupling of partitioning and partition placement, and (ii) +enabling the possibility of changing the placement scheme at runtime. + +> As before mentioned, Riak uses equal sized partitions, but not +> random distribution. + +**<figure id="figure-7" style="text-align:center;"> + <img src="/riak-docs/images/dynamo/figure7-small.png"> + <figcaption> + Figure 7: Partitioning and placement of keys in the three strategies. A, B, + and C depict the three unique nodes that form the preference list for the + key k1 on the consistent hashing ring (N=3). The shaded area indicates the + key range for which nodes A, B, and C form the preference list. Dark arrows + indicate the token locations for various nodes. + </figcaption> +</figure>** + +<strong>Strategy 3:</strong> Q/S tokens per node, equal-sized partitions: +Similar to strategy 2, this strategy divides the hash space into Q equally sized +partitions and the placement of partition is decoupled from the partitioning +scheme. Moreover, each node is assigned Q/S tokens where S is the number of +nodes in the system. When a node leaves the system, its tokens are randomly +distributed to the remaining nodes such that these properties are preserved. +Similarly, when a node joins the system it "steals" tokens from nodes in the +system in a way that preserves these properties. + +> Riak most closely follows strategy 3. +> +> See [The Node Join Process] and [Replacing a Node]. + +[The Node Join Process]: {{<baseurl>}}riak/kv/2.2.6/using/cluster-operations/adding-removing-nodes/#joining-nodes-to-form-a-cluster +[Replacing a Node]: {{<baseurl>}}riak/kv/2.2.6/using/cluster-operations/replacing-node/ + +The efficiency of these three strategies is evaluated for a system with S=30 and +N=3. However, comparing these different strategies in a fair manner is hard as +different strategies have different configurations to tune their efficiency. For +instance, the load distribution property of strategy 1 depends on the number of +tokens (i.e., T) while strategy 3 depends on the number of partitions (i.e., Q). +One fair way to compare these strategies is to evaluate the skew in their load +distribution while all strategies use the same amount of space to maintain their +membership information. For instance, in strategy 1 each node needs to maintain +the token positions of all the nodes in the ring and in strategy 3 each node +needs to maintain the information regarding the partitions assigned to each +node. + +In our next experiment, these strategies were evaluated by varying the relevant +parameters (T and Q). The load balancing efficiency of each strategy was +measured for different sizes of membership information that needs to be +maintained at each node, where Load balancing efficiency is defined as the ratio +of average number of requests served by each node to the maximum number of +requests served by the hottest node. + +The results are given in <a href="#figure-8">Figure 8</a>. As seen in the +figure, strategy 3 achieves the best load balancing efficiency and strategy 2 +has the worst load balancing efficiency. For a brief time, Strategy 2 served as +an interim setup during the process of migrating Dynamo instances from using +Strategy 1 to Strategy 3. Compared to Strategy 1, Strategy 3 achieves better +efficiency and reduces the size of membership information maintained at each +node by three orders of magnitude. While storage is not a major issue the nodes +gossip the membership information periodically and as such it is desirable to +keep this information as compact as possible. In addition to this, strategy 3 is +advantageous and simpler to deploy for the following reasons: (i) Faster +bootstrapping/recovery: Since partition ranges are fixed, they can be stored in +separate files, meaning a partition can be relocated as a unit by simply +transferring the file (avoiding random accesses needed to locate specific +items). This simplifies the process of bootstrapping and recovery. (ii) Ease of +archival: Periodical archiving of the dataset is a mandatory requirement for +most of Amazon storage services. Archiving the entire dataset stored by Dynamo +is simpler in strategy 3 because the partition files can be archived separately. +By contrast, in Strategy 1, the tokens are chosen randomly and, archiving the +data stored in Dynamo requires retrieving the keys from individual nodes +separately and is usually inefficient and slow. The disadvantage of strategy 3 +is that changing the node membership requires coordination in order to preserve +the properties required of the assignment. + +**<figure id="figure-8" style="text-align:center;"> + <img src="/riak-docs/images/dynamo/figure8.png"> + <figcaption> + Figure 8: Comparison of the load distribution efficiency of different + strategies for system with 30 nodes and N=3 with equal amount of metadata + maintained at each node. The values of the system size and number of + replicas are based on the typical configuration deployed for majority of + our services. + </figcaption> +</figure>** + +### 6.3 Divergent Versions: When and How Many? + +As noted earlier, Dynamo is designed to tradeoff consistency for availability. +To understand the precise impact of different failures on consistency, detailed +data is required on multiple factors: outage length, type of failure, component +reliability, workload etc. Presenting these numbers in detail is outside of the +scope of this paper. However, this section discusses a good summary metric: the +number of divergent versions seen by the application in a live production +environment. + +> This first statement should be read carefully. It's probably more correct to +> say that Dynamo (and Riak) provides no consistency guarantees, and allows +> users to trade availability for durability/latency. + +Divergent versions of a data item arise in two scenarios. The first is when the +system is facing failure scenarios such as node failures, data center failures, +and network partitions. The second is when the system is handling a large number +of concurrent writers to a single data item and multiple nodes end up +coordinating the updates concurrently. From both a usability and efficiency +perspective, it is preferred to keep the number of divergent versions at any +given time as low as possible. If the versions cannot be syntactically +reconciled based on vector clocks alone, they have to be passed to the business +logic for semantic reconciliation. Semantic reconciliation introduces additional +load on services, so it is desirable to minimize the need for it. + +In our next experiment, the number of versions returned to the shopping cart +service was profiled for a period of 24 hours. During this period, 99.94% of +requests saw exactly one version; 0.00057% of requests saw 2 versions; 0.00047% +of requests saw 3 versions and 0.00009% of requests saw 4 versions. This shows +that divergent versions are created rarely. + +Experience shows that the increase in the number of divergent versions is +contributed not by failures but due to the increase in number of concurrent +writers. The increase in the number of concurrent writes is usually triggered by +busy robots (automated client programs) and rarely by humans. This issue is not +discussed in detail due to the sensitive nature of the story. + +### 6.4 Client-driven or Server-driven Coordination + +As mentioned in Section 5, Dynamo has a request coordination component that uses +a state machine to handle incoming requests. Client requests are uniformly +assigned to nodes in the ring by a load balancer. Any Dynamo node can act as a +coordinator for a read request. Write requests on the other hand will be +coordinated by a node in the key’s current preference list. This restriction is +due to the fact that these preferred nodes have the added responsibility of +creating a new version stamp that causally subsumes the version that has been +updated by the write request. Note that if Dynamo’s versioning scheme is based +on physical timestamps, any node can coordinate a write request. + +> In Riak, a server-side load-balancer is an optional configuration. You +> generally use either virtual IPs or reverse-proxies. +> +> See [Load Balancing] for more information. + +[Load Balancing]: {{<baseurl>}}riak/kv/2.2.6/configuring/load-balancing-proxy/ + +An alternative approach to request coordination is to move the state machine to +the client nodes. In this scheme client applications use a library to perform +request coordination locally. A client periodically picks a random Dynamo node +and downloads its current view of Dynamo membership state. Using this +information the client can determine which set of nodes form the preference list +for any given key. Read requests can be coordinated at the client node thereby +avoiding the extra network hop that is incurred if the request were assigned to +a random Dynamo node by the load balancer. Writes will either be forwarded to a +node in the key’s preference list or can be coordinated locally if Dynamo is +using timestamps based versioning. + +> Many [client libraries] provide built-in node request coordination. +> +> For example, using the Ruby driver, you could specify three nodes like this: +> +> client = Riak::Client.new(nodes: [ +> {host: '10.0.0.1'}, +> {host: '10.0.0.2'}, +> {host: '10.0.0.3'} +> ]) +> +> Note that the Riak clients do not coordinate with Riak's preference list, but +> simply round-robin requests, letting the Riak cluster handle routing. + +[client libraries]: {{<baseurl>}}riak/kv/2.2.6/developing/client-libraries/ + +An important advantage of the client-driven coordination approach is that a load +balancer is no longer required to uniformly distribute client load. Fair load +distribution is implicitly guaranteed by the near uniform assignment of keys to +the storage nodes. Obviously, the efficiency of this scheme is dependent on how +fresh the membership information is at the client. Currently clients poll a +random Dynamo node every 10 seconds for membership updates. A pull based +approach was chosen over a push based one as the former scales better with large +number of clients and requires very little state to be maintained at servers +regarding clients. However, in the worst case the client can be exposed to stale +membership for duration of 10 seconds. In case, if the client detects its +membership table is stale (for instance, when some members are unreachable), it +will immediately refresh its membership information. + +<a href="#table-2">Table 2</a> shows the latency improvements at the 99.9th +percentile and averages that were observed for a period of 24 hours using +client-driven coordination compared to the server-driven approach. As seen in +the table, the client-driven coordination approach reduces the latencies by at +least 30 milliseconds for 99.9th percentile latencies and decreases the average +by 3 to 4 milliseconds. The latency improvement is because the client-driven +approach eliminates the overhead of the load balancer and the extra network hop +that may be incurred when a request is assigned to a random node. As seen in the +table, average latencies tend to be significantly lower than latencies at the +99.9th percentile. This is because Dynamo’s storage engine caches and write +buffer have good hit ratios. Moreover, since the load balancers and network +introduce additional variability to the response time, the gain in response time +is higher for the 99.9th percentile than the average. + +<table id="table-2"> + <caption> + Table 2: Performance of client-driven and server-driven + coordination approaches. + </caption> + <tr> + <th></th> + <th>99.9th percentile read latency (ms)</th> + <th>99.9th percentile write latency (ms)</th> + <th>Average read latency (ms)</th> + <th>Average write latency (ms)</th> + </tr> + <tr> + <th>Server-driven</th> + <td>68.9</td> + <td>68.5</td> + <td>3.9</td> + <td>4.02</td> + </tr> + <tr> + <th>Client-driven</th> + <td>30.4</td> + <td>30.4</td> + <td>1.55</td> + <td>1.9</td> + </tr> +</table> + +### 6.5 Balancing background vs. foreground tasks + +Each node performs different kinds of background tasks for replica +synchronization and data handoff (either due to hinting or adding/removing +nodes) in addition to its normal foreground put/get operations. In early +production settings, these background tasks triggered the problem of resource +contention and affected the performance of the regular put and get operations. +Hence, it became necessary to ensure that background tasks ran only when the +regular critical operations are not affected significantly. To this end, the +background tasks were integrated with an admission control mechanism. Each of +the background tasks uses this controller to reserve runtime slices of the +resource (e.g. database), shared across all background tasks. A feedback +mechanism based on the monitored performance of the foreground tasks is employed +to change the number of slices that are available to the background tasks. + +> Riak does this, too. For example, hinted handoff runs in the background at a +> low level, so as not to overwhelm a cluster when nodes are added/removed. + +The admission controller constantly monitors the behavior of resource accesses +while executing a "foreground" put/get operation. Monitored aspects include +latencies for disk operations, failed database accesses due to lock-contention +and transaction timeouts, and request queue wait times. This information is used +to check whether the percentiles of latencies (or failures) in a given trailing +time window are close to a desired threshold. For example, the background +controller checks to see how close the 99th percentile database read latency +(over the last 60 seconds) is to a preset threshold (say 50ms). The controller +uses such comparisons to assess the resource availability for the foreground +operations. Subsequently, it decides on how many time slices will be available +to background tasks, thereby using the feedback loop to limit the intrusiveness +of the background activities. Note that a similar problem of managing background +tasks has been studied in [4]. + +### 6.6 Discussion + +This section summarizes some of the experiences gained during the process of +implementation and maintenance of Dynamo. Many Amazon internal services have +used Dynamo for the past two years and it has provided significant levels of +availability to its applications. In particular, applications have received +successful responses (without timing out) for 99.9995% of its requests and no +data loss event has occurred to date. + +Moreover, the primary advantage of Dynamo is that it provides the necessary +knobs using the three parameters of (N,R,W) to tune their instance based on +their needs.. Unlike popular commercial data stores, Dynamo exposes data +consistency and reconciliation logic issues to the developers. At the outset, +one may expect the application logic to become more complex. However, +historically, Amazon’s platform is built for high availability and many +applications are designed to handle different failure modes and inconsistencies +that may arise. Hence, porting such applications to use Dynamo was a relatively +simple task. For new applications that want to use Dynamo, some analysis is +required during the initial stages of the development to pick the right conflict +resolution mechanisms that meet the business case appropriately. Finally, Dynamo +adopts a full membership model where each node is aware of the data hosted by +its peers. To do this, each node actively gossips the full routing table with +other nodes in the system. This model works well for a system that contains +couple of hundreds of nodes. However, scaling such a design to run with tens of +thousands of nodes is not trivial because the overhead in maintaining the +routing table increases with the system size. This limitation might be overcome +by introducing hierarchical extensions to Dynamo. Also, note that this problem +is actively addressed by O(1) DHT systems(e.g., [14]). + +> This is equally true for Riak. As mentioned above, consider running +> [Basho Bench] to help discover your optimal setup. Nothing will give you +> better numbers than real experimentation. + +[Basho Bench]: {{<baseurl>}}riak/kv/2.2.6/using/performance/benchmarking/ + +## 7. Conclusions + +> This paper was an overview of Riak from a Dynamo point-of-view. To get a +> better sense of the Riak ecosystem, read our ever-expanding [documentation]. + +[documentation]: https://docs.basho.com + +This paper described Dynamo, a highly available and scalable data store, used +for storing state of a number of core services of Amazon.com’s e-commerce +platform. Dynamo has provided the desired levels of availability and performance +and has been successful in handling server failures, data center failures and +network partitions. Dynamo is incrementally scalable and allows service owners +to scale up and down based on their current request load. Dynamo allows service +owners to customize their storage system to meet their desired performance, +durability and consistency SLAs by allowing them to tune the parameters N, R, +and W. + +The production use of Dynamo for the past year demonstrates that decentralized +techniques can be combined to provide a single highly-available system. Its +success in one of the most challenging application environments shows that an +eventual-consistent storage system can be a building block for highly-available +applications. diff --git a/content/riak/kv/2.2.6/learn/glossary.md b/content/riak/kv/2.2.6/learn/glossary.md new file mode 100644 index 0000000000..223de4d965 --- /dev/null +++ b/content/riak/kv/2.2.6/learn/glossary.md @@ -0,0 +1,353 @@ +--- +title: "Riak KV Glossary" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Riak KV Glossary" + identifier: "learn_glossary" + weight: 103 + parent: "learn" +toc: true +--- + + +[apps replication properties]: {{<baseurl>}}riak/kv/2.2.6/developing/app-guide/replication-properties +[basho bench GH]: http://github.com/basho/basho_bench/ +[cluster ops add remove node]: {{<baseurl>}}riak/kv/2.2.6/using/cluster-operations/adding-removing-nodes +[cluster ops strong consistency]: {{<baseurl>}}riak/kv/2.2.6/using/cluster-operations/strong-consistency +[concept buckets]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/buckets +[concept causal context vc]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/causal-context/#vector-clocks +[concept clusters]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/clusters +[concept crdts]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/crdts +[concept eventual consistency]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/eventual-consistency +[concept keys objects]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/keys-and-objects +[concept replication]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/replication +[concept strong consistency]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/strong-consistency +[dev kv model]: {{<baseurl>}}riak/kv/2.2.6/developing/key-value-modeling +[concept replication aae]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/replication/#active-anti-entropy +[dev api http]: {{<baseurl>}}riak/kv/2.2.6/developing/api/http +[dev data model]: {{<baseurl>}}riak/kv/2.2.6/developing/data-modeling +[dev data types]: {{<baseurl>}}riak/kv/2.2.6/developing/data-types +[glossary read rep]: {{<baseurl>}}riak/kv/2.2.6/learn/glossary/#read-repair +[glossary vnode]: {{<baseurl>}}riak/kv/2.2.6/learn/glossary/#vnode +[Lager]: https://github.com/basho/lager +[learn dynamo]: {{<baseurl>}}riak/kv/2.2.6/learn/dynamo +[plan cluster capacity]: {{<baseurl>}}riak/kv/2.2.6/setup/planning/cluster-capacity +[repair recover failure recovery]: {{<baseurl>}}riak/kv/2.2.6/using/repair-recovery/failure-recovery +[repair recover repairs]: {{<baseurl>}}riak/kv/2.2.6/using/repair-recovery/repairs +[Riak Core]: https://github.com/basho/riak_core +[Riak KV]: https://github.com/basho/riak_kv +[Riak Pipe]: https://github.com/basho/riak_pipe +[Riak Pipe - the New MapReduce Power]: http://basho.com/riak-pipe-the-new-mapreduce-power/ +[Riak Pipe - Riak's Distributed Processing Framework]: http://vimeo.com/53910999 +[Understanding Riak's Configurable Behaviors]: http://basho.com/riaks-config-behaviors-part-2/ +[usage mapreduce]: {{<baseurl>}}riak/kv/2.2.6/developing/usage/mapreduce +[usage search]: {{<baseurl>}}riak/kv/2.2.6/developing/usage/search +[usage secondary-indexes]: {{<baseurl>}}riak/kv/2.2.6/developing/usage/secondary-indexes +[Where To Start With Riak Core]: http://basho.com/where-to-start-with-riak-core/ +[Wikipedia:Consistent Hashing]: http://en.wikipedia.org/wiki/Consistent_hashing + + +Below is a list of terms that you may run into frequently in the +documentation for Riak, along with links to more in-depth treatments. + +## Active Anti-Entropy (AAE) + +A continuous background process that compares and repairs any divergent, +missing, or corrupted replicas. Unlike [read +repair][glossary read rep], which is only triggered when data is +read, the Active Anti-Entropy system ensures the integrity of all data +stored in Riak. This is particularly useful in clusters containing “cold +data,” i.e. data that may not be read for long periods of time, +potentially years. Furthermore, unlike the repair command, Active +Anti-Entropy is an automatic process requiring no user intervention. It +is enabled by default in Riak 1.3 and greater. + +* [Replication][concept replication aae] + +## Basho Bench + +Basho Bench is a benchmarking tool created to conduct accurate and +repeatable performance tests and stress tests and to produce performance +graphs. + +* [Basho Bench]({{<baseurl>}}riak/kv/2.2.6/using/performance/benchmarking) +* [GitHub repository][basho bench GH] + +## Bucket + +A bucket is a namespace for data stored in Riak, with a set of common +properties for its contents, e.g. the number of replicas (`n_val`), +whether siblings are returned on reads (`allow_mult`), etc. Buckets' +properties are determined by their bucket type (see below). + +* [Buckets][concept buckets] +* [HTTP Bucket Operations][dev api http] + +## Bucket Type + +Bucket types enable you to create and manage sets of bucket properties +that, when applied to buckets, dictate those buckets' behavior. They +also act as a third namespace in Riak in addition to buckets and keys. + +* [Bucket Types]({{<baseurl>}}riak/kv/2.2.6/developing/usage/bucket-types) + +## Cluster + +A Riak cluster is a 160-bit integer space which is divided into +equally-sized partitions. Each vnode in the Riak Ring is responsible for +one of these partitions. + +* [Clusters][concept clusters] +* [Dynamo][learn dynamo] + +## Consistent Hashing + +Consistent hashing is a technique used to limit the reshuffling of keys +when a hash-table data structure is rebalanced (i.e. when slots are +added or removed). Riak uses consistent hashing to organize its data +storage and replication. Specifically, the vnodes in the Riak Ring +responsible for storing each object are determined using the consistent +hashing technique. + +* [Clusters][concept clusters] +* [Dynamo][learn dynamo] +* [Wikipedia:Consistent Hashing] + +## Data Types + +Riak Data Types are data objects inspired by research on +[CRDTs](http://hal.upmc.fr/docs/00/55/55/88/PDF/techreport.pdf) that use +certain rules of convergence to dictate how conflicts between replicas +are resolved in Riak's eventually consistent system. There are five Riak +Data Types in total: flags, registers, counters, sets, and maps. + +* [Data Types Concept][concept crdts] +* [Using Data Types][dev data types] +* [Data Modeling with Riak Data Types][dev data model] + +## Eventual Consistency + +A consistency model that informally guarantees that if no new updates +are made to a given data item, all reads on that item will eventually +return the last updated value. Details about what this means in Riak can +be found in the document below. + +* [Eventual Consistency][concept eventual consistency] + +## Gossiping + +Riak uses a "gossip protocol" to share and communicate ring state and +bucket properties around the cluster. Whenever a node changes its claim +on the ring, it announces its change via this protocol. Each node also +periodically sends its current view of the ring state to a randomly +selected peer in case any nodes missed previous updates. + +* [Clusters][concept clusters] +* [Adding and Removing Nodes][cluster ops add remove node] + +## Hinted Handoff + +Hinted handoff is a technique for dealing with node failure in the Riak +cluster in which neighboring nodes temporarily take over storage +operations for the failed node. When the failed node returns to the +cluster, the updates received by the neighboring nodes are handed off to +it. + +Hinted handoff allows Riak to ensure database availability. When a node +fails, Riak can continue to handle requests as if the node were still +there. + +* [Recovering a Failed Node][repair recover failure recovery] + +## Key + +Keys are unique object identifiers in Riak and are scoped within buckets +and bucket types. + +* [Keys and Objects][concept keys objects] +* [Key/Value Development][dev kv model] + +## Lager + +[Lager] is an Erlang/OTP framework that +ships as Riak's default logger. + +## MapReduce + +Riak's MapReduce gives developers the capability to perform more +powerful queries over the data stored in their key/value data. + +* [Using MapReduce][usage mapreduce] + +## Node + +A node is analogous to a physical server. Nodes run a certain number of +vnodes, each of which claims a partition in the Riak Ring key space. + +* [Clusters][concept clusters] +* [Adding and Removing Nodes][cluster ops add remove node] + +## Object + +An object is another name for a value. + +* [Keys and Objects][concept keys objects] +* [Key/Value Development][dev kv model] + +## Partition + +Partitions are the spaces into which a Riak cluster is divided. Each +vnode in Riak is responsible for a partition. Data is stored on a set +number of partitions determined by the `n_val` setting, with the target +partitions chosen statically by applying consistent hashing to an +object's key. + +* [Clusters][concept clusters] +* [Eventual Consistency][concept eventual consistency] +* [Cluster Capacity Planning][plan cluster capacity] + +## Quorum + +Quorum in Riak has two meanings: + +* The quantity of replicas that must respond to a read or write request + before it is considered successful. This is defined as a bucket + property or as one of the relevant parameters to a single request + (R,W,DW,RW). +* A symbolic quantity for the above, `quorum`, which is equivalent to + `n_val` / 2 + 1. The default setting is `2`. + +* [Eventual Consistency][concept eventual consistency] +* [Replication properties][apps replication properties] +* [Understanding Riak's Configurable Behaviors] + +## Sloppy Quorum + +During failure scenarios, in which available nodes < total nodes, sloppy +quorum is used to ensure that Riak is still available to take writes. +When a primary node is unavailable, another node will accept its write +requests. When the node returns, data is transferred to the primary node +via the [Hinted Handoff](#hinted-handoff) process. + +## Read Repair + +Read repair is an anti-entropy mechanism that Riak uses to +optimistically update stale replicas when they reply to a read request +with stale data. + +* [More about Read Repair][concept replication] + +## Replica + +Replicas are copies of data stored in Riak. The number of replicas +required for both successful reads and writes is configurable in Riak +and should be set based on your application's consistency and +availability requirements. + +* [Eventual Consistency][concept eventual consistency] +* [Understanding Riak's Configurable Behaviors] + +## Riak Core + +Riak Core is the modular distributed systems framework that serves as +the foundation for Riak's scalable architecture. + +* [Riak Core] +* [Where To Start With Riak Core] + +## Riak KV + +Riak KV is the key/value datastore for Riak. + +* [Riak KV] + +## Riak Pipe + +Riak Pipe is the processing layer that powers Riak's MapReduce. It's +best described as "UNIX pipes for Riak." + +* [Riak Pipe] +* [Riak Pipe - the New MapReduce Power] +* [Riak Pipe - Riak's Distributed Processing Framework] + +## Riak Search + +Riak Search is a distributed, scalable, failure-tolerant, realtime, +full-text search engine integrating [Apache +Solr](https://lucene.apache.org/solr/) with Riak KV. + +* [Using Search][usage search] + +## Ring + +The Riak Ring is a 160-bit integer space. This space is equally divided +into partitions, each of which is claimed by a vnode, which themselves +reside on actual physical server nodes. + +* [Clusters][concept clusters] +* [Dynamo][learn dynamo] +* [Cluster Capacity Planning][plan cluster capacity] + +## Secondary Indexing (2i) + +Secondary Indexing in Riak gives developers the ability to tag an object +stored in Riak with one or more values which can then be queried. + +* [Using Secondary Indexes][usage secondary-indexes] +* [Repairing Indexes][repair recover repairs] + +## Strong Consistency + +While Riak is most well known as an [eventually consistent][concept eventual consistency] data storage system, versions of Riak 2.0 and greater +enable you to apply strong consistency guarantees to some or all of your +data, thus using Riak as a CP (consistent plus partition-tolerant) +rather than AP (highly available plus partition-tolerant) system. + +* [Strong Consistency Concept][concept strong consistency] +* [Using Strong Consistency][cluster ops strong consistency] + +## Value + +Riak is best described as a key/value store. In versions of Riak prior +to 2.0, all "values" are opaque BLOBs (binary large objects) identified +with a unique key. Values can be any type of data, including a string, a +JSON object, a text document, etc. Modifying values involves fetching +the value that exists in Riak and substituting it for a new value; +operations on values are thus basic CRUD operations. + +[Riak Data Types][dev data types], added in version 2.0, are an important +exception to this. While still considered values---because they are +stored in bucket type/bucket/key locations, like anything in Riak---Riak +Data Types are not BLOBs and are modified by Data Type-specific +operations. + +* [Keys and Objects][concept keys objects] +* [Key/Value Development][dev kv model] +* [Data Types][dev data types] + + +## Vector Clock + +Riak utilizes vector clocks (or _vclocks_) to handle version control. +Since any node in a Riak cluster is able to handle a request, and not +all nodes need to participate, data versioning is required to keep track +of a current value. When a value is stored in Riak, it is tagged with a +vector clock and establishes the initial version. When it is updated, +the client provides the vector clock of the object being modified so +that this vector clock can be extended to reflect the update. Riak can +then compare vector clocks on different versions of the object and +determine certain attributes of the data. + +* [Vector clocks][concept causal context vc] + +## Vnode + +Vnodes, or "virtual nodes," are responsible for claiming a partition in +the Riak Ring, and they coordinate requests for these partitions. + +* [vnodes][glossary vnode] +* [Clusters][concept clusters] +* [Dynamo][learn dynamo] diff --git a/content/riak/kv/2.2.6/learn/new-to-nosql.md b/content/riak/kv/2.2.6/learn/new-to-nosql.md new file mode 100644 index 0000000000..dbcf217557 --- /dev/null +++ b/content/riak/kv/2.2.6/learn/new-to-nosql.md @@ -0,0 +1,16 @@ +--- +draft: true +title: "New to NoSQL?" +description: "" +project: "riak_kv" +project_version: "2.2.6" +#menu: +# riak_kv-2.2.6: +# name: "New to NoSQL?" +# identifier: "learn_new_nosql" +# weight: 102 +# parent: "learn" +toc: true +--- + +**TODO: Add content (not sure where this lives in existing docs)** diff --git a/content/riak/kv/2.2.6/learn/use-cases.md b/content/riak/kv/2.2.6/learn/use-cases.md new file mode 100644 index 0000000000..765ad1a59b --- /dev/null +++ b/content/riak/kv/2.2.6/learn/use-cases.md @@ -0,0 +1,401 @@ +--- +title: "Use Cases For Riak KV" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Use Cases" + identifier: "learn_use_cases" + weight: 101 + parent: "learn" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/data-modeling/ + - /riak-docs/riak/kv/2.2.6/dev/data-modeling/ +--- + + +[dev data model articles etc]: {{<baseurl>}}riak/kv/2.2.6/developing/data-modeling/#articles-blog-posts-and-other-content +[dev data model log data]: {{<baseurl>}}riak/kv/2.2.6/developing/data-modeling/#log-data +[dev data model sensor data]: {{<baseurl>}}riak/kv/2.2.6/developing/data-modeling/#sensor-data +[dev data model serve advertisements]: {{<baseurl>}}riak/kv/2.2.6/developing/data-modeling/#serving-advertisements +[dev data model sess storage]: {{<baseurl>}}riak/kv/2.2.6/developing/data-modeling/#session-storage +[dev data model user acct]: {{<baseurl>}}riak/kv/2.2.6/developing/data-modeling/#user-accounts +[dev data model user events]: {{<baseurl>}}riak/kv/2.2.6/developing/data-modeling/#user-events-and-timelines +[dev data model user settings]: {{<baseurl>}}riak/kv/2.2.6/developing/data-modeling/#user-settings-and-preferences +[dev data types]: {{<baseurl>}}riak/kv/2.2.6/developing/data-types +[plan backend bitcask]: {{<baseurl>}}riak/kv/2.2.6/setup/planning/backend/bitcask +[replication properties]: {{<baseurl>}}riak/kv/2.2.6/developing/app-guide/replication-properties +[usage mapreduce]: {{<baseurl>}}riak/kv/2.2.6/developing/usage/mapreduce +[usage search]: {{<baseurl>}}riak/kv/2.2.6/developing/usage/search +[usage secondary-indexes]: {{<baseurl>}}riak/kv/2.2.6/developing/usage/secondary-indexes + +Riak is a flexible data storage technology capable of addressing a wide variety +of problems in a scalable way. In this guide, we'll list a number of use cases +and data models that are a good fit for Riak. All of these use cases are already +being used in production for projects large and small. We'll also suggest +possibilities for implementation and provide links to videos and documentation +for further exploration. + +How you structure your application to run on Riak should take into account the +unique needs of your use case, including access patterns such as read/write +distribution, latency differences between various operations, use of Riak +features including [Data Types][dev data types], [MapReduce][usage mapreduce], +[Search][usage search], [secondary indexes (2i)][usage secondary-indexes], and +more. This guide is intended to be illustrative only. + +## High Read/Write, Simple Applications + +The following are examples of Riak use cases that require high read/write +performance without necessarily utilizing complex data structures: + +* [Session Storage][dev data model sess storage] +* [Serving Advertisements][dev data model serve advertisements] +* [Log Data][dev data model log data] +* [Sensor Data][dev data model sensor data] + +## Content Management, Social Applications + +The following application types require more subtle relationships between +objects, e.g. one-to-many and many-to-many relationships. + +* [User Accounts][dev data model user acct] +* [User Settings and Preferences][dev data model user settings] +* [User Events and Timelines][dev data model user events] +* [Articles, Blog Posts, and Other Content][dev data model articles etc] + +## Session Storage + +Riak was originally created to serve as a highly scalable session store. This is +an ideal use case for Riak, which is always most performant and predictable when +used as a key/value store. Since user and session IDs are usually stored in +cookies or otherwise known at lookup time, Riak is able to serve these requests +with predictably low latency. Riak's content-type agnosticism also imposes no +restrictions on the value, so session data can be encoded in many ways and can +evolve without administrative changes to schemas. + +### Complex Session Storage Case + +Riak has features that allow for more complex session storage use cases. The +[Bitcask][plan backend bitcask] storage backend, for example, supports automatic +expiry of keys, which frees application developers from implementing manual +session expiry. Riak's [MapReduce][usage mapreduce] system can also be used to +perform batch processing analysis on large bodies of session data, for example +to compute the average number of active users. If sessions must be retrieved +using multiple keys (e.g. a UUID or email address), +[using secondary indexes][usage secondary-indexes] can provide an easy solution. + +### Session Storage Community Examples + +<table class="use-cases__image-links"> + <tr> + <td> + <a href="https://player.vimeo.com/video/42744689" target="_blank" title="Scaling Riak at Kiip"> + <img src="http://b.vimeocdn.com/ts/296/624/29662.2.6_960.jpg"/> + </a> + </td> + <td> + <a href="https://player.vimeo.com/video/42744689" target="_blank" title="Riak at OpenX">Scaling Riak at Kiip</a> + <br> + In this talk, recorded at the May 2012 San Francisco Riak Meetup, Armon + Dadgar and Mitchell Hashimoto of Kiip give an overview of how and why they + are using Riak in production, and the road they took to get there. One of + the first subsystems they switched over to Riak was Sessions. You can also + read the blog post and catch the slides <a + href="http://basho.com/posts/business/Scaling-Riak-At-Kiip/" class="riak" + target="_blank">here.</a> + </td> + </tr> +</table> + +## Serving Advertisements + +Riak is often a good choice for serving advertising content to many different +web and mobile users simultaneously with low latency. Content of this sort, e.g. +images or text, can be stored in Riak using unique generated either by the +application or by Riak. Keys can be created based on, for example, a campaign or +company ID for easy retrieval. + +### Serving Advertisements Complex Case + +In the advertising industry, being able to serve ads quickly to many users and +platforms is often the most important factor in selecting and tuning a database. +Riak's tunable [apps replication properties][replication properties] can be set +to favor fast read performance. By setting R to 1, only one of N replicas will +need to be returned to complete a read operation, yielding lower read latency +than an R value equal to the number of replicas (i.e. R=N). This is ideal for +advertising traffic, which primarily involves serving reads. + +### Serving Advertisements Community Examples + +<table class="use-cases__image-links"> + <tr> + <td> + <a href="http://player.vimeo.com/video/49775483" target="_blank" title="Riak at OpenX"> + <img src="http://b.vimeocdn.com/ts/343/417/343417336_960.jpg"/> + </a> + </td> + <td> + <a href="http://player.vimeo.com/video/49775483" target="_blank" title="Riak at OpenX">Riak at OpenX</a> + <br> + Los Angeles-based OpenX will serves trillions of ads a year. In this talk, + Anthony Molinaro, Engineer at OpenX, goes in depth on their architecture, + how they've built their system, and why/how they're switching to Riak for + data storage after using databases like CouchDB and Cassandra in + production. + </td> + </tr> +</table> + +## Log Data + +A common use case for Riak is storing large amounts of log data, either for +analysis [using MapReduce][usage secondary-indexes] or as a storage system used +in conjunction with a secondary analytics cluster used to perform more advanced +analytics tasks. To store log data, you can use a bucket called `logs` (just to +give an example) and use a unique value, such as a date, for the key. Log files +would then be the values associated with each unique key. + +For storing log data from different systems, you could use unique buckets for +each system (e.g. `system1_log_data`, `system2_log_data`, etc.) and write +associated logs to the corresponding buckets. To analyze that data, you could +use Riak's MapReduce system for aggregation tasks, such as summing the counts of +records for a date or Riak Search for a more robust, text-based queries. + +### Log Data Complex Case + +For storing a large amount of log data that is frequently written to Riak, some +users might consider doing primary storage of logs in a Riak cluster and then +replicating data to a secondary cluster to run heavy analytics jobs, either over +another Riak cluster or another solution such as Hadoop. Because the access +patterns of reading and writing data to Riak is very different from the access +pattern of something like a MapReduce job, which iterates over many keys, +separating the write workload from the analytics workload will let you maintain +higher performance and yield more predictable latency. + +### Log Data Community Examples + +<table class="use-cases__image-links"> + <tr> + <td> + <a href="http://www.simonbuckle.com/2011/08/27/analyzing-apache-logs-with-riak/" target="_blank" title="Riak at OpenX"> + <img src="/riak-docs/images/simon-analyzing-logs.png"/> + </a> + </td> + <td> + Simon Buckle on <a href="http://www.simonbuckle.com/2011/08/27/analyzing-apache-logs-with-riak/" target="_blank">analyzing Apache logs with Riak.</a> + </td> + </tr> +</table> + +## Sensor Data + +Riak's scalable design makes it useful for data sets, like sensor data, that +scale rapidly and are subject to heavy read/write loads. Many sensors collect +and send data at a given interval. One way to model this in Riak is to use a +bucket for each sensor device and use the time interval as a unique key (i.e. a +date or combination of date and time), and then store update data as the value. + +That data could then be queried on the basis of the interval. Alternatively, a +timestamp could be attached to each object as a +[secondary index][usage secondary-indexes], which would allow you to perform +queries on specific time interval ranges or to perform +[MapReduce][usage mapreduce] queries against the indexes. + +### Sensor Data Complex Case + +If you are dealing with thousands or millions of sensors yet with very small +data sets, storing all of a single device's updates as unique keys may be +cumbersome when it comes to reading that device's data. Retrieving it all would +mean calling a number of keys. + +Instead, you could store all of a device's updates in a document with a unique +key to identify the device. Stored as a JSON document, you could read and parse +all of those updates on the client side. Riak, however, doesn't allow you to +append data to a document without reading the object and writing it back to the +key. This strategy would mean more simplicity and performance on the read side +as a tradeoff for slightly more work at write time and on the client side. + +It's also important to keep an eye out for the total size of documents as they +grow, as we tend to recommend that Riak objects stay smaller than 1-2 MB and +preferably below 100 KB. Otherwise, performance problems in the cluster are +likely. + +## User Accounts + +User accounts tend to rely on fairly straightforward data models. One way of +storing user account data in Riak would be store each user's data as a JSON +object in a bucket called `users` (or whatever you wish). Keys for user data +objects could be constructed using application-specific considerations. If your +application involves user logins, for example, the simplest and most read- +efficient strategy would be to use the login username as the object key. The +username could be extracted upon login, and a read request could be performed on +the corresponding key. + +There are, however, several drawbacks to this approach. What happens if a user +wants to change their username later on? The most common solution would be to +use a UUID-type key for the user and store the user's username as a +[secondary index][usage secondary-indexes] for efficient lookup. + +### User Accounts Complex Case + +For simple retrieval of a specific account, a user ID (plus perhaps a secondary +index on a username or email) is enough. If you foresee the need to make queries +on additional user attributes (e.g. creation time, user type, or region), plan +ahead and either set up additional secondary indexes or consider using +[Riak Search][usage search] to index the JSON contents of the user account. + +### User Accounts Community Examples + +<table class="use-cases__image-links"> + <tr> + <td> + <a href="https://player.vimeo.com/video/47535803" target="_blank" title="Riak at Braintree"> + <img class="vid_img"src="http://b.vimeocdn.com/ts/329/711/329711886_640.jpg"/> + </a> + </td> + <td> + <a href="https://player.vimeo.com/video/47535803" target="_blank" title="Riak at Braintree">Riak at Braintree</a> + <br> + Ben Mills, a developer at Braintree, discusses how their backend team came + to find and begin to integrate Riak into their production environment. + They also cover their model and repository framework for Ruby, Curator. + Check out more details and slides on the + <a href="http://basho.com/posts/business/riak-at-braintree/" target="_blank">Riak blog.</a> + </td> + </tr> +</table> + +## User Settings and Preferences + +For user account-related data that is simple and frequently read but rarely +changed (such as a privacy setting or theme preference), consider storing it in +the user object itself. Another common pattern is to create a companion user +settings-type of object, with keys based on user ID for easy one-read retrieval. + +### User Settings and Preferences Complex Case + +If you find your application frequently writing to the user account or have +dynamically growing user-related data such as bookmarks, subscriptions, or +multiple notifications, then a more advanced data model may be called for. + +## User Events and Timelines + +Sometimes you may want to do more complex or specific kinds of modeling user +data. A common example would be storing data for assembling a social network +timeline. To create a user timeline, you could use a `timeline` bucket in Riak +and form keys on the basis of a unique user ID. You would store timeline +information as the value, e.g. a list of status update IDs which could then be +used to retrieve the full information from another bucket, or perhaps containing +the full status update. If you want to store additional data, such as a +timestamp, category or list of properties, you can turn the list into an array +of hashes containing this additional information. + +Note than in Riak you cannot append information to an object, so adding events +in the timeline would necessarily involve reading the full object, modifying it, +and writing back the new value. + +### User Events and Timelines Community Examples + +<table class="use-cases__image-links"> + <tr> + <td> + <a href="http://player.vimeo.com/video/21598799" target="_blank" title="Riak at Yammer"> + <img src="http://b.vimeocdn.com/ts/139/033/139033664_640.jpg"/> + </a> + </td> + <td> + <a href="http://player.vimeo.com/video/21598799" target="_blank" title="Riak at Yammer">Riak at Yammer</a> + <br> + This video was recorded at the March 2012 San Francisco Riak Meetup and is + worth every minute of your time. Coda Hale and Ryan Kennedy of Yammer give + an excellent and in depth look into how they built “Streamie”, user + notifications, why Riak was the right choice, and the lessons learned in + the process. Read more and get the slides in the Riak blog + <a href="http://basho.com/posts/business/Riak-And-Scala-At-Yammer/" target="_blank">here.</a> + </td> + </tr> + <tr> + <td> + <a href="http://player.vimeo.com/video/44498491" target="_blank" title="Riak at Voxer"> + <img src="http://b.vimeocdn.com/ts/309/154/309154350_960.jpg"/> + </a> + </td> + <td> + <a href="http://player.vimeo.com/video/44498491" target="_blank" title="Riak at Voxer">Riak at Voxer</a> + <br> + The team at Voxer has long relied on Riak as their primary data store for + various production services. They have put Riak through its paces and have + served as one of our more exciting customers and use cases: Riak was in + place when they shot to the top of the App Store at the end of 2011. We + also love them because they open-sourced their Node.js client. Read more + and get the slides in the Riak blog + <a href="http://basho.com/posts/business/Riak-in-Production-at-Voxer/" target="_blank">here.</a> + </td> + </tr> +</table> + +## Articles, Blog Posts, and Other Content + +The simplest way to model blog posts, articles, or similar content is to use a +bucket in Riak with some unique attribute for logical division of content, such +as `blogs` or `articles`. Keys could be constructed out of unique identifiers +for posts, perhaps the title of each article, a combination of the title and +data/time, an integer that can be used as part of a URL string, etc. + +In Riak, you can store content of any kind, from HTML files to plain text to +JSON or XML or another document type entirely. Keep in mind that data in Riak is +opaque, with the exception of [Riak Data Types][dev data types], and so Riak +won't "know" about the object unless it is indexed +[using Riak Search][usage search] or +[using secondary indexes][usage secondary-indexes]. + +### Articles et al Complex Case + +Setting up a data model for content becomes more complex based on the querying +and search requirements of your application. For example, you may have different +kinds of content that you want to generate in a view, e.g. not just a post but +also comments, user profile information, etc. + +For many Riak developers, it will make sense to divide content into different +buckets, e.g. a bucket for comments that would be stored in the Riak cluster +along with the posts bucket. Comments for a given post could be stored as a +document with the same key as the content post, though with a different +bucket/key combination. Another possibility would be to store each comment with +its own ID. Loading the full view with comments would require your application +to call from the posts and comments buckets to assemble the view. + +Other possible cases may involve performing operations on content beyond +key/value pairs. [Riak Search][usage search] is recommended for use cases +involving full-text search. For lighter-weight querying, +[using secondary indexes][usage secondary-indexes] \(2i) enables you to add +metadata to objects to either query for exact matches or to perform range +queries. 2i also enables you to tag posts with dates, timestamps, topic areas, +or other pieces of information useful for later retrieval. + +### Articles et al Community Examples + +<table class="use-cases__image-links"> + <tr> + <td> + <a href="http://media.basho.com/pdf/Linkfluence-Case-Study-v2-1.pdf" class="vid_img" link target="_blank"> + <img src="/riak-docs/images/linkfluence-case-study.png" title="Milking Performance"> + </a> + </td> + <td> + Linkfluence case study on using Riak to <a href="http://media.basho.com/pdf/Linkfluence-Case-Study-v2-1.pdf" target="_blank">store social web content</a>. + </td> + </tr> + <tr> + <td> + <a href="http://basho.com/assets/Basho-Case-Study-ideeli.pdf" class="vid_img" link target="_blank"> + <img src="/riak-docs/images/ideeli-case-study.png" title="Milking Performance"> + </a> + </td> + <td> + ideeli case study on <a href="http://basho.com/assets/Basho-Case-Study-ideeli.pdf" target="_blank">serving web pages with Riak</a>. + </td> + </tr> +</table> + diff --git a/content/riak/kv/2.2.6/learn/why-riak-kv.md b/content/riak/kv/2.2.6/learn/why-riak-kv.md new file mode 100644 index 0000000000..ee8883ee16 --- /dev/null +++ b/content/riak/kv/2.2.6/learn/why-riak-kv.md @@ -0,0 +1,221 @@ +--- +title: "Why Riak KV?" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Why Riak KV?" + identifier: "learn_why_riak_kv" + weight: 100 + parent: "learn" +toc: true +aliases: + - /riak-docs/riak/2.2.6/theory/why-riak/ + - /riak-docs/riak/kv/2.2.6/theory/why-riak/ +--- + + +[apps replication properties]: {{<baseurl>}}riak/kv/2.2.6/developing/app-guide/replication-properties +[Basho Bench]: {{<baseurl>}}riak/kv/2.2.6/using/performance/benchmarking +[cluster ops strong consistency]: {{<baseurl>}}riak/kv/2.2.6/using/cluster-operations/strong-consistency +[concept eventual consistency]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/eventual-consistency +[convergent replicated data types]: http://hal.upmc.fr/docs/00/55/55/88/PDF/techreport.pdf +[Datomic]: http://www.datomic.com/overview.html +[dev data types]: {{<baseurl>}}riak/kv/2.2.6/developing/data-types +[glossary read rep]: {{<baseurl>}}riak/kv/2.2.6/learn/glossary/#read-repair + + +## What is Riak? + +Riak is a distributed database designed to deliver maximum data +availability by distributing data across multiple servers. As long as +your Riak client can reach *one* Riak server, it should be able to write +data. + +Riak is used as an **eventually consistent** system in that the data you want to read should remain available in most failure scenarios, although it may not be the most up-to-date version of that data. + + +### Basho's goals for Riak + +Goal | Description +-------|------- +**Availability** | Riak writes to and reads from multiple servers to offer data availability even when hardware or the network itself are experiencing failure conditions +**Operational simplicity** | Easily add new machines to your Riak cluster without incurring a larger operational burden +**Scalability** | Riak automatically distributes data around the cluster and yields a near-linear performance increase as you add capacity +**Masterless** | Your requests are not held hostage to a specific server in the cluster that may or may not be available + +### When Riak makes sense + +If your data does not fit on a single server and demands a distributed +database architecture, you should take a close look at Riak as a +potential solution to your data availability issues. Getting distributed +databases right is **very** difficult, and Riak was built to address the +problem of data availability with as few trade-offs and downsides as +possible. + +Riak's focus on availability makes it a good fit whenever downtime is +unacceptable. No one can promise 100% uptime, but Riak is designed to +survive network partitions and hardware failures that would +significantly disrupt most databases. + +A less-heralded feature of Riak is its predictable latency. Because its +fundamental operations---read, write, and delete---do not involve +complex data joins or locks, it services those requests promptly. Thanks +to this capability, Riak is often selected as a data storage backend for +data management software from a variety of paradigms, such as +[Datomic]. + +From the standpoint of the actual content of your data, Riak might also +be a good choice if your data can be modeled as one of Riak's currently +available [Data Types][dev data types]: flags, registers, counters, +sets, or maps. These Data Types enable you to take advantage of Riak's +high availability approach while simplifying application development. + +### When Riak is Less of a Good Fit + +We recommend running no fewer than 5 data servers in a cluster. +This means that Riak can be overkill for small databases. If you're not +already sure that you will need a distributed database, there's a good +chance that you won't need Riak. + +If explosive growth is a possibility, however, you are always highly +advised to prepare for that in advance. Scaling at Internet speeds is +sometimes compared to overhauling an airplane mid-flight. If you feel +that such a transition might be necessary in the future, then you might +want to consider Riak. + +Riak's simple data model, consisting of keys and values as its atomic +elements, means that your data must be denormalized if your system is to +be reasonably performant. For most applications this is not a serious +hurdle. But if your data simply cannot be effectively managed as keys +and values, Riak will most likely not be the best fit for you. + +Correspondingly, if your application demands a high query load by any +means other than key/value lookup---e.g. SQL-style `SELECT * FROM table` +operations---Riak will not be as efficient as other databases. If you +wish to compare Riak with other data technologies, Basho offers a tool +called [Basho Bench] to help measure its performance, so that you can +decide whether the availability and operational benefits of Riak +outweigh its disadvantages. + +## How Does a Riak Cluster Work? + +A Riak cluster is a group of **nodes** that are in constant +communication to ensure data availability and partition tolerance. + +### What is a Riak Node? + +A Riak node is not quite the same as a server, but in a production +environment the two should be equivalent. A developer may run multiple +nodes on a single laptop, but this would never be advisable in a real +production cluster. + +Each node in a Riak cluster is equivalent, containing a complete, +independent copy of the whole Riak package. There is no "master" node; +no node has more responsibilities than others; and no node has special +tasks not performed by other nodes. This uniformity provides the basis +for Riak's fault tolerance and scalability. + +Each node is responsible for multiple data partitions, as discussed +below: + +### Riak Automatically Re-Distributes Data When Capacity is Added + +When you add (or remove) machines, data is rebalanced automatically with +no downtime. New machines claim data until ownership is equally spread +around the cluster, with the resulting cluster status updates shared to +every node via a gossip protocol and used to route requests. This is +what makes it possible for any node in the cluster to receive requests. +The end result is that developers don't need to deal with the underlying +complexity of where data lives. + +### Consistent Hashing + +Data is distributed across nodes using consistent hashing. Consistent +hashing ensures that data is evenly distributed around the cluster and +makes possible the automatic redistribution of data as the cluster +scales. + +### Intelligent Replication + +Riak's replication scheme ensures that you can still read, write, and +update data if nodes go down. Riak allows you to set a replication +variable, N (also known as the `n_val`), that specifies the number of +nodes on which a value will be replicated. + +An `n_val` value of 3 (the default) means that each object is replicated +3 times. When an object's key is mapped onto a given node, Riak will +continue on and automatically replicate the data onto two more nodes. +This parameter enables you to replicate values to 7 nodes in a 10-node +cluster, 10 nodes in a 15-node cluster, and so on. + +## When Things Go Wrong + +Riak retains fault tolerance, data integrity, and availability even in +failure conditions such as hardware failure and network partitions. Riak +has a number of means of addressing these scenarios and other bumps in +the road, like version conflicts in data. + +### Hinted Handoff + +Hinted handoff enables Riak to handle node failure. If a node goes down, +a neighboring node will take over its storage operations. When the +failed node returns, the updates received by the neighboring node are +handed back to it. This ensures that availability for writes and updates +is maintained automatically, minimizing the operational burden of +failure conditions. + +### Version Conflicts + +In any system that replicates data, conflicts can arise, for example +when two clients update the same object at the exact same time or when +not all updates have yet reached hardware that is experiencing lag. + +In Riak, replicas are [eventually consistent][concept eventual consistency], +meaning that while data is always available, not all replicas may have +the most recent update at the exact same time, causing brief +periods---generally on the order of milliseconds---of inconsistency +while all state changes are synchronized. + +Riak addresses data conflicts as follows: When you make a read request, +Riak looks up all replicas for that object. By default, Riak will return +the most recently updated version, determined by looking at the object's +vector clock. Vector clocks are metadata attached to each replica when +it is created. They are extended each time a replica is updated to keep +track of versions. You can also allow clients to resolve conflicts +themselves if that is a better fit for your use case. + +### Riak Data Types + +If you are not interested in dealing with version conflicts on the +application side, [Riak Data Types][dev data types] offer a powerful +yet easy-to-use means of storing certain types of data while allowing +Riak to handle merge conflicts. These conflicts are resolved +automatically by Riak using Data Type-specific algorithms inspired by +research into [convergent replicated data types]. + +### Read Repair + +When an outdated replica is returned as part of a read request, Riak +will automatically update the out-of-sync replica to make it consistent. +[Read repair][glossary read rep], a self-healing property of +the database, will even update a replica that returns a `not_found` in +the event that a node loses the data due to physical failure. + +### Reading and Writing Data in Failure Conditions + +In Riak, you can set an R value for reads and a W value for writes. +These values give you control over how many replicas must respond to a +request for it to succeed. + +Let's say that you have an N value of 3 (aka `n_val=3`) for a particular +key/value pair, but one of the physical nodes responsible for a replica +is down. With an `r=2` setting, only 2 replicas must return results for +read to be deemed successful. This allows Riak to provide read +availability even when nodes are down or laggy. The same applies for the +W in writes. If this value is not specified, Riak defaults to `quorum`, +according to which the majority of nodes must respond. + +There is more on [replication properties][apps replication properties] elsewhere in the +documentation. diff --git a/content/riak/kv/2.2.6/release-notes.md b/content/riak/kv/2.2.6/release-notes.md new file mode 100644 index 0000000000..a7d480d11b --- /dev/null +++ b/content/riak/kv/2.2.6/release-notes.md @@ -0,0 +1,192 @@ +--- +title: "Riak KV 2.2.6 Release Notes" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Release Notes" + identifier: "index_release_notes" + weight: 101 + parent: index +toc: false +aliases: + - /riak-docs/riak/2.2.6/community/release-notes + - /riak-docs/riak/kv/2.2.6/intro-v20 + - /riak-docs/riak/2.2.6/intro-v20 + - /riak-docs/riak/kv/2.2.6/introduction +--- + +Released April 25, 2018. + +> This release is dedicated to the memory of Andy Gross. Thank you and RIP. + +## Overview + +This is the first full community release of Riak, post-Basho's +collapse into bankruptcy. A lot has happened, in particular [bet365](https://twitter.com/bet365Tech) bought Basho's +assets and donated the code to the community. They kept the Basho +website running, the mailing list, the documents site (after [TI Tokyo](https://www.tiot.jp/) +had helpfully mirrored the docs in the interim) and have done a huge amount to +provide continuity to the community. + +The development work on this release of Riak has received significant +funding from [NHS Digital](https://twitter.com/NHSDigital), who depend on Riak for Spine II, and other +critical services. Thanks also to [ESL](https://twitter.com/ErlangSolutions), [TI Tokyo](https://www.tiot.jp/), and all the other +individuals and organisations involved. + +This release of Riak is based on the last known-good release of Riak, +riak-2.2.3. There is good work in the `develop` branches of many Basho +repos, but since much of it was unfinished, unreleased, untested, or +just status-unknown, we decided as a community to go forward based on +riak-2.2.3. + +This is the first release with open source multi-data-centre +replication. The rest of the changes are fixes ([riak-core claim](#core-claim-fixes), +repl), new features ([gsets](#gsets), [participate in coverage](#participate-in-2i), [node-confirms](#node-confirms)), +and [fixes to tests](#developer-improvements) and the build/development process. + +[Improvements](#improvements) + +[Known Issues](#known-issues) - please read **before upgrading** from a previous Riak release + +[Log of Changes](#change-log-for-this-release) + +[Previous Release Notes](#previous-release-notes) + +## Improvements + +#### Multi Datacentre Replication + +Previously a paid for enterprise addition to Riak as part of the Riak +EE product, this release includes Multi-Datacentre Replication +(MDC). There is no longer a Riak EE product. All is now Open +Source. Please consult the existing documentation for +[MDC]({{<baseurl>}}riak/kv/2.2.3/configuring/v3-multi-datacenter/). Again, +many thanks to bet365 Technology for this addition. See also +[Known Issues](#known-issues) below. + +#### Core Claim Fixes + +Prior to this release, in some scenarios, multiple partitions from the +same preflist could be stored on the same node, potentially leading to +data loss. [This write up](https://github.com/basho/riak_core/blob/c9c924ef006af1121b7eec04c7e1eefe54f4cf26/docs/claim-fixes.md) +explains the fixes in detail, and links to +[another post](https://github.com/infinityworks/riak_core/blob/ada7030a2b2c3463d6584f1d8b20e2c4bc5ac3d8/docs/ring_claim.md) +that gives a deep examination of riak-core-ring and the issues fixed +in this release. + +#### Node Confirms + +This feature adds a new bucket property, and write-option of +`node_confirms`. Unlike `w` and `pw` that are tunables for +consistency, `node_confirms` is a tunable for durability. When +operating in a failure state, Riak will store replicas in fallback +vnodes, and in some case multiple fallbacks may be on the same +physical node. `node_confirms` is an option that specifies how many +distinct physical nodes must acknowledge a write for it to be +considered successful. There is a +[detailed write up here](https://github.com/ramensen/riak_kv/blob/30b0e50374196d9a8cfef37871955a5f5b2bb472/docs/Node-Diversity.md), +and more in the documentation. + +#### Participate In 2i + +This feature was added to bring greater consistency to 2i query +results. When a node has just been joined to a riak cluster it may not +have any, or at least up-to-date, data. However the joined node is +immediately in the ring and able to take part in coverage queries, +which can lead to incomplete results. This change adds an operator +flag to a node's configuration that will exclude it from coverage +plans. When all transfers are complete, the operator can remove the +flag. See documentation for more details. + +#### GSets + +This release adds another Riak Data Type, the GSet CRDT. The GSet is a +grow only set, and has simpler semantics and better merge performance +than the existing Riak Set. See documentation for details. + +#### Developer Improvements + +The tests didn't pass. Now they do. More details +[here](https://github.com/russelldb/russelldb.github.io/blob/b228eacd4fd3246b4eb7f8d0b98c6bed747e2514/make_test.md) + +## Known Issues + +#### Advanced.config changes + +With the inclusion of Multi-Datacentre Replication in riak-2.2.6 there +are additional `advanced.config` parameters. If you have an existing +`advanced.config` you must merge it with the new one from the install +of riak-2.2.6. Some package installs will simply replace the old with +new (e.g. .deb), others may leave the old file unchanged. YOU MUST +make sure that the `advanced.config` contains valid `riak_repl` +entries. + +Example default entries to add to your existing advanced.config: + +``` +{riak_core, + [ + {cluster_mgr, {"0.0.0.0", 9080 } } + ]}, + {riak_repl, + [ + {data_root, "/var/lib/riak/riak_repl/"}, + {max_fssource_cluster, 5}, + {max_fssource_node, 1}, + {max_fssink_node, 1}, + {fullsync_on_connect, true}, + {fullsync_interval, 30}, + {rtq_max_bytes, 104857600}, + {proxy_get, disabled}, + {rt_heartbeat_interval, 15}, + {rt_heartbeat_timeout, 15}, + {fullsync_use_background_manager, true} + ]}, +``` + +Read more about configuring +[MDC]({{<baseurl>}}riak/kv/2.2.3/configuring/v3-multi-datacenter/) +replication. + +More details about the issue can be found in riak\_repl/782: [2.2.6 - \[enoent\] - riak_repl couldn't create log dir +"data/riak_repl/logs"](https://github.com/basho/riak/issues/940) + +## Change Log for This Release + +* riak_pipe/113: [Some flappy test failure fixes](https://github.com/basho/riak_pipe/pull/113) +* riak_kv/1657: [Intermittent test failure fixes](https://github.com/basho/riak_kv/pull/1657) +* riak_kv/1658: [ Move schedule_timeout to execute in put_fsm](https://github.com/basho/riak_kv/pull/1658) +* riak_kv/1663: [Add bucket property `node_confirms` for physical diversity](https://github.com/basho/riak_kv/pull/1663) +* riak_kv/1664: [Add option 'participate_in_2i_coverage' with default 'enabled'](https://github.com/basho/riak_kv/pull/1664) +* riak_kv/1665: [enable gset support](https://github.com/basho/riak_kv/pull/1665) +* riak_kv/1666: [Fix schema paths for make test](https://github.com/basho/riak_kv/pull/1666) +* eleveldb/243: [Add a 10% fuzz factor to the resident memory calc (intermittent test failure "fixes")](https://github.com/basho/eleveldb/pull/243) +* riak_core/911: [Fix brops intermittent test failures](https://github.com/basho/riak_core/pull/911) +* riak_core/913: [ Fix claim tail violations and unbalanced rings](https://github.com/basho/riak_core/pull/913) +* riak_core/915: [Add `node_confirms` default bucket props](https://github.com/basho/riak_core/pull/915) +* riak_core/917: [Add participate_in_2i_coverage riak option](https://github.com/basho/riak_core/pull/917) +* sidejob/18: [Address some intermittent test failures](https://github.com/basho/sidejob/pull/18) +* riak_pb/228: [Add `node_confirms` option to write messages](https://github.com/basho/riak_pb/pull/228) +* riak_pb/229: [add gsets support](https://github.com/basho/riak_pb/pull/229) +* basho_stats/13: [Non-deterministic test needs a little ?SOMETIMES](https://github.com/basho/basho_stats/pull/13) +* basho_stats/4: [Add Makefile](https://github.com/basho/basho_stats/pull/4) +* exometer_core/17: [Fix failing test with correct tuple entry](https://github.com/basho/exometer_core/pull/17) +* yokozuna/741: [Fix broken eqc test](https://github.com/basho/yokozuna/pull/741) +* yokozuna/746: [remove -XX:+UseStringCache](https://github.com/basho/yokozuna/pull/746) +* yokozuna/747: [Remove jvm directive from test too](https://github.com/basho/yokozuna/pull/747) +* clique/81: [Fix failing test on some environments](https://github.com/basho/clique/pull/81) +* riak_dt/121: [doc related fix & explanation](https://github.com/basho/riak_dt/pull/121) +* riak_dt/127: [bring develop-2.2 up-to-date with develop](https://github.com/basho/riak_dt/pull/127) +* riak_dt/129: [Add gset support](https://github.com/basho/riak_dt/pull/129) +* riak_dt/135: [Fix `equal/2` bug around unordered dict usage](https://github.com/basho/riak_dt/pull/135) +* riak_repl/776: [Fix bug when passing utc timestamps into httpd_util:rfc1123/1.](https://github.com/basho/riak_repl/pull/776) +* riak_repl/777: [Fix badarg in binary construction for args to ebloom](https://github.com/basho/riak_repl/pull/777) +* riak_repl/779: [Sticking plaster fix for basho/riak_repl#772](https://github.com/basho/riak_repl/pull/779) +* riak_repl/780: [Fix sometime failing test](https://github.com/basho/riak_repl/pull/780) +* riak_repl/782: [Change ETS queue table permissions to protected](https://github.com/basho/riak_repl/pull/782) + +## Previous Release Notes + +Please see the KV 2.2.3 release notes [here]({{<baseurl>}}riak/kv/2.2.3/release-notes/), and the KV 2.2.2 release notes [here]({{<baseurl>}}riak/kv/2.2.2/release-notes/). diff --git a/content/riak/kv/2.2.6/setup.md b/content/riak/kv/2.2.6/setup.md new file mode 100644 index 0000000000..3cd6490225 --- /dev/null +++ b/content/riak/kv/2.2.6/setup.md @@ -0,0 +1,45 @@ +--- +title: "Setup Riak KV" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Setup" + identifier: "setup_index" + weight: 110 + pre: install +toc: false +--- + +[plan index]: ../setup/planning +[install index]: ../setup/installing +[upgrade index]: ../setup/upgrading +[downgrade]: ../setup/downgrade + +## In This Section + +#### [Planning][plan index] + +Information on planning your Riak KV cluster including software & hardware recommendations. + +[Learn More >>][plan index] + +#### [Installing][install index] + +Step-by-step tutorials on installing Riak KV. + +[Learn More >>][install index] + +#### [Upgrading][upgrade index] + +Guides on upgrading your Riak KV cluster. + +[Learn More >>][upgrade index] + +#### [Downgrading][downgrade] + +A guide on downgrading your Riak KV cluster. + +[Learn More >>][downgrade] + diff --git a/content/riak/kv/2.2.6/setup/downgrade.md b/content/riak/kv/2.2.6/setup/downgrade.md new file mode 100644 index 0000000000..4591d383ec --- /dev/null +++ b/content/riak/kv/2.2.6/setup/downgrade.md @@ -0,0 +1,174 @@ +--- +title: "Downgrading" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Downgrading" + identifier: "downgrading" + weight: 103 + parent: "setup_index" +toc: true +aliases: + - /riak-docs/riak/2.2.6/ops/upgrading/rolling-downgrades/ + - /riak-docs/riak/kv/2.2.6/ops/upgrading/rolling-downgrades/ +--- + +[rolling upgrade]: {{<baseurl>}}riak/kv/2.2.6/setup/upgrading/cluster +[config ref]: {{<baseurl>}}riak/kv/2.2.6/configuring/reference +[concept aae]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/active-anti-entropy/ +[aae status]: {{<baseurl>}}riak/kv/2.2.6/using/admin/riak-admin/#aae-status + +Downgrades of Riak KV are tested and supported for two feature release versions, with the general procedure being similar to that of a [rolling upgrade][rolling upgrade]. + +Depending on the versions involved in the downgrade, there are additional steps to be performed before, during, and after the upgrade on on each node. These steps are related to changes or new features that are not present in the downgraded version. + +## Overview + +For every node in the cluster: + +1. Stop Riak KV. +2. Back up Riak's `etc` and `data` directories. +3. Downgrade the Riak KV. +4. Remove Riak search index and temporary data. +5. Reconfigure Solr cores. +6. Start Riak KV and disable Riak search. +7. Monitor the reindex of the data. +8. Finalize process and restart Riak KV & Riak search. + +### Guidelines + +* Riak control should be disabled throughout the rolling downgrade process. +* [Configuration Files][config ref] must be replaced with those of the version being downgraded to. + + +### Components That Complicate Downgrades + +| Feature | automatic | required | Notes | +|:---|:---:|:---:|:---| +|Migration to Solr 4.10.4 |✔ | ✔| Applies to all clusters using Riak Search. +| Active Anti-Entropy file format changes | ✔ | | Can be opted out using a [capability](#aae_tree_capability) + + +### When Downgrading is No Longer an Option + +If you enabled LZ4 compression in LevelDB and/or enabled global expiration in LevelDB when you installed KV 2.2.6, you cannot downgrade. + + +## General Process + +{{% note %}} +While the cluster contains mixed version members, if you have not set the cluster to use the legacy AAE tree format, you will see the `bad_version` error emitted to the log any time nodes with differing versions attempt to exchange AAE data (including AAE fullsync). + +This is benign and similar to the `not_built` and `already_locked` errors which can be seen during normal AAE operation. These events will stop once the downgrade is complete. +{{% /note %}} + +### Stop Riak KV and remove Riak search index & temporary data + +1\. Stop Riak KV: + +```bash +riak stop +``` +2\. Back up your Riak KV /etc and /data directories: + +```bash +sudo tar -czf riak_backup.tar.gz /var/lib/riak /etc/riak +``` + +3\. Downgrade Riak KV: + +```RHEL/CentOS +sudo rpm -Uvh »riak_package_name«.rpm +``` + +```Ubuntu +sudo dpkg -i »riak_package_name«.deb +``` + +4\. Remove the Riak search index data and AAE data: + + 1. Remove the cached Solr web application from the yz_temp folder. For the default package paths, this would be `/var/lib/riak/yz_temp/solr-webapp`. + + ```bash + rm -rf /var/lib/riak/yz_temp/solr-webapp + ``` + 2. Delete the Solr cores located in the yz directory. If you have custom solrconfig.xml files, you will need to restore the core from backup instead. + + For example: + + ```bash + rm -rf /var/lib/riak/yz/example_core1 + rm -rf /var/lib/riak/yz/example_core2 + ``` + +### Prepare to Re-index Solr Cores + +5\. (**Optional**) You can increase the AAE operation concurrency and increase the number of build operations while lowering the build limit's interval. This will increase the speed at which the AAE trees are rebuilt and the search indexes are repopulated. However, if you have a latency sensitive application, you should adjust these settings with care. + +```riak.conf +anti_entropy.concurrency_limit = 8 +anti_entropy.tree.build_limit.number = 4 +anti_entropy.tree.build_limit.per_timespan = 5m +``` + +### Start the node and disable Yokozuna + +6\. Start Riak KV: +{{% note %}} +Search results will be inconsistent until **Step 8.1** is complete. +{{% /note %}} + +```bash +riak start +``` + +7\. Wait for Riak search to start by running the following command: + +```bash +riak-admin wait-for-service yokozuna +``` + +8\. Run `riak attach`. + + 1. Run the following snippet to prevent this node from participating in distributed Riak Search queries: + + ``` + riak_core_node_watcher:service_down(yokozuna). + ``` + + 2. Expire the Yokozuna AAE Trees: + + ``` + yz_entropy_mgr:expire_trees(). + ``` + + 3. Exit the attach session by pressing **Ctrl-G** then **q**. + +### Monitor the reindex of the data + +9\. Monitor the build and exchange progress using the `riak-admin aae-status` and `riak-admin search aae-status` commands. + +The **All** column shows how long it has been since a partition exchanged with all of its sibling replicas. Consult the [`riak-admin aae-status` documentation][aae status] for more information about the AAE status output. + +Once both riak-admin aae-status and riak-admin search aae-status show values in the **All** column, the node will have successfully rebuilt all of the indexed data. + +### Finalize process and restart Yokozuna + + +10\. If you raised the concurrency AAE currency settings in riak.conf during **Step 5**, stop the node and remove the increased AAE thresholds. + +11\. If you chose not to increase the AAE concurrency via configuration and want to start Yokozuna without restarting the node, run `riak attach` and enter the following snippet: + +```erlang +riak_core_node_watcher:service_up(yokozuna,whereis(yz_solr_proc)). +``` + +12\. Exit the attach session by pressing **Ctrl-G** then **q**. + +13\. Verify that transfers have completed: + +```bash +riak-admin transfers +``` diff --git a/content/riak/kv/2.2.6/setup/installing.md b/content/riak/kv/2.2.6/setup/installing.md new file mode 100644 index 0000000000..134c76bbe4 --- /dev/null +++ b/content/riak/kv/2.2.6/setup/installing.md @@ -0,0 +1,60 @@ +--- +title: "Installing Riak KV" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Installing" + identifier: "installing" + weight: 101 + parent: "setup_index" + pre: cog +toc: true +aliases: + - /riak-docs/riak/2.2.6/ops/building/installing + - /riak-docs/riak/kv/2.2.6/ops/building/installing + - /riak-docs/riak/2.2.6/installing/ + - /riak-docs/riak/kv/2.2.6/installing/ +--- + +[install aws]: {{<baseurl>}}riak/kv/2.2.6/setup/installing/amazon-web-services +[install debian & ubuntu]: {{<baseurl>}}riak/kv/2.2.6/setup/installing/debian-ubuntu +[install freebsd]: {{<baseurl>}}riak/kv/2.2.6/setup/installing/freebsd +[install mac osx]: {{<baseurl>}}riak/kv/2.2.6/setup/installing/mac-osx +[install rhel & centos]: {{<baseurl>}}riak/kv/2.2.6/setup/installing/rhel-centos +[install smartos]: {{<baseurl>}}riak/kv/2.2.6/setup/installing/smartos +[install solaris]: {{<baseurl>}}riak/kv/2.2.6/setup/installing/solaris +[install suse]: {{<baseurl>}}riak/kv/2.2.6/setup/installing/suse +[install windows azure]: {{<baseurl>}}riak/kv/2.2.6/setup/installing/windows-azure +[install source index]: {{<baseurl>}}riak/kv/2.2.6/setup/installing/source +[community projects]: {{<baseurl>}}community/projects +[upgrade index]: {{<baseurl>}}riak/kv/2.2.6/setup/upgrading + +## Supported Platforms + +Riak is supported on numerous popular operating systems and virtualized +environments. The following information will help you to +properly install or upgrade Riak in one of the supported environments: + + * [Amazon Web Services][install aws] + * [Debian & Ubuntu][install debian & ubuntu] + * [FreeBSD][install freebsd] + * [Mac OS X][install mac osx] + * [RHEL & CentOS][install rhel & centos] + * [SmartOS][install smartos] + * [Solaris][install solaris] + * [SUSE][install suse] + * [Windows Azure][install windows azure] + +## Building from Source + +If your platform isn’t listed above, you may be able to build Riak from source. See [Installing Riak from Source][install source index] for instructions. + +## Community Projects + +Check out [Community Projects][community projects] for installing with tools such as [Chef](https://www.chef.io/chef/), [Ansible](http://www.ansible.com/), or [Cloudsoft](http://www.cloudsoftcorp.com/). + +## Upgrading + +For information on upgrading an existing cluster see [Upgrading Riak KV][upgrade index]. diff --git a/content/riak/kv/2.2.6/setup/installing/amazon-web-services.md b/content/riak/kv/2.2.6/setup/installing/amazon-web-services.md new file mode 100644 index 0000000000..ea817851ee --- /dev/null +++ b/content/riak/kv/2.2.6/setup/installing/amazon-web-services.md @@ -0,0 +1,100 @@ +--- +title_supertext: "Installing on" +title: "Amazon Web Services" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Amazon Web Services" + identifier: "installing_amazon_web_services" + weight: 301 + parent: "installing" +toc: true +aliases: + - /riak-docs/riak/2.2.6/ops/building/installing/Installing-on-AWS-Marketplace + - /riak-docs/riak/kv/2.2.6/ops/building/installing/Installing-on-AWS-Marketplace + - /riak-docs/riak/2.2.6/installing/amazon-web-services/ + - /riak-docs/riak/kv/2.2.6/installing/amazon-web-services/ +--- + +## Launching Riak VMs via the AWS Marketplace + +In order to launch a Riak virtual machine via the AWS Marketplace, you will first need to sign up for an [Amazon Web Services](http://aws.amazon.com) account. + +1. Navigate to [https://aws.amazon.com/marketplace/](https://aws.amazon.com/marketplace/) and sign in with your Amazon Web Services account. + +2. Locate Riak in the **Databases & Caching** category or search for Riak from any page. + +3. Set your desired AWS region, EC2 instance type, firewall settings, and key pair + + ![AWS Marketplace Instance Settings]({{<baseurl>}}images/aws-marketplace-settings.png) + +4. Click the **Accept Terms and Launch with 1-Click** button. + +### Security Group Settings + +Once the virtual machine is created, you should verify that your selected EC2 security group is properly configured for Riak. + +1. In the AWS EC2 Management Console, click **Security Groups**, then click the name of the security group for your Riak VM. + +2. Click on the **Inbound** tab in the lower pane. Your security group should include the following open ports: + + * 22 (SSH) + * 8087 (Riak Protocol Buffers Interface) + * 8098 (Riak HTTP Interface) + +3. You will need to add additional rules within this security group to allow your Riak instances to communicate. For each port range below, create a new **Custom TCP rule** with the source set to the current security group ID (found on the **Details** tab). + + * Port range: 4369 + * Port range: 6000-7999 + * Port range: 8099 + +4. When complete, your security group should contain all of the rules listed below. If you are missing any rules, add them in the lower panel and then click the **Apply Rule Changes** button. + + ![EC2 Security Group Settings]({{<baseurl>}}images/aws-marketplace-security-group.png) + +We also recommend that you read more about Riak's [Security and Firewalls]({{<baseurl>}}riak/kv/2.2.6/using/security/). + +## Clustering Riak on AWS + +You will need need to launch at least 3 instances to form a Riak cluster. When the instances have been provisioned and the security group is configured, you can connect to them using SSH or PuTTY as the ec2-user. + +You can find more information on connecting to an instance on the official [Amazon EC2 instance guide](http://docs.amazonwebservices.com/AWSEC2/latest/UserGuide/AccessingInstances.html). + +{{% note title="Note" %}} +The following clustering setup will _not_ be resilient to instance restarts +unless deployed in Amazon VPC. +{{% /note %}} + +1. On the first node, obtain the internal IP address: + + ```bash + curl http://169.254.169.254/latest/meta-data/local-ipv4 + ``` + +2. For all other nodes, use the internal IP address of the first node: + + ```bash + sudo riak-admin cluster join riak@<ip.of.first.node> + ``` + +3. After all of the nodes are joined, execute the following: + + ```bash + sudo riak-admin cluster plan + ``` + + If this looks good: + + ```bash + sudo riak-admin cluster commit + ``` + + To check the status of clustering use: + + ```bash + sudo riak-admin member_status + ``` + +You now have a Riak cluster running on AWS. diff --git a/content/riak/kv/2.2.6/setup/installing/debian-ubuntu.md b/content/riak/kv/2.2.6/setup/installing/debian-ubuntu.md new file mode 100644 index 0000000000..29e52bb884 --- /dev/null +++ b/content/riak/kv/2.2.6/setup/installing/debian-ubuntu.md @@ -0,0 +1,226 @@ +--- +title_supertext: "Installing on" +title: "Debian and Ubuntu" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Debian & Ubuntu" + identifier: "installing_debian_ubuntu" + weight: 302 + parent: "installing" +toc: true +aliases: + - /riak-docs/riak/2.2.6/ops/building/installing/Installing-on-Debian-and-Ubuntu + - /riak-docs/riak/kv/2.2.6/ops/building/installing/Installing-on-Debian-and-Ubuntu + - /riak-docs/riak/2.2.6/installing/debian-ubuntu/ + - /riak-docs/riak/kv/2.2.6/installing/debian-ubuntu/ +--- + +[install source index]: {{<baseurl>}}riak/kv/2.2.6/setup/installing/source/ +[security index]: {{<baseurl>}}riak/kv/2.2.6/using/security/ +[install source erlang]: {{<baseurl>}}riak/kv/2.2.6/setup/installing/source/erlang +[install verify]: {{<baseurl>}}riak/kv/2.2.6/setup/installing/verify + +Riak KV can be installed on Debian or Ubuntu-based systems using a binary +package or by compiling from source code. + +The following steps have been tested to work with Riak KV on: + +- Ubuntu 16.04 +- Ubuntu 14.04 +- Ubuntu 12.04 +- Debian 8.6 +- Debian 7.6 + +## Installing with apt + +Basho has Riak KV packages on the [packagecloud.io](https://packagecloud.io/basho/riak?filter=debs) hosting service. Instructions for installing via shell scripts, manual installation, Chef, and Puppet can be found in packagecloud's [installation docs](https://packagecloud.io/basho/riak/install). + +Platform-specific pages are linked below: + +* [Ubuntu 16.04 (Xenial)](https://packagecloud.io/basho/riak/packages/ubuntu/xenial/riak_2.2.6-1_amd64.deb) +* [Ubuntu 14.04 (Trusty)](https://packagecloud.io/basho/riak/packages/ubuntu/trusty/riak_2.2.6-1_amd64.deb) +* [Ubuntu 12.04 (Precise)](https://packagecloud.io/basho/riak/packages/ubuntu/precise/riak_2.2.6-1_amd64.deb) +* [Debian 8 (Jessie)](https://packagecloud.io/basho/riak/packages/debian/jessie/riak_2.2.6-1_amd64.deb) +* [Debian 7 (Wheezy)](https://packagecloud.io/basho/riak/packages/debian/wheezy/riak_2.2.6-1_amd64.deb) + +Our documentation also includes instructions regarding signing keys and +sources lists, which can be found in the Advanced apt Installation section below. + +## Advanced apt Installation + +> **Note on Debian 7** +> +> If you wish to install Riak on Debian 7, you may need to install +[libc6](https://packages.debian.org/search?keywords=libc6) version 2.15 or +later, which in turn requires upgrading your system to +[sid](https://www.debian.org/releases/sid/). Installation instructions +can be found +[here](https://wiki.debian.org/DebianUnstable#How_do_I_install_Sid.3F). +> +> Once sid has been installed, you can install libc6 with the following +command: +> +>```bash +apt-get -t sid install libc6 libc6-dev libc6-dbg +``` + + +For the simplest installation process on LTS (Long-Term Support) +releases, use `apt-get`. First, you must retrieve the signing key: + +```curl +curl https://packagecloud.io/gpg.key | sudo apt-key add - +``` + +Second, you must install the `apt-transport-https` package in order to +be able to fetch packages over HTTPS: + +```bash +sudo apt-get install -y apt-transport-https +``` + +With HTTPS enabled, we recommend adding the desired Riak package to your +`.list` file. packagecloud can autogenerate such a file on the basis of +a name that you specify, e.g. a hostname, and the desired operating +system and distribution. The following example script would store your +hostname in the variable `HOSTNAME`, send that information to +packagecloud to autogenerate a `.list` file, and then store the return +value in a file called `basho.list`, which is stored in the +`/etc/apt/sources.list.d` directory. This example script is specific to +the Precise Ubuntu distribution: + +```bash +#!/bin/bash + +HOSTNAME=`hostname -f` +FILENAME=/etc/apt/sources.list.d/basho.list +OS=ubuntu +DIST=precise +PACKAGE_CLOUD_RIAK_DIR=https://packagecloud.io/install/repositories/basho/riak +curl "${PACKAGE_CLOUD_RIAK_DIR}/config_file.list?os=${OS}&dist=${DIST}&name=${HOSTNAME}" > $FILENAME +``` + +The `name` that you submit to packagecloud can be anything you like. The +`HOSTNAME` used above was for example purposes. The resulting file +should hold contents like the following: + +``` +# this file was generated by packagecloud.io for +# the repository at https://packagecloud.io/basho/riak + +deb https://packagecloud.io/basho/riak/ubuntu/ precise main +deb-src https://packagecloud.io/basho/riak/ubuntu/ precise main +``` + +With your `basho.list` file populated, you can update your apt sources +list: + +```bash +sudo apt-get update +``` + +Now install the `riak` package. + +```bash +sudo apt-get install riak +``` + +That should be all. + +## Installing From Package + +If you wish to install the deb packages by hand, follow these +instructions. + +### Installing on Non-LTS Ubuntu Releases + +Typically we only package Riak for LTS releases to keep our build and +testing matrix focused. In some cases, such as Ubuntu 11.04 (Natty), +there are changes that affect how Riak is packaged, so we will release a +separate package for that non-LTS release. In most other cases, however, +if you are running a non-LTS release (such as 12.10) it is safe to +follow the below instructions for the LTS release prior to your release. +In the case of Ubuntu 12.10, follow the installation instructions for +Ubuntu 12.04. + +### PAM Library Requirement for Ubuntu + +One dependency that may be missing on your machine is the `libpam0g-dev` +package used for Pluggable Authentication Module (PAM) authentication, +associated with [Riak security][security index]. + +To install: + +```bash +sudo apt-get install libpam0g-dev +``` + +### SSL Library Requirement for Ubuntu + +Riak currently requires libssl version 0.9.8 on some versions of Ubuntu. +Starting at Ubuntu 12.04 this is no longer an issue. Before installing +Riak via package on Ubuntu, install the `libssl0.9.8` package. Note that +this version of libssl can be safely installed alongside +current/existing libssl installations. + +To install the libssl version 0.9.8 package, execute the following +command: + +```bash +sudo apt-get install libssl0.9.8 +``` + +After the libssl package installation, proceed to installing Riak from +the pre-built package by executing the following commands as appropriate +for the target platform: + +### Riak 64-bit Installation + +#### Ubuntu Lucid Lynx (10.04) + +```bash +wget https://files.tiot.jp/riak/kv/2.2/2.2.6/ubuntu/lucid/riak_2.2.6-1_amd64.deb +sudo dpkg -i riak_2.2.6-1_amd64.deb +``` + +#### Ubuntu Natty Narwhal (11.04) + +```bash +wget https://files.tiot.jp/riak/kv/2.2/2.2.6/ubuntu/natty/riak_2.2.6-1_amd64.deb +sudo dpkg -i riak_2.2.6-1_amd64.deb +``` + +#### Ubuntu Precise Pangolin (12.04) + +```bash +wget https://files.tiot.jp/riak/kv/2.2/2.2.6/ubuntu/precise/riak_2.2.6-1_amd64.deb +sudo dpkg -i riak_2.2.6-1_amd64.deb +``` + +## Installing From Source + +First, install Riak dependencies using apt: + +```bash +sudo apt-get install build-essential libc6-dev-i386 git +``` + +Riak requires an [Erlang](http://www.erlang.org/) installation. +Instructions can be found in [Installing Erlang][install source erlang]. + +```bash +wget https://files.tiot.jp/riak/kv/2.2/2.2.6/riak-2.2.6.tar.gz +tar zxvf riak-2.2.6.tar.gz +cd riak-2.2.6 +make rel +``` + +If the build was successful, a fresh build of Riak will exist in the +`rel/riak` directory. + +## Next Steps + +Now that Riak is installed, check out [Verifying a Riak Installation][install verify]. diff --git a/content/riak/kv/2.2.6/setup/installing/freebsd.md b/content/riak/kv/2.2.6/setup/installing/freebsd.md new file mode 100644 index 0000000000..620a9778d3 --- /dev/null +++ b/content/riak/kv/2.2.6/setup/installing/freebsd.md @@ -0,0 +1,130 @@ +--- +title_supertext: "Installing on" +title: "FreeBSD" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "FreeBSD" + identifier: "installing_freebsd" + weight: 303 + parent: "installing" +toc: true +aliases: + - /riak-docs/riak/2.2.6/ops/building/installing/Installing-on-FreeBSD + - /riak-docs/riak/kv/2.2.6/ops/building/installing/Installing-on-FreeBSD + - /riak-docs/riak/2.2.6/installing/freebsd/ + - /riak-docs/riak/kv/2.2.6/installing/freebsd/ +--- + + + +[install source erlang]: {{<baseurl>}}riak/kv/2.2.6/setup/installing/source/erlang +[downloads]: {{<baseurl>}}riak/kv/2.2.6/downloads/ +[install verify]: {{<baseurl>}}riak/kv/2.2.6/setup/installing/verify + +You can install Riak on FreeBSD for the AMD64 architecture with a binary package or by building from source code. + +## Installing From Binary Package + +> **Note:** The Riak 1.2 binary package is supported on FreeBSD version 9. Users have reported success building Riak from source on a number of FreeBSD versions, however. + +Installing Riak from a binary package is the simplest method with least required dependencies, and requires less time to complete than building from source. + +### Prerequisites and Dependencies + +Riak depends on `sudo` to be installed if the Riak command line tools are to be executed by users other than the *riak* user. Please ensure that `sudo` is installed via packages or the ports collection prior to installing the Riak package. + +The Riak binary package also depends on a packaged version of OpenSSL. Prior to installing Riak 1.2 on FreeBSD 9, you'll need to install `openssl-1.0.0_7` either from package or the ports collection. + +### Installation + +You can install the Riak binary package on FreeBSD remotely using the +`pkg_add` remote option. For this example, we're installing `riak-2.2.6-FreeBSD-amd64.tbz`. + +```bash +sudo pkg_add -r https://files.tiot.jp/riak/kv/2.2/2.2.6/freebsd/9/riak-2.2.6-FreeBSD-amd64.tbz +``` + +When Riak is installed, a message is displayed with information about the installation and available documentation. + +``` +Thank you for installing Riak. + +Riak has been installed in /usr/local owned by user:group riak:riak + +The primary directories are: + + {platform_bin_dir, "/usr/local/sbin"} + {platform_data_dir, "/var/db/riak"} + {platform_etc_dir, "/usr/local/etc/riak"} + {platform_lib_dir, "/usr/local/lib/riak"} + {platform_log_dir, "/var/log/riak"} + +These can be configured and changed in the platform_etc_dir/app.config. + +Add /usr/local/sbin to your path to run the riak and riak-admin scripts directly. + +Man pages are available for riak(1) and riak-admin(1) +``` + +If instead of this message, you receive an error during installation regarding OpenSSL, similar to this one: + +``` +Package dependency openssl-1.0.0_7 for /tmp/riak-2.2.6-FreeBSD-amd64.tbz not found! +``` + +Be sure that you've installed the required OpenSSL version from packages or the ports collection as described in the **Prerequisites and Dependencies** section. + +## Installing From Source + +Installing Riak from source on FreeBSD is a straightforward process which requires installation of more dependencies (such as Erlang) prior to building, and requires more time than a binary package installation. + +That said, installing from source provides for greater flexibility with respect to configuration, data root locations, and more fine grained control over specific dependency versions. + +### Prerequisites and Dependencies + +When building and installing Riak from source, you might be required to install some prerequisite software before proceeding with the build. + +If you do not currently have the following software installed, please install it with packages or the ports collection before proceeding. + +* Erlang ([Installing Erlang][install source erlang]) +* Curl +* Git +* OpenSSL (version 1.0.0_7) +* Python +* sudo + +### Installation +First download the version you wish to install from [Basho downloads][downloads]. + +Next, unpack and build a release from source: + +```bash +tar zxf <riak-x.x.x> +cd riak-x.x.x +gmake rel +``` + +Upon conclusion of the build, the `rel/riak` directory will contain a full Riak node environment, including configuration, data, and log directories: + +```bash +bin # Riak binaries +data # Riak data and metadata +erts-5.9.2 # Erlang Run-Time System +etc # Riak Configuration +lib # Third party libraries +log # Operational logs +releases # Release information +``` + +If you'd prefer to build a development environment consisting of 4 nodes which can be run as a cluster on one machine, specify the `devrel` target instead of the `rel` target, like this: + +```bash +gmake devrel +``` + +## Next Steps + +Now that Riak is installed, check out [Verifying a Riak Installation][install verify]. diff --git a/content/riak/kv/2.2.6/setup/installing/mac-osx.md b/content/riak/kv/2.2.6/setup/installing/mac-osx.md new file mode 100644 index 0000000000..c9ec36cb3c --- /dev/null +++ b/content/riak/kv/2.2.6/setup/installing/mac-osx.md @@ -0,0 +1,118 @@ +--- +title_supertext: "Installing on" +title: "Mac OS X" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Mac OS X" + identifier: "installing_macosx" + weight: 303 + parent: "installing" +toc: true +aliases: + - /riak-docs/riak/2.2.6/ops/building/installing/Installing-on-Mac-OS-X + - /riak-docs/riak/kv/2.2.6/ops/building/installing/Installing-on-Mac-OS-X + - /riak-docs/riak/2.2.6/installing/mac-osx/ + - /riak-docs/riak/kv/2.2.6/installing/mac-osx/ +--- + + + +[perf open files]: {{<baseurl>}}riak/kv/2.2.6/using/performance/open-files-limit +[install source erlang]: {{<baseurl>}}riak/kv/2.2.6/setup/installing/source/erlang +[install verify]: {{<baseurl>}}riak/kv/2.2.6/setup/installing/verify + +The following steps are known to work with Mac OS X 10.8, 10.9 +(Mavericks), and Yosemite. You can install from source or download a +precompiled tarball. + +> **`ulimit` on OS X** +> +> OS X gives you a very small limit on open file handles, so even with a +backend that uses very few file handles, it's possible to run out. See +[Open Files Limit][perf open files] for more information about changing the limit. + + +## From Precompiled Tarballs + +To run Riak from our precompiled tarball, run these commands for the +appropriate platform: + +### 64-bit + +```bash +curl -O https://files.tiot.jp/riak/kv/2.2/2.2.6/osx/10.8/riak-2.2.6-OSX-x86_64.tar.gz +tar xzvf riak-2.2.6-osx-x86_64.tar.gz +``` + +After the release is untarred, you will be able to `cd` into the `riak` +directory and execute `bin/riak start` to start the Riak node. + +## Homebrew + +{{% note title="Warning: Homebrew not always up to date" %}} +Homebrew's Riak recipe is community supported, and thus is not always up to +date with the latest Riak package. Please ensure that the current recipe is +using the latest supported code (and don't be afraid to update it if it's +not). +{{% /note %}} + +Installing Riak 2.0 with [Homebrew](http://brew.sh/) is easy: + +```bash +brew install --devrel riak +``` + +By default, this will place a `2.2.6` folder in +`/usr/local/Cellar/riak`. + +Be aware that you will most likely see the following message after +running `brew install`: + +``` +Error: The `brew link` step did not complete successfully +The formula built, but is not symlinked into /usr/local + +You can try again using: + brew link riak +``` + +We do not recommend using `brew link` with Riak. Instead, we recommend +either copying that directory to a desired location on your machine, +aliasing the executables in the `/bin` directory, or interacting with +the Riak installation directory via environment variables. + +**Note**: Homebrew will install Erlang if you don't have it already. + +## Installing From Source + +You must have Xcode tools installed from [Apple's Developer +website](http://developer.apple.com/). + +{{% note title="Note on Clang" %}} +Riak has had problems compiling with Clang in the past. As of Riak KV +2.2.6 and Clang 902.0.39.1, Clang can build Riak. +{{% /note %}} + +Riak requires [Erlang](http://www.erlang.org/) R16B02+. + +If you do not have Erlang already installed, see [Installing Erlang][install source erlang]. + +Next, download and unpack the source distribution. + +```bash +curl -O https://files.tiot.jp/riak/kv/2.2/2.2.6/riak-2.2.6.tar.gz +tar zxvf riak-2.2.6.tar.gz +cd riak-2.2.6 +make rel +``` + +If you receive errors when building about "incompatible architecture," +please verify that you built Erlang with the same architecture as your +system (Snow Leopard and higher: 64bit). + +## Next Steps + +Now that Riak is installed, check out [Verifying a Riak Installation][install verify]. diff --git a/content/riak/kv/2.2.6/setup/installing/rhel-centos.md b/content/riak/kv/2.2.6/setup/installing/rhel-centos.md new file mode 100644 index 0000000000..97a32cc938 --- /dev/null +++ b/content/riak/kv/2.2.6/setup/installing/rhel-centos.md @@ -0,0 +1,172 @@ +--- +title_supertext: "Installing on" +title: "RHEL and CentOS" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "RHEL & CentOS" + identifier: "installing_rhel_centos" + weight: 304 + parent: "installing" +toc: true +aliases: + - /riak-docs/riak/2.2.6/ops/building/installing/Installing-on-RHEL-and-CentOS + - /riak-docs/riak/kv/2.2.6/ops/building/installing/Installing-on-RHEL-and-CentOS + - /riak-docs/riak/2.2.6/installing/rhel-centos/ + - /riak-docs/riak/kv/2.2.6/installing/rhel-centos/ +--- + + + +[install source index]: {{<baseurl>}}riak/kv/2.2.6/setup/installing/source +[install source erlang]: {{<baseurl>}}riak/kv/2.2.6/setup/installing/source/erlang +[install verify]: {{<baseurl>}}riak/kv/2.2.6/setup/installing/verify + +Riak KV can be installed on CentOS- or Red-Hat-based systems using a binary +package or by [compiling Riak from source code][install source index]. The following steps have been tested to work with Riak on +CentOS/RHEL 6.9, and 7.5.1804. + +> **Note on SELinux** +> +> CentOS enables SELinux by default, so you may need to disable SELinux if +you encounter errors. + +## Installing with rpm + +For versions of Riak prior to 2.0, Basho used a self-hosted +[rpm](http://www.rpm.org/) repository for CentOS and RHEL packages. For +versions 2.0 and later, Basho has moved those packages to the +[packagecloud.io](https://packagecloud.io/) hosting service. +Instructions for installing via shell scripts, manual installation, +Chef, and Puppet can be found in packagecloud's [installation +docs](https://packagecloud.io/basho/riak/install). + +Platform-specific pages are linked below: + +* [el6](https://files.tiot.jp/riak/packages/el/6/riak-2.2.5-1.el6.x86_64.rpm) +* [el7](https://files.tiot.jp/riak/packages/el/7/riak-2.2.5-1.el7.centos.x86_64.rpm) + +Our documentation also includes instructions regarding signing keys and +sources lists, which can be found in the section immediately below. + +## Advanced rpm Installation + +For the simplest installation process on LTS (Long-Term Support) +releases, use yum. First, you must install the `pygpgme` package, which +enables yum to handle [GPG](https://www.gnupg.org/) signatures: + +```bash +sudo yum install pygpgme +``` + +If you wish to install using a `.repo` file, packagecloud can generate +one for you on the basis of a name that you specify, e.g. a hostname, +and the desired operating system and distribution. The following example +script would store your hostname in the variable `HOSTNAME`, send that +information to packagecloud to generate a `.repo` file, and then store +the return value in a file called `basho.repo`, which is stored in the +`/etc/yum.repos.d` directory: + +```bash +#!/bin/bash + +HOSTNAME=`hostname -f` +FILENAME=/etc/yum.repos.d/basho.repo +OS=el +DIST=5 +PACKAGE_CLOUD_RIAK_DIR=https://packagecloud.io/install/repositories/basho/riak +curl "${PACKAGE_CLOUD_RIAK_DIR}/config_file.repo?os=${OS}&dist=${DIST}&name=${HOSTNAME}" > $FILENAME +``` + +The `name` that you submit to packagecloud can be anything you like. The +`HOSTNAME` used above was for example purposes. The resulting file +should contents like the following: + +``` +[basho_riak] +name=basho_riak +baseurl=https://packagecloud.io/basho/riak/el/6/$basesearch +repo_gpgcheck=1 +gpgcheck=0 +enabled=1 +gpgkey=https://packagecloud.io/gpg.key +sslverify=1 +sslcacert=/etc/pki/tls/certs/ca-bundle.crt +``` + +With your `basho.repo` file population, you can update your rpm sources +list. + +## Installing From Package + +If you wish to install the RHEL/CentOS packages by hand, follow these +instructions. + +### For Centos 7 / RHEL 7 + +You can install CentOS 7/RHEL 7 using yum, which we recommend: + +```bash +wget https://files.tiot.jp/riak/kv/2.2/2.2.6/rhel/7/riak-2.2.6-1.el7.x86_64.rpm +sudo yum localinstall -y riak-2.2.6-1.el7.x86_64.rpm +``` + +Or you can install the `.rpm` package manually: + +```bash +wget https://files.tiot.jp/riak/kv/2.2/2.2.6/rhel/7/riak-2.2.6-1.el7.x86_64.rpm +sudo rpm -Uvh riak-2.2.6-1.el7.x86_64.rpm +``` + +### For Centos 6 / RHEL 6 + +You can install using yum, which we recommend: + +```bash +wget https://files.tiot.jp/riak/kv/2.2/2.2.6/rhel/6/riak-2.2.6-1.el6.x86_64.rpm +sudo yum localinstall -y riak-2.2.6-1.el6.x86_64.rpm + +``` + +Or you can install the `.rpm` package manually: + +```bash +wget https://files.tiot.jp/riak/kv/2.2/2.2.6/rhel/6/riak-2.2.6-1.el6.x86_64.rpm +sudo rpm -Uvh riak-2.2.6-1.el6.x86_64.rpm +``` + +## Installing From Source + +Riak requires an [Erlang](http://www.erlang.org/) installation. +Instructions can be found in [Installing Erlang][install source erlang]. + +Building from source will require the following packages: + +* `gcc` +* `gcc-c++` +* `glibc-devel` +* `make` +* `pam-devel` + +You can install these with yum: + +```bash +sudo yum install gcc gcc-c++ glibc-devel make git pam-devel +``` + +Now we can download and install Riak: + +```bash +wget https://files.tiot.jp/riak/kv/2.2/2.2.6/riak-2.2.6.tar.gz +tar zxvf riak-2.2.6.tar.gz +cd riak-2.2.6 +make rel +``` + +You will now have a fresh build of Riak in the `rel/riak` directory. + +## Next Steps + +Now that Riak is installed, check out [Verifying a Riak Installation][install verify]. diff --git a/content/riak/kv/2.2.6/setup/installing/smartos.md b/content/riak/kv/2.2.6/setup/installing/smartos.md new file mode 100644 index 0000000000..cfc9a3b3f3 --- /dev/null +++ b/content/riak/kv/2.2.6/setup/installing/smartos.md @@ -0,0 +1,117 @@ +--- +title_supertext: "Installing on" +title: "SmartOS" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "SmartOS" + identifier: "installing_smartos" + weight: 305 + parent: "installing" +toc: true +aliases: + - /riak-docs/riak/2.2.6/ops/building/installing/Installing-on-SmartOS + - /riak-docs/riak/kv/2.2.6/ops/building/installing/Installing-on-SmartOS + - /riak-docs/riak/2.2.6/installing/smartos/ + - /riak-docs/riak/kv/2.2.6/installing/smartos/ +--- + +[install verify]: {{<baseurl>}}riak/kv/2.2.6/setup/installing/verify + +{{% note title="SmartOS End of Life (EOL) for Riak KV 2.2.6" %}} +SmartOS is no longer supported in Riak KV 2.2.6+. If you are interested in using Riak KV on SmartOS, you can still [build from source](../source). +{{% /note %}} + +The following steps have been tested to work with Riak version 1.2 on SmartOS version **joyent_20120614T184600Z**. They demonstrate installation of a Riak node on SmartOS as the root user. + +## Open Files Limit + +Before proceeding with installation, you should ensure that the system's open +files limit is at least 65536. Check the current limits to verify this: + +```bash +ulimit -a +``` + +To temporarily increase this limit *for the life of your session*, use the following command: + +```bash +ulimit -n 65536 +``` + +To increase this value in a persistent manner that will be enforced after restarting the system, add the following to `/etc/system`: + +```bash +set rlim_fd_max=65536 +``` + +## Choosing a Version + +SmartOS, albeit powerful, can make some easy tasks (like figuring out a "version" of SmartOS) difficult. Defining the correct version is a combination of the Global Zone snapshot version and the pkgsrc version in the guest zones. Here is the way to determine which Riak package to use. + +The thing that really matters for Riak is what dataset was used to make the SmartOS VM. These datasets come from joyent and appear like this with the `dsadm` command: + +``` +fdea06b0-3f24-11e2-ac50-0b645575ce9d smartos 2012-12-05 sdc:sdc:base64:1.8.4 +f4c23828-7981-11e1-912f-8b6d67c68076 smartos 2012-03-29 sdc:sdc:smartos64:1.6.1 +``` + +This is where the `1.6` and `1.8` versions come from in the package naming. It isn't perfect, but if you know what dataset you used to make your SmartOS VM, you will know which package to use. + +For Joyent Cloud users who don't know what dataset was used, in the guest zone type: + +``` +cat /opt/local/etc/pkgin/repositories.conf +``` + +* If this returns `http://pkgsrc.joyent.com/sdc6/2012Q2/x86_64/All` or any other *2012Q2* you need to use the `1.8` download. +* If this returns `http://pkgsrc.joyent.com/sdc6/2011Q4/x86_64/All` or any other *2011* you need to use the `1.6` download. + +## Download and Install + +Download your version of the Riak binary package for SmartOS: + +```bash +curl -o /tmp/riak-2.2.6-SmartOS-x86_64.tgz https://files.tiot.jp/riak/kv/2.2/2.2.6/smartos/1.8/riak-2.2.6-SmartOS-x86_64.tgz +``` + +Next, install the package: + +``` +pkg_add /tmp/riak-2.2.6-SmartOS-x86_64.tgz +``` + +After installing the package, enable the Riak and Erlang Port Mapper Daemon (epmd) services: + +```bash +svcadm -v enable -r riak +``` + +Finally, after enabling the services, check to see that they are online: + +``` +svcs -a | grep -E 'epmd|riak' +``` + +Output from the above command should resemble the following: + +``` +online 17:17:16 svc:/network/epmd:default +online 17:17:16 svc:/application/riak:default +``` + +Finally, and provided that the services are shown to be in an **online** state, go ahead and ping Riak: + +```bash +riak ping +``` + +Pinging Riak will result in a `pong` response if the node is up and reachable. If the node is not up and reachable, a `Node <nodename> not responding to pings` error will result instead. + +If all responses indicate that riak is up and running, then you have successfully installed and configured Riak as service on SmartOS. + +## Next Steps + +Now that Riak is installed, check out [Verifying a Riak Installation][install verify]. diff --git a/content/riak/kv/2.2.6/setup/installing/solaris.md b/content/riak/kv/2.2.6/setup/installing/solaris.md new file mode 100644 index 0000000000..d15834a9ef --- /dev/null +++ b/content/riak/kv/2.2.6/setup/installing/solaris.md @@ -0,0 +1,86 @@ +--- +title_supertext: "Installing on" +title: "Solaris" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Solaris" + identifier: "installing_solaris" + weight: 306 + parent: "installing" +toc: true +aliases: + - /riak-docs/riak/2.2.6/ops/building/installing/Installing-on-Solaris + - /riak-docs/riak/kv/2.2.6/ops/building/installing/Installing-on-Solaris + - /riak-docs/riak/2.2.6/installing/solaris/ + - /riak-docs/riak/kv/2.2.6/installing/solaris/ +--- + + + +[install verify]: {{<baseurl>}}riak/kv/2.2.6/setup/installing/verify + +The following steps have been tested to work with Riak version 1.3.1 on Solaris 10 i386. They demonstrate installation of a Riak node on Solaris as the root user. + +> **Note:** Before installing Riak on Solaris, be sure that you've installed `sudo` as Riak's scripts require it for proper operation. + +## Open Files Limit + +Before proceeding with installation, you should ensure that the system's open files limit is at least 65536 by verifying the current value of `nofiles(descriptors)`. Check the current value with the `ulimit` command: + +```bash +ulimit -a +``` + +To temporarily increase this limit for the life of your session, use the following command: + +```bash +ulimit -n 65536 +``` + +To increase this value in a persistent manner that will be enforced after restarting the system, add the following to the `/etc/system` file: + +``` +set rlim_fd_max=65536 +set rlim_fd_cur=65536 +``` + +Note that you must restart to have the above settings take effect. + +## Download and Install + +Download your version of the Riak binary package for Solaris 10: + +```bash +curl -o /tmp/BASHOriak-2.2.6-Solaris10-i386.pkg.gz https://files.tiot.jp/riak/kv/2.2/2.2.6/solaris/10/BASHOriak-2.2.6-Solaris10-x86_64.pkg.gz +``` + +Next, install the package: + +```bash +gunzip /tmp/BASHOriak-2.2.6-Solaris10-i386.pkg.gz +pkgadd /tmp/BASHOriak-2.2.6-Solaris10-i386.pkg +``` + +After installing the package, be sure to include `/opt/riak/bin` in the +appropriate user's `PATH`. After doing so, you can then start Riak: + +```bash +riak start +``` + +Finally, go ahead and ping Riak to ensure it is running: + +```bash +riak ping +``` + +Pinging Riak will result in a `pong` response if the node is up and reachable. If the node is not up and reachable, a `Node <nodename> not responding to pings` error will result instead. + +If all responses indicate that riak is up and running, then you have successfully installed Riak on Solaris 10. + +## Next Steps + +Now that Riak is installed, check out [Verifying a Riak Installation][install verify]. diff --git a/content/riak/kv/2.2.6/setup/installing/source.md b/content/riak/kv/2.2.6/setup/installing/source.md new file mode 100644 index 0000000000..1343d052e3 --- /dev/null +++ b/content/riak/kv/2.2.6/setup/installing/source.md @@ -0,0 +1,105 @@ +--- +title_supertext: "Installing" +title: "Riak KV From Source" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Installing From Source" + identifier: "installing_source" + weight: 310 + parent: "installing" +toc: true +aliases: + - /riak-docs/riak/2.2.6/ops/building/Installing-Riak-from-Source + - /riak-docs/riak/kv/2.2.6/ops/building/Installing-Riak-from-Source + - /riak-docs/riak/2.2.6/installing/source/ + - /riak-docs/riak/kv/2.2.6/installing/source/ +--- + + + +[install source erlang]: {{<baseurl>}}riak/kv/2.2.6/setup/installing/source/erlang +[downloads]: {{<baseurl>}}riak/kv/2.2.6/downloads/ +[install debian & ubuntu#source]: {{<baseurl>}}riak/kv/2.2.6/setup/installing/debian-ubuntu/#installing-from-source +[install freebsd#source]: {{<baseurl>}}riak/kv/2.2.6/setup/installing/freebsd/#installing-from-source +[install mac osx#source]: {{<baseurl>}}riak/kv/2.2.6/setup/installing/mac-osx/#installing-from-source +[install rhel & centos#source]: {{<baseurl>}}riak/kv/2.2.6/setup/installing/rhel-centos/#installing-from-source +[install verify]: {{<baseurl>}}riak/kv/2.2.6/setup/installing/verify + +Riak should be installed from source if you are building on a platform +for which a package does not exist or if you are interested in +contributing to Riak. + +## Dependencies + +### Erlang + +To install Riak, you will need to have [Erlang](http://www.erlang.org/) installed. We strongly recommend using Basho's patched version of Erlang to install Riak 2.0+. All of the patches in this version have been incorporated into later versions of the official Erlang/OTP release. + +See [Installing Erlang][install source erlang] for instructions. + +### Git + +Riak depends on source code located in multiple Git repositories. Install [Git](https://git-scm.com/) on the target system before attempting the build. + +### GCC + +Riak will not compile with Clang. Please make sure your default C/C++ +compiler is [GCC](https://gcc.gnu.org/). + +## Installation + +The following instructions generate a complete, self-contained build of +Riak in `$RIAK/rel/riak` where `$RIAK` is the location of the unpacked +or cloned source. + +### Installing from source package + +Download the Riak source package from the [Download Center][downloads] and build: + +```bash +curl -O https://files.tiot.jp/riak/kv/2.2/2.2.6/riak-2.2.6.tar.gz +tar zxvf riak-2.2.6.tar.gz +cd riak-2.2.6 +make locked-deps +make rel +``` + +### Installing from GitHub + +The [Riak Github respository](http://github.com/basho/riak) has much +more information on building and installing Riak from source. To clone +and build Riak from source, follow the steps below. + +Clone the repository using [Git](http://git-scm.com) and build: + +```bash +git clone git://github.com/basho/riak.git +cd riak +make locked-deps +make rel +``` + +## Platform-Specific Instructions + +For instructions about specific platforms, see: + + * [Debian & Ubuntu][install debian & ubuntu#source] + * [FreeBSD][install freebsd#source] + * [Mac OS X][install mac osx#source] + * [RHEL & CentOS][install rhel & centos#source] + +If you are running Riak on a platform not in the list above and need +some help getting it up and running, join The Riak Mailing List and +inquire about it there. We are happy to help you get up and running with +Riak. + +### Windows + +Riak is not currently supported on Microsoft Windows. + +## Next Steps + +Now that Riak is installed, check out [Verifying a Riak Installation][install verify]. diff --git a/content/riak/kv/2.2.6/setup/installing/source/erlang.md b/content/riak/kv/2.2.6/setup/installing/source/erlang.md new file mode 100644 index 0000000000..842c589a57 --- /dev/null +++ b/content/riak/kv/2.2.6/setup/installing/source/erlang.md @@ -0,0 +1,566 @@ +--- +title: "Installing Erlang" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Installing Erlang" + identifier: "installing_source_erlang" + weight: 301 + parent: "installing_source" +toc: true +aliases: + - /riak-docs/riak/2.2.6/ops/building/installing/erlang + - /riak-docs/riak/kv/2.2.6/ops/building/installing/erlang + - /riak-docs/riak/2.2.6/installing/source/erlang/ + - /riak-docs/riak/kv/2.2.6/installing/source/erlang/ +--- + +[install index]: {{<baseurl>}}riak/kv/2.2.6/setup/installing +[security basics]: {{<baseurl>}}riak/kv/2.2.6/using/security/basics + +Pre-packaged versions of Riak include an Erlang installation. If you are building Riak from source, you will need to install [Basho's patched version of Erlang](https://files.tiot.jp/riak/erlang/otp_src_R16B02-basho10.tar.gz). **If you do not use this version of Erlang, you will not be able to use Riak's [security features][security basics].** + +> **Note on Official Support** +> +> Please note that only packaged Riak KV installs are officially supported. Visit [Installing Riak KV][install index] for installing a supported Riak package. + +## Prerequisites + +#### Contents + +* [kerl](#kerl-prerequisites) +* [Debian/Ubuntu](#debian-ubuntu-prerequisites) +* [FreeBSD/Solaris](#freebsd-solaris-prerequisites) +* [Mac OS X](#mac-os-x-prerequisites) +* [RHEL/CentOS](#rhel-centos-prerequisites) + +To build and install Erlang you must have a GNU-compatible build system and these tools: + +**Unpacking** + +* [GNU unzip](http://www.gzip.org/) or a modern uncompressing utility. +* [GNU Tar](http://www.gnu.org/software/tar/) for working with GNU TAR archives. + +**Building** + +* [autoconf](http://www.gnu.org/software/autoconf/autoconf.html): generates configure scripts. +* [make](http://www.gnu.org/software/make/): generates executables and other non-source files of a program. +* [gcc](https://gcc.gnu.org/): for compiling C. +* [ncurses](http://www.gnu.org/software/ncurses/): for terminal-based interfaces. +* [OpenSSL](https://www.openssl.org/): toolkit that implements SSL and TSL protocols. +* [Java SE JDK](http://www.oracle.com/technetwork/java/javase/downloads/index.html): platform for deploying Java. + + +## kerl Prerequisites + +[kerl](https://github.com/yrashk/kerl) is the quickest way to install different versions of Erlang on most systems. + +Install kerl by running the following command: + +```bash +curl -O https://raw.githubusercontent.com/spawngrid/kerl/master/kerl +chmod a+x kerl +``` + +If you are using Mac OS X, FreeBSD, or Solaris, see the following sections for additional requirements before building with kerl. + +Otherwise, continue with [Installing with kerl](#installing-with-kerl). + +### Configuring kerl on FreeBSD/Solaris + +Start by by creating a `~/.kerlrc` file: + +```bash +touch ~/.kerlrc +``` + +Next add the following contents to your `~/.kerlrc` file: + +```shell +KERL_CONFIGURE_OPTIONS="--disable-hipe --enable-smp-support --enable-threads + --enable-kernel-poll --without-odbc --enable-darwin-64bit" +``` + +Then check for the presence of autoconf by running: + +```shell +which autoconf +``` +If this returns `autoconf not found`, install autoconf by running: + +```shell +sudo pkg update +sudo pkg install autoconf +``` + +Once you've configured kerl and installed autoconf continue with [Installing with kerl](#installing-with-kerl). + + +### Configuring kerl on Mac OS X + +To compile Erlang as 64-bit on Mac OS X you need to instruct kerl to pass the correct flags to the `configure` command. + +Start by by creating a `~/.kerlrc` file: + +```bash +touch ~/.kerlrc +``` + +Next add the following contents to your `~/.kerlrc` file: + +```shell +KERL_CONFIGURE_OPTIONS="--disable-hipe --enable-smp-support --enable-threads + --enable-kernel-poll --without-odbc --enable-darwin-64bit" +``` + +On OS X 10.9 (Mavericks) or later, you may need to install [autoconf](https://www.gnu.org/software/autoconf/). + +Check for the presence of autoconf by running: + +```shell +which autoconf +``` + +If this returns `autoconf not found`, install autoconf with: + +With Homebrew: + +```shell +brew install autoconf +``` + +Or with curl: + +```shell +curl -O http://ftp.gnu.org/gnu/autoconf/autoconf-2.69.tar.gz +tar zxvf autoconf-2.69.tar.gz +cd autoconf-2.69 +./configure && make && sudo make install +``` + +Once you've configured kerl and installed autoconf continue with [Installing with kerl](#installing-with-kerl). + + + +## Debian/Ubuntu Prerequisites + +### Dependencies + +To install the required dependencies run the following `apt-get` commands: + +```bash +sudo apt-get update +sudo apt-get install build-essential autoconf libncurses5-dev openssl libssl-dev fop xsltproc unixodbc-dev git +``` + +### GUI Dependencies + +If you're using a graphical environment and want to use Erlang's GUI utilities, you will need to install additional dependencies. + +> **Note on build output** +> +>These packages are not required for operation of a Riak node. +Notes in the build output about missing support for wxWidgets can be +safely ignored when installing Riak in a typical non-graphical server +environment. + +To install packages for graphics support use the following `apt-get` command: + +```bash +sudo apt-get install libwxbase2.8 libwxgtk2.8-dev libqt4-opengl-dev +``` + +### Next Steps + +Once you've installed the prerequisites, continue with [Installing on Debian/Ubuntu](#installing-on-debian-ubuntu). + + + +## FreeBSD/Solaris Prerequisites + +### Dependencies + +To install the required dependencies run the following `pkg` command: + +```bash +sudo pkg update +sudo pkg install gcc autoconf gmake flex +``` + +### GUI Dependencies + +If you're using a graphical environment and want to use Erlang's GUI utilities, you will need to install additional dependencies. + +To install packages for graphics support use the following `pkg` command: + +```bash +sudo pkg install wx28-gtk2-2.8.12_4 +``` + +### Next Steps + +Once you've installed the prerequisites, continue with [Installing on FreeBSD/Solaris](#installing-on-freebsd-solaris). + + + +## Mac OS X Prerequisites + +* [XCode Developer Tools](http://developer.apple.com/) - Apple Software Development Tools. +* [Homebrew](http://brew.sh/) (*optional*) - Package Manager. + +First install [XCode Developer Tools](http://developer.apple.com/). XCode is a set software development tools for developing on OS X. + +We also recommend installing [Homebrew](http://brew.sh/), a package manager for OS X. Homebrew is not required to install Erlang and is optional. + +Next, if you are running OS X 10.9 (Mavericks) or later, you may need to +install [autoconf](https://www.gnu.org/software/autoconf/). To check for +the presence of autoconf run: + +```bash +which autoconf +``` + +If this returns `autoconf not found`, install autoconf with: + +With Homebrew: + +```bash +brew install autoconf +``` + +Or with curl: + +```bash +curl -O http://ftp.gnu.org/gnu/autoconf/autoconf-2.69.tar.gz +tar zxvf autoconf-2.69.tar.gz +cd autoconf-2.69 +./configure && make && sudo make install +``` + +Once you've installed the prerequisites continue with [Installing on Mac OS X](#installing-on-mac-os-x). + +## RHEL/CentOS Prerequisites + +### Dependencies + +To install the required dependencies run the following `yum` command: + +```bash +sudo yum install gcc gcc-c++ glibc-devel make ncurses-devel openssl-devel autoconf java-1.8.0-openjdk-devel git +``` + +### GUI Dependencies + +If you're using a graphical environment and want to use Erlang's GUI utilities, you will need to install additional dependencies. + +To install packages for graphics support use the following `blank` command: + +```bash +sudo yum install wxBase.x86_64 +``` + +### Next Steps + +Once you've installed the prerequisites, continue with [Installing on RHEL/CentOS](#installing-on-rhel-centos). + + + +## Installation + +* [Installing with kerl](#installing-with-kerl) +* [Installing on Debian/Ubuntu](#installing-on-debian-ubuntu) +* [Installing on FreeBSD/Solaris](#installing-on-freebsd-solaris) +* [Installing on Mac OS X](#installing-on-mac-os-x) +* [Installing on RHEL/CentOS](#installing-on-rhel-centos) + +## Installing with kerl + +First make sure you have installed the necessary dependencies and prerequisites found in [kerl Prerequisites](#kerl-prerequisites). + +With [kerl](https://github.com/yrashk/kerl) installed, you can install Basho's recommended version of +Erlang [from Github](https://github.com/basho/otp) using the following +command: + +```bash +./kerl build git git://github.com/basho/otp.git OTP_R16B02_basho10 R16B02-basho10 +``` + +This builds the Erlang distribution and performs all of the steps +required to manually install Erlang for you. + +After Erlang is successfully built, you can install the build as follows: + +```bash +./kerl install R16B02-basho10 ~/erlang/R16B02-basho10 +. ~/erlang/R16B02-basho10/activate +``` + +The last line activates the Erlang build that was just installed into +`~/erlang/R16B02-basho10`. + +> See the kerl [README](https://github.com/yrashk/kerl) for more details on the available commands. + +Confirm Erlang installed to the correct location: + +```bash +which erl +``` + +And start Erlang from your terminal with: + +```bash +erl +``` + + +## Installing on Debian/Ubuntu + +First make sure you have installed the necessary dependencies found in [Debian/Ubuntu Prerequisites](#debian-ubuntu-prerequisites). + +Next download [Basho's patched version of Erlang](https://files.tiot.jp/riak/erlang/otp_src_R16B02-basho10.tar.gz). + +Using `wget`: + +```bash +wget https://files.tiot.jp/riak/erlang/otp_src_R16B02-basho10.tar.gz +``` + +Then unpack the download with: + +```bash +tar zxvf otp_src_R16B02-basho10.tar.gz +``` + +Next `cd` into the unpacked directory, build and install Erlang with: + +```bash +cd OTP_R16B02_basho10 +./otp_build autoconf +./configure && make && sudo make install +``` + +Confirm Erlang installed to the correct location: + +```bash +which erl +``` + +And start Erlang from your terminal with: + +```bash +erl +``` + +## Installing on FreeBSD/Solaris + +First make sure you installed the necessary dependencies in [FreeBSD/Solaris Prerequisites](#freebsd-solaris-prerequisites). + +Next download [Basho's patched version of Erlang](https://files.tiot.jp/riak/erlang/otp_src_R16B02-basho10.tar.gz): + +```bash +ftp https://files.tiot.jp/riak/erlang/otp_src_R16B02-basho10.tar.gz +``` + +Then unpack the download with: + +```bash +tar zxvf otp_src_R16B02-basho10.tar.gz +``` + +Next `cd` into the unpacked directory, build and install Erlang with: + +```bash +cd OTP_R16B02_basho10 +./otp_build autoconf +./configure && gmake && sudo gmake install +``` + +Confirm Erlang installed to the correct location by running: + +```bash +which erl +``` + +And start Erlang from your terminal with: + +```bash +erl +``` + + +## Installing on Mac OS X + +First make sure you have installed the necessary dependencies found in [Mac OS X Prerequisites](#mac-os-x-prerequisites). + +You can install Erlang in several ways on OS X: + +* [From Source](#installing-on-mac-os-x-from-source) +* [Homebrew](#installing-on-mac-os-x-with-homebrew) +* [MacPorts](#installing-on-mac-os-x-with-macports) + +## Installing on Mac OS X from Source + +Next download [Basho's patched version of Erlang](https://files.tiot.jp/riak/erlang/otp_src_R16B02-basho10.tar.gz): + +```bash +curl -O https://files.tiot.jp/riak/erlang/otp_src_R16B02-basho10.tar.gz +``` + +Then unpack the download with: + +```bash +tar zxvf otp_src_R16B02-basho10.tar.gz +``` + +Follow the steps below to configure Erlang for your operating system. + +#### Configuring Erlang on Mavericks (OS X 10.9), Mountain Lion (OS X 10.8), and Lion (OS X 10.7) + +If you're on Mavericks (OS X 10.9), Mountain Lion (OS X 10.8), or Lion +(OS X 10.7) you can use LLVM (the default) or GCC to compile Erlang. + +Using LLVM: + +```bash +CFLAGS=-O0 ./configure --disable-hipe --enable-smp-support --enable-threads \ +--enable-kernel-poll --enable-darwin-64bit +``` + +Or if you prefer GCC: + +```bash +CC=gcc-4.2 CPPFLAGS='-DNDEBUG' MAKEFLAGS='-j 3' \ +./configure --disable-hipe --enable-smp-support --enable-threads \ +--enable-kernel-poll --enable-darwin-64bit +``` + +#### Configuring Erlang on Snow Leopard (OS X 10.6) + +If you're on Snow Leopard (OS X 10.6) or Leopard (OS X 10.5) with an +Intel processor: + +```bash +./configure --disable-hipe --enable-smp-support --enable-threads \ +--enable-kernel-poll --enable-darwin-64bit +``` + +#### Configuring Erlang on older versions of OS X + +If you're on a non-Intel processor or older version of OS X: + +```bash +./configure --disable-hipe --enable-smp-support --enable-threads \ +--enable-kernel-poll +``` + +After you've configured your system `cd` into the unpacked directory, build and install Erlang with: + +```bash +cd OTP_R16B02_basho10 +./otp_build autoconf +./configure && make && sudo make install +``` + +Confirm Erlang installed to the correct location by running: + +```bash +which erl +``` + +And start Erlang from your terminal with: + +```bash +erl +``` + +## Installing on Mac OS X with Homebrew + +To install Erlang with Homebrew, use this command: + +```bash +brew install erlang +``` + +Confirm Erlang installed to the correct location by running: + +```bash +which erl +``` + +And start Erlang from your terminal with: + +```bash +erl +``` + +## Installing on Mac OS X with MacPorts + +Installing with MacPorts: + +```bash +port install erlang +ssl +``` + +Confirm Erlang installed to the correct location by running: + +```bash +which erl +``` + +And start Erlang from your terminal with: + +```bash +erl +``` + +## Installing on RHEL/CentOS + +First make sure you have installed the necessary dependencies and prerequisites found in [RHEL/CentOS Prerequisites](#rhel-centos-prerequisites). + +Using `wget`: + +```bash +wget https://files.tiot.jp/riak/erlang/otp_src_R16B02-basho10.tar.gz +``` + +Then unpack the download with: + +```bash +tar zxvf otp_src_R16B02-basho10.tar.gz +``` + +Next `cd` into the unpacked directory, build and install Erlang with: + +```bash +cd OTP_R16B02_basho10 +./otp_build autoconf +./configure && make && sudo make install +``` + +> **Note for RHEL6/CentOS6** +> +> In certain versions of RHEL6 and CentO6 the `openSSL-devel` package +ships with Elliptical Curve Cryptography partially disabled. To +communicate this to Erlang and prevent compile- and run-time errors, the +environment variable `CFLAGS="-DOPENSSL_NO_EC=1"` needs to be added to +Erlang's `./configure` call. +> +> The full `make` invocation then becomes +> +> ```bash +CFLAGS="-DOPENSSL_NO_EC=1" ./configure && make && sudo make install +``` + +Confirm Erlang installed to the correct location: + +```bash +which erl +``` + +And start Erlang from your terminal with: + +```bash +erl +``` diff --git a/content/riak/kv/2.2.6/setup/installing/source/jvm.md b/content/riak/kv/2.2.6/setup/installing/source/jvm.md new file mode 100644 index 0000000000..07d4b0d23d --- /dev/null +++ b/content/riak/kv/2.2.6/setup/installing/source/jvm.md @@ -0,0 +1,51 @@ +--- +title: "Installing the JVM" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Installing the JVM" + identifier: "installing_source_jvm" + weight: 302 + parent: "installing_source" +toc: true +aliases: + - /riak-docs/riak/2.2.6/ops/building/installing/jvm + - /riak-docs/riak/kv/2.2.6/ops/building/installing/jvm + - /riak-docs/riak/2.2.6/ops/building/installing/Installing-the-JVM + - /riak-docs/riak/kv/2.2.6/ops/building/installing/Installing-the-JVM + - /riak-docs/riak/2.2.6/installing/source/jvm/ + - /riak-docs/riak/kv/2.2.6/installing/source/jvm/ +--- + +[usage search]: {{<baseurl>}}riak/kv/2.2.6/developing/usage/search + +If you are using [Riak Search 2.0][usage search], codename Yokozuna, +you will need to install **Java 1.6 or later** to run [Apache +Solr](https://lucene.apache.org/solr/), the search platform that powers +Riak Search. + +We recommend using Oracle's [JDK +7u25](http://www.oracle.com/technetwork/java/javase/7u25-relnotes-1955741.html). +Installation packages can be found on the [Java SE 7 Downloads +page](http://www.oracle.com/technetwork/java/javase/downloads/java-archive-downloads-javase7-521261.html#jdk-7u25-oth-JPR) +and instructions on the [documentation +page](http://www.oracle.com/technetwork/java/javase/documentation/index.html). + +## Installing Solr on OS X + +If you're using Riak Search on Mac OS X, you may see the following +error: + +```java +java.net.MalformedURLException: Local host name unknown: <YOUR_HOST_NAME> +``` + +If you encounter this error, we recommend manually setting the hostname +for `localhost` using +[scutil](https://developer.apple.com/library/mac/documentation/Darwin/Reference/ManPages/man8/scutil.8.html). + +```bash +scutil --set HostName "localhost" +``` diff --git a/content/riak/kv/2.2.6/setup/installing/suse.md b/content/riak/kv/2.2.6/setup/installing/suse.md new file mode 100644 index 0000000000..57aed39cec --- /dev/null +++ b/content/riak/kv/2.2.6/setup/installing/suse.md @@ -0,0 +1,43 @@ +--- +title_supertext: "Installing on" +title: "SUSE" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "SUSE" + identifier: "installing_suse" + weight: 307 + parent: "installing" +toc: false +aliases: + - /riak-docs/riak/2.2.6/ops/building/installing/Installing-on-SUSE + - /riak-docs/riak/kv/2.2.6/ops/building/installing/Installing-on-SUSE + - /riak-docs/riak/2.2.6/installing/suse/ + - /riak-docs/riak/kv/2.2.6/installing/suse/ +--- + +[install verify]: {{<baseurl>}}riak/kv/2.2.6/setup/installing/verify + +Riak KV can be installed on OpenSuse and SLES systems using a binary package. The following steps have been tested to work with Riak on +the following x86/x86_64 flavors of SuSE: + +* SLES11-SP1 +* SLES11-SP2 +* SLES11-SP3 +* SLES11-SP4 +* OpenSUSE 11.2 +* OpenSUSE 11.3 +* OpenSUSE 11.4 + +## Installing with rpm + +```bash +wget https://files.tiot.jp/riak/kv/2.2/2.2.6/sles/11/riak-2.2.6-1.SLES11.x86_64.rpm +sudo rpm -Uvh riak-2.2.6-1.SLES11.x86_64.rpm +``` + +## Next Steps + +Now that Riak is installed, check out [Verifying a Riak Installation][install verify]. diff --git a/content/riak/kv/2.2.6/setup/installing/verify.md b/content/riak/kv/2.2.6/setup/installing/verify.md new file mode 100644 index 0000000000..ad9c0e03b7 --- /dev/null +++ b/content/riak/kv/2.2.6/setup/installing/verify.md @@ -0,0 +1,164 @@ +--- +title: "Verifying a Riak KV Installation" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Verifying an Installation" + identifier: "installing_verify" + weight: 311 + parent: "installing" +toc: true +aliases: + - /riak-docs/riak/2.2.6/ops/installing/Post-Installation + - /riak-docs/riak/kv/2.2.6/ops/installing/Post-Installation + - /riak-docs/riak/2.2.6/installing/verify-install/ + - /riak-docs/riak/kv/2.2.6/installing/verify-install/ +--- + +[client libraries]: {{<baseurl>}}riak/kv/2.2.6/developing/client-libraries +[perf open files]: {{<baseurl>}}riak/kv/2.2.6/using/performance/open-files-limit +[cluster ops bucket types]: {{<baseurl>}}riak/kv/2.2.6/using/cluster-operations/bucket-types +[cluster ops inspect node]: {{<baseurl>}}riak/kv/2.2.6/using/cluster-operations/inspecting-node + +After you've installed Riak KV, we recommend checking the liveness of +each node to ensure that requests are being properly served. + +In this document, we cover ways of verifying that your Riak nodes are operating +correctly. After you've determined that your nodes are functioning and you're +ready to put Riak KV to work, be sure to check out the resources in the +**Now What?** section below. + +## Starting a Riak Node + +> **Note about source installations** +> +> To start a Riak KV node that was installed by compiling the source code, you +can add the Riak KV binary directory from the installation directory you've +chosen to your `PATH`. +> +> For example, if you compiled Riak KV from source in +the `/home/riak` directory, then you can add the binary directory +(`/home/riak/rel/riak/bin`) to your `PATH` so that Riak KV commands can be used in the same manner as with a packaged installation. + +To start a Riak node, use the `riak start` command: + +```bash +riak start +``` + +A successful start will return no output. If there is a problem starting the +node, an error message is printed to standard error. + +To run Riak with an attached interactive Erlang console: + +```bash +riak console +``` + +A Riak node is typically started in console mode as part of debugging or +troubleshooting to gather more detailed information from the Riak startup +sequence. Note that if you start a Riak node in this manner, it is running as +a foreground process that will be exited when the console is closed. + +You can close the console by issuing this command at the Erlang prompt: + +```erlang +q(). +``` + +Once your node has started, you can initially check that it is running with +the `riak ping` command: + +```bash +riak ping +``` + +The command will respond with `pong` if the node is running or `Node <nodename> not responding to pings` if it is not. + +> **Open Files Limit** +> +> As you may have noticed, if you haven't adjusted your open files limit (`ulimit -n`), Riak will warn you at startup. You're advised +to increase the operating system default open files limit when running Riak. +You can read more about why in the [Open Files Limit][perf open files] documentation. + +## Does it work? + +One convenient means of testing the readiness of an individual Riak node and +its ability to read and write data is with the `riak-admin test` command: + +```bash +riak-admin test +``` + +Successful output from `riak-admin test` looks like this: + +```text +Attempting to restart script through sudo -H -u riak +Successfully completed 1 read/write cycle to '<nodename>' +``` + +You can also test whether Riak is working by using the `curl` command-line +tool. When you have Riak running on a node, try this command to retrieve +the the properties associated with the [bucket type][cluster ops bucket types] `test`: + +```bash +curl -v http://127.0.0.1:8098/types/default/props +``` + +Replace `127.0.0.1` in the example above with your Riak node's IP address or +fully qualified domain name, and you should get a response that looks like this: + +``` +* About to connect() to 127.0.0.1 port 8098 (#0) +* Trying 127.0.0.1... connected +* Connected to 127.0.0.1 (127.0.0.1) port 8098 (#0) +> GET /riak/test HTTP/1.1 +> User-Agent: curl/7.21.6 (x86_64-pc-linux-gnu) +> Host: 127.0.0.1:8098 +> Accept: */* +> +< HTTP/1.1 200 OK +< Vary: Accept-Encoding +< Server: MochiWeb/1.1 WebMachine/1.9.0 (someone had painted it blue) +< Date: Wed, 26 Dec 2012 15:50:20 GMT +< Content-Type: application/json +< Content-Length: 422 +< +* Connection #0 to host 127.0.0.1 left intact +* Closing connection #0 +{"props":{"name":"test","allow_mult":false,"basic_quorum":false, + "big_vclock":50,"chash_keyfun":{"mod":"riak_core_util", + "fun":"chash_std_keyfun"},"dw":"quorum","last_write_wins":false, + "linkfun":{"mod":"riak_kv_wm_link_walker","fun":"mapreduce_linkfun"}, + "n_val":3,"notfound_ok":true,"old_vclock":86400,"postcommit":[],"pr":0, + "precommit":[],"pw":0,"r":"quorum","rw":"quorum","small_vclock":50, + "w":"quorum","young_vclock":20}} +``` + +The output above shows a successful response (`HTTP 200 OK`) and additional +details from the verbose option. The response also contains the bucket +properties for the `default` bucket type. + +## Riaknostic + +It is a good idea to verify some basic configuration and general health +of the Riak node after installation by using Riak's built-in diagnostic +utility [Riaknostic](http://riaknostic.basho.com/). + +To start up Riaknostic, ensure that Riak is running on the node and issue the following command: + +```bash +riak-admin diag +``` + +More extensive documentation for Riaknostic can be found in the [Inspecting a Node][cluster ops inspect node] guide. + +## Now what? + +You have a working Riak node! + +From here you might want to check out the following resources: + +* [Client Libraries][client libraries] to use Riak with your favorite programming language diff --git a/content/riak/kv/2.2.6/setup/installing/windows-azure.md b/content/riak/kv/2.2.6/setup/installing/windows-azure.md new file mode 100644 index 0000000000..b0d170ccd8 --- /dev/null +++ b/content/riak/kv/2.2.6/setup/installing/windows-azure.md @@ -0,0 +1,185 @@ +--- +title_supertext: "Installing on" +title: "Windows Azure" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Windows Azure" + identifier: "installing_windows_azure" + weight: 308 + parent: "installing" +toc: true +aliases: + - /riak-docs/riak/2.2.6/ops/building/installing/Installing-on-Windows-Azure + - /riak-docs/riak/kv/2.2.6/ops/building/installing/Installing-on-Windows-Azure + - /riak-docs/riak/2.2.6/installing/windows-azure/ + - /riak-docs/riak/kv/2.2.6/installing/windows-azure/ +--- + +## Creating CentOS VMs + +You will need to sign up for the Windows Azure Virtual Machines preview feature in order to create a virtual machine. You can also sign up for a free trial account if you do not have a Windows Azure account. + +1. Navigate to [https://account.windowsazure.com](https://account.windowsazure.com/) and sign in with your Windows Azure account. + +2. Click "preview features" to view the available previews. + + ![]({{<baseurl>}}images/antares-iaas-preview-01.png) + +3. Scroll down to Virtual Machines & Virtual Networks and click "try it now". + + ![]({{<baseurl>}}images/antares-iaas-preview-02.png) + +4. Select your subscription and click the check. + + ![]({{<baseurl>}}images/antares-iaas-preview-04.png) + +### Create a virtual machine running CentOS Linux + +1. Login to the Windows Azure (Preview) Management Portal using your Windows Azure account. + +2. In the Management Portal, at the bottom left of the web page, click "+New", click "Virtual Machine", and then click "From Gallery". + + ![]({{<baseurl>}}images/createvm_small.png) + +3. Select a CentOS virtual machine image from "Platform Images", and then click the next arrow at the bottom right of the page. + + ![]({{<baseurl>}}images/vmconfiguration0.png) + +4. On the VM Configuration page, provide the following information: + - Provide a "Virtual Machine Name", such as "testlinuxvm". + - Specify a "New User Name", such as "newuser", which will be added to the Sudoers list file. **Do NOT** use the username "riak", as it may conflict with the installation package. + - In the "New Password" box, type a strong password. + - In the "Confirm Password" box, retype the password. + - Select the appropriate "Size" from the drop down list. + - Click the next arrow to continue. + + ![]({{<baseurl>}}images/vmconfiguration1.png) + +5. On the VM Mode page, provide the following information: + - **If this is the first node**, select the "STANDALONE VIRTUAL MACHINE" radio button. **Otherwise**, select the "CONNECT TO EXISTING VIRTUAL MACHINE" radio button, and select the first node in the drop down list. + - In the "DNS Name" box, type a valid DNS address, e.g "testlinuxvm". + - In the "Storage Account" box, select "Use Automatically Generated Storage Account". + - In the "Region/Affinity Group/Virtual Network" box, select a region where this virtual image will be hosted. + - Click the next arrow to continue. + + ![]({{<baseurl>}}images/vmconfiguration2.png) + +6. On the VM Options page, select "(none)" in the "Availability Set" box. Click the check mark to continue. + + ![]({{<baseurl>}}images/vmconfiguration3.png) + +7. Wait while Windows Azure prepares your virtual machine. + +### Configure Endpoints + +Once the virtual machine is created you must configure endpoints in order to remotely connect. + +1. In the Management Portal, click "Virtual Machines", then click the name of your new VM, then click "Endpoints". + +2. **If this is the first node**, click "Add Endpoint", leave "Add Endpoint" checked, hit the right arrow and fill out the next form as follows: + - Name: https + - Protocol: leave set to 'TCP' + - Public Port: 443 + - private Port: 8069 + +## Connect to CentOS VMs using PuTTY or SSH + +When the virtual machine has been provisioned and the endpoints configured you can connect to it using SSH or PuTTY. + +### Connecting Using SSH + +**For Linux & Mac Users:** + +```bash +ssh newuser@testlinuxvm.cloudapp.net -o ServerAliveInterval=180 +``` +Enter the user's password. + +**For Windows Users, use PuTTY:** + +If you are using a Windows computer, connect to the VM using PuTTY. PuTTY can be downloaded from the [PuTTY Download Page](http://www.chiark.greenend.org.uk/~sgtatham/putty/download.html). + +1. Download and save putty.exe to a directory on your computer. Open a command prompt, navigate to that folder, and execute putty.exe. + +2. Enter the SSH DETAILS as found on the Node's Dashboard, i.e., "testlinuxvm.cloudapp.net" for the Host Name and "22" for the Port. + + ![]({{<baseurl>}}images/putty.png) + +## Install Riak and configure using a shell script + +1. **On each node**, once you've connected using the steps above, execute: + +```bash +sudo su - +curl -s https://raw.githubusercontent.com/basho/riak_on_azure/1.0/azure_install_riak.sh | sh +``` + +## Configure Riak using Riak Control + +You can either use Riak Control or the command line to add nodes to your Riak Cluster. If you wish to add nodes via the command line, skip down to the section entitled "Configure Riak using Command Line" + +1. Find the dns name and "Deployment ID" in the virtual machine dashboard of the VM you created the https endpoint for. For Example: + - **dns:** basho-example.cloudapp.net + - **Deployment ID:** 7ea145743aeb4402a088da1234567890 + +2. Visit https://dns-name.cloudapp.net/admin in your browser + +3. Enter 'admin' as the username, and the "Deployment ID" as the password. + +4. Select 'Cluster' on the left. + +5. Add VMs which also have the Riak software installed and configured by entering riak@yourhostnamehere in the input box, and clicking 'Add Node'. Use the short name of each vm, not the DNS name. For Example: + - riak@basho-centos1 + +You now have a Riak cluster on Azure + +## Configure Riak using Command Line + +If you have already followed the instructions in the section "Configure Riak using Riak Control", skip this section. + +First, SSH into the second (and subsequent nodes) and execute: + +```bash +riak-admin cluster join riak@yourhostnamehere +``` + +(Where 'yourhostnamehere' is the short name of the **first node** in your cluster) + +(NOTE: The host you choose can actually be any host that has already joined the cluster. The first host has no special significance, but it's important not to attempt to join to a node that hasn't joined a cluster yet. Doing this would create a second cluster; thus we use the first node for these instructions.) + +After all the nodes have have been joined to the first node via the previous command, connect to any of the nodes via SSH or PuTTY and execute the following: + +```bash +riak-admin cluster plan +``` + +Verify all the nodes are listed as expected. If the cluster plan looks good: + +```bash +riak-admin cluster commit +``` + +To check the status of clustering use: + +```bash +riak-admin member-status +``` + +You now have a Riak cluster on Azure + +## Load Test Data + +Execute on any one of the nodes: + +```bash +curl -s http://rekon.basho.com | sh +``` + +Visit DNS address listed on the dashboard, at the port we opened as an endpoint: + +``` +http://testlinuxvm.cloudapp.net:8098/riak/rekon/go +``` diff --git a/content/riak/kv/2.2.6/setup/planning.md b/content/riak/kv/2.2.6/setup/planning.md new file mode 100644 index 0000000000..7f38ff8c4d --- /dev/null +++ b/content/riak/kv/2.2.6/setup/planning.md @@ -0,0 +1,55 @@ +--- +title: "Planning Overview" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Planning" + identifier: "planning" + weight: 100 + parent: "setup_index" +toc: true +--- + +[plan start]: ./start +[plan backend]: ./backend +[plan cluster capacity]: ./cluster-capacity +[plan bitcask capacity]: ./bitcask-capacity-calc +[plan backend bitcask]: ./backend/bitcask +[plan best practices]: ./best-practices +[plan future]: ./future + +## In This Section + +### [Start Planning][plan start] + +Steps and recommendations for designing and configuring a Riak KV cluster. + +[Learn More >>][plan start] + +### [Choosing a Backend][plan backend] + +Information on choosing the right storage backend for your Riak KV cluster. + +[Learn More >>][plan backend] + +### [Cluster Capacity Planning][plan cluster capacity] + +Outlines variables (such as memory requirements) to keep in mind when planning your Riak KV cluster. + +[Learn More >>][plan cluster capacity] + +### [Bitcask Capacity Calculator][plan bitcask capacity] + +A calculator that will assist you in sizing your cluster if you plan to use the default ([Bitcask][plan backend bitcask]) storage back end. + +[Learn More >>][plan bitcask capacity] + +### [Scaling & Operating Best Practices][plan best practices] + +A set of best practices that will enable you to improve performance and reliability at all stages in the life of your Riak KV cluster. + +[Learn More >>][plan best practices] + + diff --git a/content/riak/kv/2.2.6/setup/planning/backend.md b/content/riak/kv/2.2.6/setup/planning/backend.md new file mode 100644 index 0000000000..b201e44399 --- /dev/null +++ b/content/riak/kv/2.2.6/setup/planning/backend.md @@ -0,0 +1,54 @@ +--- +title: "Choosing a Backend" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Choosing a Backend" + identifier: "planning_choose_backend" + weight: 102 + parent: "planning" +toc: true +aliases: + - /riak-docs/riak/2.2.6/ops/building/planning/backends/ + - /riak-docs/riak/kv/2.2.6/ops/building/planning/backends/ +--- + +[plan backend bitcask]: {{<baseurl>}}riak/kv/2.2.6/setup/planning/backend/bitcask +[plan backend leveldb]: {{<baseurl>}}riak/kv/2.2.6/setup/planning/backend/leveldb +[plan backend memory]: {{<baseurl>}}riak/kv/2.2.6/setup/planning/backend/memory +[plan backend multi]: {{<baseurl>}}riak/kv/2.2.6/setup/planning/backend/multi +[dev api backend]: {{<baseurl>}}riak/kv/2.2.6/developing/api/backend + +Pluggable storage backends are a key feature of Riak KV. They enable you to +choose a low-level storage engine that suits specific operational needs. +For example, if your use case requires maximum throughput, data +persistence, and a bounded keyspace, then Bitcask is a good choice. On +the other hand, if you need to store a large number of keys or to use +secondary indexes, LevelDB is likely a better choice. + +The following backends are supported: + +* [Bitcask][plan backend bitcask] +* [LevelDB][plan backend leveldb] +* [Memory][plan backend memory] +* [Multi][plan backend multi] + +Riak KV supports the use of custom storage backends as well. See the +storage [Backend API][dev api backend] for more details. + +Feature or Characteristic |Bitcask|LevelDB|Memory| +:----------------------------------------------|:-----:|:-----:|:----:| +Default Riak KV backend |✓ | | | +Persistent |✓ |✓ | | +Keyspace in RAM |✓ | |✓ | +Keyspace can be greater than available RAM | |✓ | | +Keyspace loaded into RAM on startup<sup>1</sup>|✓ | | | +Objects in RAM | | |✓ | +Object expiration |✓ | |✓ | +Secondary indexes | |✓ |✓ | +Tiered storage + +<sup>1</sup> Noted here since this can affect Riak start times for large +keyspaces. diff --git a/content/riak/kv/2.2.6/setup/planning/backend/bitcask.md b/content/riak/kv/2.2.6/setup/planning/backend/bitcask.md new file mode 100644 index 0000000000..9f241fdebf --- /dev/null +++ b/content/riak/kv/2.2.6/setup/planning/backend/bitcask.md @@ -0,0 +1,990 @@ +--- +title: "Bitcask" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Bitcask" + identifier: "planning_backend_bitcask" + weight: 100 + parent: "planning_choose_backend" +toc: true +aliases: + - /riak-docs/riak/2.2.6/ops/advanced/backends/bitcask/ + - /riak-docs/riak/kv/2.2.6/ops/advanced/backends/bitcask/ +--- + +[github bitcask]: https://github.com/basho/bitcask +[bitcask design pdf]: http://basho.com/assets/bitcask-intro.pdf +[use admin riak cli]: {{<baseurl>}}riak/kv/2.2.6/using/admin/riak-cli +[config reference]: {{<baseurl>}}riak/kv/2.2.6/configuring/reference +[glossary vnode]: {{<baseurl>}}riak/kv/2.2.6/learn/glossary/#vnode +[learn clusters]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/clusters +[plan backend multi]: {{<baseurl>}}riak/kv/2.2.6/setup/planning/backend/multi +[usage search]: {{<baseurl>}}riak/kv/2.2.6/developing/usage/search + +[glossary aae]: {{<baseurl>}}riak/kv/2.2.6/learn/glossary/#active-anti-entropy-aae +[perf open files]: {{<baseurl>}}riak/kv/2.2.6/using/performance/open-files-limit + +[plan bitcask capacity]: {{<baseurl>}}riak/kv/2.2.6/setup/planning/bitcask-capacity-calc +[usage delete objects]: {{<baseurl>}}riak/kv/2.2.6/developing/usage/deleting-objects + +[Bitcask][github bitcask] is an Erlang application that provides an API for storing and retrieving key/value data using log-structured hash tables that provide very fast access. The [design][bitcask design pdf] of Bitcask was inspired, in part, by log-structured filesystems and log file merging. + +## Bitcask's Strengths + +* **Low latency per item read or written** + + This is due to the write-once, append-only nature of Bitcask + database files. + +* **High throughput, especially when writing an incoming stream of + random items** + + Write operations to Bitcask generally saturate I/O and disk + bandwidth, which is a good thing from a performance perspective. + This saturation occurs for two reasons: because (1) data that is + written to Bitcask doesn't need to be ordered on disk, and (2) the + log-structured design of Bitcask allows for minimal disk head + movement during writes. + +* **Ability to handle datasets larger than RAM without degradation** + + Access to data in Bitcask involves direct lookup from an in-memory + hash table. This makes finding data very efficient, even when + datasets are very large. + +* **Single seek to retrieve any value** + + Bitcask's in-memory hash table of keys points directly to locations + on disk where the data lives. Bitcask never uses more than one disk + seek to read a value and sometimes even that isn't necessary due to + filesystem caching done by the operating system. + +* **Predictable lookup _and_ insert performance** + + For the reasons listed above, read operations from Bitcask have + fixed, predictable behavior. This is also true of writes to Bitcask + because write operations require, at most, one seek to the end of + the current open file followed by and append to that file. + +* **Fast, bounded crash recovery** + + Crash recovery is easy and fast with Bitcask because Bitcask files + are append only and write once. The only items that may be lost are + partially written records at the tail of the last file that was + opened for writes. Recovery operations need to review only the last + record or two written and verify CRC data to ensure that the data is + consistent. + +* **Easy Backup** + + In most systems, backup can be very complicated. Bitcask simplifies + this process due to its append-only, write-once disk format. Any + utility that archives or copies files in disk-block order will + properly back up or copy a Bitcask database. + +## Weaknesses + +* Keys must fit in memory + + Bitcask keeps all keys in memory at all times, which means that your + system must have enough memory to contain your entire keyspace, plus + additional space for other operational components and operating- + system-resident filesystem buffer space. + +## Installing Bitcask + +Bitcask is the default storage engine for Riak. You can verify that +Bitcask is currently being used as the storage backend with the +[`riak`][use admin riak cli] command interface: + +```bash +riak config effective | grep backend +``` + +If this operation returns anything other than `bitcask`, read +the following section for instructions on switching the backend to Bitcask. + +## Enabling Bitcask + +You can set Bitcask as the storage engine using each node's +[configuration files][config reference]: + +```riakconf +storage_backend = bitcask +``` + +```appconfig +{riak_kv, [ + {storage_backend, riak_kv_bitcask_backend}, + %% Other riak_kv settings... + + ]}, +``` + +## Configuring Bitcask + +Bitcask enables you to configure a wide variety of its behaviors, from +filesystem sync strategy to merge settings and more. + +> **Note on configuration systems** +> +> Riak 2.0 enables you to use either the newer [configuration system][config reference] based on a single `riak.conf` file or the older system, based on an `app.config` configuration file. +> Instructions for both systems will be included below. Narrative +descriptions of the various settings will be tailored to the newer +configuration system, whereas instructions for the older system will +largely be contained in the code tabs. + +The default configuration values for Bitcask are as follows: + +```riakconf +bitcask.data_root = ./data/bitcask +bitcask.io_mode = erlang +``` + +```appconfig +{bitcask, [ + {data_root, "/var/lib/riak/bitcask"}, + {io_mode, erlang}, + + %% Other Bitcask-specific settings + ]} +``` + +All of the other available settings listed below can be added to your +configuration files. + +### Open Timeout + +The open timeout setting specifies the maximum time Bitcask will block +on startup while attempting to create or open the Bitcask data +directory. The default is 4 seconds. + +In general, you will not need to adjust this setting. If, however, you +begin to receive log messages of the form `Failed to start bitcask +backend: ...`, you may want to consider using a longer timeout. + +Open timeout is specified using the `bitcask.sync.open_timeout` +parameter, and can be set in terms of seconds, minutes, hours, etc. +The following example sets the parameter to 10 seconds: + +```riakconf +bitcask.sync.open_timeout = 10s +``` + +```appconfig +{bitcask, [ + ..., + {open_timeout, 10} %% This value must be expressed in seconds + ... + ]} +``` + +### Sync Strategy + +Bitcask enables you to configure the durability of writes by specifying +when to synchronize data to disk, i.e. by choosing a sync strategy. The +default setting (`none`) writes data into operating system buffers that +will be written to disk when those buffers are flushed by the operating +system. If the system fails before those buffers are flushed, e.g. due +to power loss, that data is lost. This possibility holds for any +database in which values are asynchronously flushed to disk. + +Thus, using the default setting of `none` protects against data loss in +the event of application failure, i.e. process death, but leaves open a +small window in which data could be lost in the event of a complete +system failure, e.g. hardware or OS failure. + +This possibility can be prevented by choosing the `o_sync` sync +strategy, which forces the operating system to flush to stable storage +at write time for every write. The effect of flushing each write is +better durability, although it should be noted that write throughput +will suffer because each write will have to wait for the write to +complete. + +The following sync strategies are available: + + * `none` --- lets the operating system manage syncing writes + (default) + * `o_sync` --- uses the `O_SYNC` flag, which forces syncs on every + write + * Time interval --- Riak will force Bitcask to sync at specified + intervals + +The following are possible configurations: + + +```riakconf +bitcask.sync.strategy = none +bitcask.sync.strategy = o_sync + +bitcask.sync.strategy = interval +bitcask.sync.interval = 65s +``` + +```appconfig +{bitcask, [ + ..., + {sync_strategy, none}, + {sync_strategy, o_sync}, + {sync_strategy, {seconds, 10}}, %% The time interval must be specified in seconds + ... + ]} +``` + +> **Sync strategy interval limitations** +> +> Setting the sync interval to a value lower or equal to + `riak_core.vnode_inactivity_timeout` (default: 60 seconds), will + prevent Riak from performing handoffs. +> +> A vnode must be inactive (not receive any messages) for a certain amount of time before the handoff process can start. The sync mechanism causes a message to be sent to the vnode for every sync, thus preventing the vnode from ever becoming inactive. + +### Max File Size + +The `max_file_size` setting describes the maximum permitted size for any +single data file in the Bitcask directory. If a write causes the current +file to exceed this size threshold then that file is closed, and a new +file is opened for writes. The default is 2 GB. + +Increasing `max_file_size` will cause Bitcask to create fewer, larger +files that are merged less frequently, while decreasing it will cause +Bitcask to create more numerous, smaller files that are merged more +frequently. + +To give an example, if your ring size is 16, your servers could see as +much as 32 GB of data in the bitcask directories before the first merge +is triggered, irrespective of your working set size. You should plan +storage accordingly and be aware that it is possible to see disk data +sizes that are larger than the working set. + +The `max_file_size` setting can be specified using kilobytes, megabytes, +etc. The following example sets the max file size to 1 GB: + +```riakconf +bitcask.max_file_size = 1GB +``` + +```appconfig +%% The max_file_size setting must be expressed in bytes, as in the +%% example below + +{bitcask, [ + ..., + {max_file_size, 16#40000000}, %% 1 GB expressed in bytes + ... + ]} +``` + +### Hint File CRC Check + +During startup, Bitcask will read from `.hint` files in order to build +its in-memory representation of the key space, falling back to `.data` +files if necessary. This reduces the amount of data that must be read +from the disk during startup, thereby also reducing the time required to +start up. You can configure Bitcask to either disregard `.hint` files +that don't contain a CRC value or to use them anyway. + +If you are using the newer, `riak.conf`-based configuration system, you +can instruct Bitcask to disregard `.hint` files that do not contain a +CRC value by setting the `hintfile_checksums` setting to `strict` (the +default). To use Bitcask in a backward-compatible mode that allows for +`.hint` files without CRC signatures, change the setting to +`allow_missing`. + +The following example sets the parameter to `strict`: + +```riakconf +bitcask.hintfile_checksums = strict +``` + +```appconfig +%% In the app.config-based system, substitute "require_hint_crc" for +%% "hintfile_checksums", "true" for "strict", and "false" for +%% "allow_missing" + +{bitcask, [ + ..., + {require_hint_crc, true}, + ... + ]} +``` + +### I/O Mode + +The `io_mode` setting specifies which code module Bitcask should use for +file access. The available settings are: + +* `erlang` (default) --- Writes are made via Erlang's built-in file API +* `nif` --- Writes are made via direct calls to the POSIX C API + +The following example sets `io_mode` to `erlang`: + +```riakconf +bitcask.io_mode = erlang +``` + +```appconfig +{bitcask, [ + ..., + {io_mode, erlang}, + ... + ]} +``` + +In general, the `nif` IO mode provides higher throughput for certain +workloads, but it has the potential to negatively impact the Erlang VM, +leading to higher worst-case latencies and possible throughput collapse. + +### `O_SYNC` on Linux + +Synchronous file I/O via +[`o_sync`](http://linux.about.com/od/commands/l/blcmdl2_open.htm) is +supported in Bitcask if `io_mode` is set to `nif` and is not supported +in the `erlang` mode. + +If you enable `o_sync` by setting `io_mode` to `nif`, however, you will +still get an incorrect warning along the following lines: + +```log +[warning] <0.445.0>@riak_kv_bitcask_backend:check_fcntl:429 {sync_strategy,o_sync} not implemented on Linux +``` + +If you are using the older, `app.config`-based configuration system, you +can disable the check that generates this warning by adding the +following to the `riak_kv` section of your `app.config`: + +```appconfig +{riak_kv, [ + ..., + {o_sync_warning_logged, false}, + ... + ]} +``` + +### Disk Usage and Merging Settings + +Riak KV stores each [vnode][glossary vnode] of the +[ring][learn clusters] as a separate Bitcask directory within the +configured Bitcask data directory. + +Each of these directories will contain multiple files with key/value +data, one or more "hint" files that record where the various keys exist +within the data files, and a write lock file. The design of Bitcask +allows for recovery even when data isn't fully synchronized to disk +(partial writes). This is accomplished by maintaining data files that +are append-only (i.e. never modified in-place) and are never reopened +for modification (i.e. they are only for reading). + +This data management strategy trades disk space for operational +efficiency. There can be a significant storage overhead that is +unrelated to your working data set but can be tuned in a way that best +fits your use case. In short, disk space is used until a threshold is +met at which point unused space is reclaimed through a process of +merging. The merge process traverses data files and reclaims space by +eliminating out-of-date of deleted key/value pairs, writing only the +current key/value pairs to a new set of files within the directory. + +The merge process is affected by all of the settings described in the +sections below. In those sections, "dead" refers to keys that no longer +contain the most up-to-date values, while "live" refers to keys that do +contain the most up-to-date value and have not been deleted. + +### Merge Policy + +Bitcask enables you to select a merge policy, i.e. when during the day +merge operations are allowed to be triggered. The valid options are: + +* `always` --- No restrictions on when merge operations can occur + (default) +* `never` --- Merge will never be attempted +* `window` --- Merge operations occur during specified hours + +If you are using the newer, `riak.conf`-based configuration system, you +can select a merge policy using the `merge.policy` setting. The +following example sets the merge policy to `never`: + +```riakconf +bitcask.merge.policy = never +``` + +```appconfig +{bitcask, [ + ..., + {merge_window, never}, + ... + ]} +``` + +If you opt to specify start and end hours for merge operations, you can +do so with the `merge.window.start` and `merge.window.end` +settings in addition to setting the merge policy to `window`. +Each setting is an integer between 0 and 23 for hours on a 24h clock, +with 0 meaning midnight and 23 standing for 11 pm. +The merge window runs from the first minute of the `merge.window.start` hour +to the last minute of the `merge.window.end` hour. +The following example enables merging between 3 am and 4:59 pm: + +```riakconf +bitcask.merge.policy = window +bitcask.merge.window.start = 3 +bitcask.merge.window.end = 17 +``` + +```appconfig +%% In the app.config-based system, you specify the merge window using +%% a tuple, as in the following example: + +{bitcask, [ + ..., + {merge_window, {3, 17}}, + ... + ]} +``` + +> **`merge_window` and the Multi backend** +> +>If you are using the older configuration system and using Bitcask with +the [Multi][plan backend multi] backend, please note that if you +wish to use a merge window, you _must_ set it in the global `bitcask` +section of your configuration file. `merge_window` settings +in per-backend sections are ignored. + +If merging has a significant impact on performance of your cluster, or +if your cluster has quiet periods in which little storage activity +occurs, you may want to change this setting from the default. + +A common way to limit the impact of merging is to create separate merge +windows for each node in the cluster and ensure that these windows do +not overlap. This ensures that at most one node at a time can be +affected by merging, leaving the remaining nodes to handle requests. +The main drawback of this approach is that merges will occur less +frequently, leading to increased disk space usage. + +### Merge Triggers + +Merge triggers determine the conditions under which merging will be +invoked. These conditions fall into two basic categories: + +* **Fragmentation** --- This describes the ratio of dead keys to total + keys in a file that will trigger merging. The value of this setting is + an integer percentage (0-100). For example, if a data file contains 6 + dead keys and 4 live keys, a merge will be triggered by the default + setting (60%). Increasing this value will cause merging to occur less + often, whereas decreasing the value will cause merging to happen more + often. + +* **Dead Bytes** --- This setting describes how much data stored for + dead keys in a single file will trigger merging. If a file meets or + exceeds the trigger value for dead bytes, a merge will be triggered. + Increasing the value will cause merging to occur less often, whereas + decreasing the value will cause merging to happen more often. The + default is 512 MB. + + When either of these constraints are met by any file in the directory, + Bitcask will attempt to merge files. + +You can set the triggers described above using +`merge.triggers.fragmentation` and `merge.triggers.dead_bytes`, +respectively. The former is expressed as an integer between 0 and 100, +whereas the latter can be expressed in terms of kilobytes, megabytes, +gigabytes, etc. The following example sets the dead bytes threshold to +55% and the fragmentation threshold to 1 GB: + +```riakconf +bitcask.merge.triggers.fragmentation = 55 +bitcask.merge.triggers.dead_bytes = 1GB +``` + +```appconfig +%% The equivalent settings in the app.config-based system are +%% frag_merge_trigger and dead_bytes_merge_trigger, respectively. The +%% latter must be expressed in bytes. + +{bitcask, [ + ..., + {frag_merge_trigger, 55}, + {dead_bytes_merge_trigger, 1073741824}, + ... + ]} +``` + +### Merge Thresholds + +Merge thresholds determine which files will be chosen for inclusion in +a merge operation. + +* **Fragmentation** --- This setting describes which ratio of dead keys + to total keys in a file will cause it to be included in the merge. The + value of this setting is a percentage (0-100). For example, if a data + file contains 4 dead keys and 6 live keys, it will be included in the + merge at the default ratio (40%). Increasing the value will cause + fewer files to be merged, while decreasing the value will cause more + files to be merged. + +* **Dead Bytes** --- This setting describes which ratio the minimum + amount of data occupied by dead keys in a file to cause it to be + included in the merge. Increasing this value will cause fewer files to + be merged, while decreasing this value will cause more files to be + merged. The default is 128 MB. + +* **Small File** --- This setting describes the minimum size a file must + be to be _excluded_ from the merge. Files smaller than the threshold + will be included. Increasing the value will cause more files to be + merged, while decreasing the value will case fewer files to be merged. + The default is 10 MB. + +You can set the thresholds described above using the +`merge.thresholds.fragmentation`, `merge.thresholds.dead_bytes`, and +`merge.threshold.small_file` settings, respectively. + +The `fragmentation` setting is expressed as an integer +between 0 and 100, and the `dead_bytes` and `small_file` settings can be +expressed in terms of kilobytes, megabytes, gigabytes, etc. The +following example sets the fragmentation threshold to 45%, the +dead bytes threshold to 200 MB, and the small file threshold to 25 MB: + +```riakconf +bitcask.merge.thresholds.fragmentation = 45 +bitcask.merge.thresholds.dead_bytes = 200MB +bitcask.merge.thresholds.small_file = 25MB +``` + +```appconfig +%% In the app.config-based system, the settings corresponding to those +%% listed above are frag_threshold, dead_bytes_threshold, and +%% small_files threshold, respectively. The latter two settings must be +%% expressed in bytes: + +{bitcask, [ + ..., + {frag_threshold, 45}, + {dead_bytes_threshold, 209715200}, + {small_file_threshold, 26214400}, + ... + ]} +``` +> **Note on choosing threshold values** +> +> The values for the fragmentation and dead bytes thresholds _must be +equal to or less than their corresponding trigger values_. If they are +set higher, Bitcask will trigger merges in cases where no files meet the +threshold, which means that Bitcask will never resolve the conditions +that triggered merging in the first place. + +### Merge Interval + +Bitcask periodically runs checks to determine whether merges are +necessary. You can determine how often those checks take place using +the `bitcask.merge_check_interval` parameter. The default is 3 minutes. + +```riakconf +bitcask.merge_check_interval = 3m +``` + +```appconfig +%% In the app.config-based system, this setting is expressed in +%% milliseconds and found in the riak_kv section rather than the bitcask +%% section: + +{riak_kv, [ + %% Other configs + + {bitcask_merge_check_interval, 180000}, + + %% Other configs + ]} +``` + +If merge check operations happen at the same time on different +[vnodes][glossary vnode] on the same node, this can produce spikes +in I/O usage and undue latency. Bitcask makes it less likely that merge +check operations will occur at the same time on different vnodes by +applying a **jitter** to those operations. A jitter is a random +variation applied to merge times that you can alter using the +`bitcask.merge_check_jitter` parameter. This parameter is expressed as a +percentage of `bitcask.merge_check_interval`. The default is 30%. + +```riakconf +bitcask.merge_check_jitter = 30% +``` + +```appconfig +%% In the app.config-based system, this setting is expressed as a float +%% and found in the riak_kv section rather than the bitcask section: + +{riak_kv, [ + %% Other configs + + {bitcask_merge_check_jitter, 0.3}, + + %% Other configs + ]} +``` + +For example, if you set the merge check interval to 4 minutes and the +jitter to 25%, merge checks will occur at intervals between 3 and 5 +minutes. With the default of 3 minutes and 30%, checks will occur at +intervals between roughly 2 and 4 minutes. + +### Log Needs Merge + +If you are using the older, `app.config`-based configuration system, you +can use the `log_needs_merge` setting to tune and troubleshoot Bitcask +merge settings. When set to `true` (as in the example below), each time +a merge trigger is met, the partition/vnode ID and mergeable files will +be logged. + +```appconfig +{bitcask, [ + ..., + {log_needs_merge, true}, + ... + ]} +``` + +> **Note on `log_needs_merge` and the Multi backend** +> +>If you are using Bitcask with the [Multi][plan backend multi] backend in conjunction with the older, `app.config`-based configuration system, please +note that `log_needs_merge` _must_ be set in the global `bitcask` section of your `app.config`. All `log_needs_merge` settings in per-backend sections are ignored. + +### Fold Keys Threshold + +Fold keys thresholds will reuse the keydir (a) if another fold was +started less than a specified time interval ago and (b) there were fewer +than a specified number of updates. Otherwise, Bitcask will wait until +all current fold keys complete and then start. The default time interval +is 0, while the default number of updates is unlimited. Both thresholds +can be disabled. + +The conditions described above can be set using the `fold.max_age` and +`fold.max_puts` parameters, respectively. The former can be expressed in +terms of minutes, hours, days, etc., while the latter is expressed as an +integer. Each threshold can be disabled by setting the value to +`unlimited`. The following example sets the `max_age` to 1/2 second and +the `max_puts` to 1000: + +```riakconf +bitcask.max_age = 0.5s +bitcask.max_puts = 1000 +``` + +```appconfig +%% In the app.config-based system, the corresponding parameters are +%% max_fold_age and max_fold_puts, respectively. The former must be +%% expressed in milliseconds, while the latter must be an integer: + +{bitcask, [ + ..., + {max_fold_age, 500}, + {max_fold_puts, 1000}, + ... + ]} + +%% Each of these thresholds can be disabled by setting the value to -1 +``` + +<a name="Automatic-Expiration"></a> +### Automatic Expiration + +By default, Bitcask keeps all of your data. But if your data has limited +time value or if you need to purge data for space reasons, you can +configure object expiration, aka expiry. This feature is disabled by +default. + +You can enable and configure object expiry using the `expiry` setting +and either specifying a time interval in seconds, minutes, hours, etc., +or turning expiry off (`off`). The following example configures objects +to expire after 1 day: + +```riakconf +bitcask.expiry = 1d +``` + +```appconfig +%% In the app.config-based system, expiry is expressed in terms of +%% seconds: + +{bitcask, [ + ..., + {expiry_secs, 86400}, %% Sets the duration to 1 day + ... + ]} + +%% Expiry can be turned off by setting this value to -1 +``` + +> **Note on stale data** +> +> Space occupied by stale data _may not be reclaimed immediately_, +but the data will become immediately inaccessible to client requests. +Writing to a key will set a new modification timestamp on the value +and prevent it from being expired. + +By default, Bitcask will trigger a merge whenever a data file contains +an expired key. This may result in excessive merging under some usage +patterns. You can prevent this by configuring an expiry grace time. +Bitcask will defer trigger a merge solely for key expiry by the +configured amount of time. The default is 0, signifying no grace time. + +If you are using the newer, `riak.conf`-based configuration system, you +can set an expiry grace time using the `expiry.grace_time` setting and +in terms of minutes, hours, days, etc. The following example sets the +grace period to 1 hour: + +```riakconf +bitcask.expiry.grace_time = 1h +``` + +```appconfig +%% The equivalent setting in the app.config-based system is +%% expiry_grace_time. This must be expressed in seconds: + +{bitcask, [ + ..., + {expiry_grace_time, 3600}, %% Sets the grace period to 1 hour + ... + ]} +``` + +#### Automatic expiration and Riak Search + +If you are using [Riak Search][usage search] in conjunction with +Bitcask, please be aware that automatic expiry does not apply to [Search Indexes](../../../../developing/usage/search). If objects are indexed using Search, +those objects can be expired by Bitcask yet still registered in Search +indexes, which means that Search queries may return keys that no longer +exist. Riak's [active anti-entropy (AAE)][glossary aae] subsystem will eventually +catch this discrepancy, but this depends on AAE being enabled (which is +the default) and could take some time. If search queries returning +expired keys is a problem for your use case, then we would recommend not +using automatic expiration. + +## Tuning Bitcask + +When tuning your environment, there are a number of things to bear in +mind that can assist you in making Bitcask as stable and reliable as +possible and to minimize latency and maximize throughput. + +### Tips & Tricks + + * **Bitcask depends on filesystem caches** + + Some data storage layers implement their own page/block buffer cache + in-memory, but Bitcask does not. Instead, it depends on the + filesystem's cache. Adjusting the caching characteristics of your + filesystem can impact performance. + + * **Be aware of file handle limits** + + Review the documentation on [open files limit][perf open files]. + + * **Avoid the overhead of updating file metadata (such as last access + time) on every read or write operation** + + You can achieve a substantial speed boost by adding the `noatime` + mounting option to Linux's `/etc/fstab`. This will disable the + recording of the last accessed time for all files, which results + in fewer disk head seeks. If you need last access times but you'd + like some of the benefits of this optimization, you can try + `relatime`. + + ``` + /dev/sda5 /data ext3 noatime 1 1 + /dev/sdb1 /data/inno-log ext3 noatime 1 2 + ``` + + * **Small number of frequently changed keys** + + When keys are changed frequently, fragmentation rapidly increases. + To counteract this, you should lower the fragmentation trigger and + threshold. + + * **Limited disk space** + + When disk space is limited, limiting the space occupied by dead keys + is of paramount importance. Lower the dead bytes threshold and + trigger to counteract wasted space. + + * **Purging stale entries after a fixed period** + + To automatically purge stale values, set the object expiry value to + the desired cutoff time. Keys that are not modified for a period + equal to or greater than this time interval will become + inaccessible. + + * **High number of partitions per node** + + Because each cluster has many partitions running, Bitcask will have + many [open files][perf open files]. To reduce the number of open + files, we suggest increasing the max file size so that larger files + will be written. You could also decrease the fragmentation and + dead-bytes settings and increase the small file threshold so that + merging will keep the number of open files small in number. + + * **High daytime traffic, low nighttime traffic** + + In order to cope with a high volume of writes without performance + degradation during the day, you might want to limit merging to + in non-peak periods. Setting the merge window to hours of the day + when traffic is low will help. + + * **Multi-cluster replication** + + If you are using Riak with the replication feature enabled, your clusters might experience + higher production of fragmentation and dead bytes. Additionally, + because the fullsync feature operates across entire partitions, it + will be made more efficient by accessing data as sequentially as + possible (across fewer files). Lowering both the fragmentation and + dead-bytes settings will improve performance. + +## FAQ + + * [[Why does it seem that Bitcask merging is only triggered when a + Riak node is restarted?|Developing on Riak + FAQs#why-does-it-seem-that-bitc]] + * [[If the size of key index exceeds the amount of memory, how does + Bitcask handle it?|Operating Riak FAQs#if-the-size-of-key-index-e]] + * [Bitcask Capacity Planning][plan bitcask capacity] + +## Bitcask Implementation Details + +Riak will create a Bitcask database directory for each [vnode][glossary vnode] +in a [cluster][learn clusters]. In each of those directories, at most one +database file will be open for writing at any given time. The file being +written to will grow until it exceeds a specified size threshold, at +which time it is closed and a new file is created for additional writes. +Once a file is closed, whether purposely or due to server exit, it is +considered immutable and will never again be opened for writing. + +The file currently open for writes is only written by appending, which +means that sequential writes do not require disk seeking, which can +dramatically speed up disk I/O. Note that this effect can be hampered if +you have `atime` enabled on your filesystem, because the disk head will +have to move to update both the data blocks _and_ the file and directory +metadata blocks. The primary speed advantage from a log-based database +stems of its ability to minimize disk head seeks. + +Deleting a value from Bitcask is a two-step process: first, a +[tombstone][usage delete objects] is recorded in the open file for writes, +which indicates that a value was marked for deletion at that time, while +references to that key are removed from the in-memory "keydir" +information; later, during a merge operation, non-active data files are +scanned, and only those values without tombstones are merged into the +active data file. This effectively removes the obsolete data and +reclaims disk space associated with it. This data management strategy +may use up a lot of space over time, since Bitcask writes new values +without touching the old ones. + +The compaction process referred to as "merging" solves this +problem. The merge process iterates over all non-active (i.e. immutable) +files in a Bitcask database and produces as output a set of data files +containing only the "live" or latest versions of each present key. + +### Bitcask Database Files + +Below are two directory listings showing what you should expect to find +on disk when using Bitcask. In this example, we use a 64-partition +[ring][learn clusters], which results in 64 separate directories, +each holding its own Bitcask database. + +```bash +ls ./data/bitcask +``` + +The result: + +``` +0 +1004782375664995756265033322492444576013453623296 +1027618338748291114361965898003636498195577569280 + +... etc ... + +9819464125817003981681007469812.2.63831329677312 +``` + +Note that when starting up the directories are created for each +[vnode][glossary vnode] partition's data. At this point, however, there are not +yet any Bitcask-specific files. + +After performing one PUT (write) into the Riak cluster running Bitcask: + +```bash +curl -XPUT http://localhost:8098/types/default/buckets/test/keys/test \ + -H "Content-Type: text/plain" \ + -d "hello" +``` + +The "N" value for this cluster is 3 (the default), so you'll see that +the three vnode partitions responsible for this data now have Bitcask +database files: + +``` +bitcask/ + +... etc ... + +|-- 1118962191081472546749696200048404186924073353216-1316787078245894 +| |-- 1316787252.bitcask.data +| |-- 1316787252.bitcask.hint +| `-- bitcask.write.lock + +... etc ... + + +|-- 1141798154164767904846628775559596109106197299200-1316787078249065 +| |-- 1316787252.bitcask.data +| |-- 1316787252.bitcask.hint +| `-- bitcask.write.lock + +... etc ... + + +|-- 116463411724806326294356135107078803128832.2.6184-1316787078254833 +| |-- 1316787252.bitcask.data +| |-- 1316787252.bitcask.hint +| `-- bitcask.write.lock + +... etc ... + +``` + +As more data is written to the cluster, more Bitcask files are created +until merges are triggered. + +``` +bitcask/ +|-- 0-1317147619996589 +| |-- 1317147974.bitcask.data +| |-- 1317147974.bitcask.hint +| |-- 1317221578.bitcask.data +| |-- 1317221578.bitcask.hint +| |-- 1317221869.bitcask.data +| |-- 1317221869.bitcask.hint +| |-- 1317222847.bitcask.data +| |-- 1317222847.bitcask.hint +| |-- 1317222868.bitcask.data +| |-- 1317222868.bitcask.hint +| |-- 1317223014.bitcask.data +| `-- 1317223014.bitcask.hint +|-- 1004782375664995756265033322492444576013453623296-1317147628760580 +| |-- 1317147693.bitcask.data +| |-- 1317147693.bitcask.hint +| |-- 13172.2.65.bitcask.data +| |-- 13172.2.65.bitcask.hint +| |-- 1317222514.bitcask.data +| |-- 1317222514.bitcask.hint +| |-- 1317223035.bitcask.data +| |-- 1317223035.bitcask.hint +| |-- 1317223411.bitcask.data +| `-- 1317223411.bitcask.hint +|-- 1027618338748291114361965898003636498195577569280-1317223690337865 +|-- 1050454301831586472458898473514828420377701515264-1317223690151365 + +... etc ... + +``` + +This is normal operational behavior for Bitcask. diff --git a/content/riak/kv/2.2.6/setup/planning/backend/leveldb.md b/content/riak/kv/2.2.6/setup/planning/backend/leveldb.md new file mode 100644 index 0000000000..1c835a2583 --- /dev/null +++ b/content/riak/kv/2.2.6/setup/planning/backend/leveldb.md @@ -0,0 +1,502 @@ +--- +title: "LevelDB" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "LevelDB" + identifier: "planning_backend_leveldb" + weight: 101 + parent: "planning_choose_backend" +toc: true +aliases: + - /riak-docs/riak/2.2.6/ops/advanced/backends/leveldb/ + - /riak-docs/riak/kv/2.2.6/ops/advanced/backends/leveldb/ +--- + +[upgrade 2.0#upgrading-leveldB]: {{<baseurl>}} +[glossary vnode]: {{<baseurl>}}riak/kv/2.2.6/learn/glossary/#vnode +[config reference]: {{<baseurl>}}riak/kv/2.2.6/configuring/reference +[perf index]: {{<baseurl>}}riak/kv/2.2.6/using/performance +[config reference#aae]: {{<baseurl>}}riak/kv/2.2.6/configuring/reference/#active-anti-entropy + +> **Note on upgrading to 2.0** +> +> If you are using LevelDB in a 1.x version of Riak, are upgrading to 2.0, +and wish to keep using your old `app.config` file for configuration, +make sure to follow the steps for setting the +`total_leveldb_mem_percent` parameter in the +[2.0 upgrade guide][upgrade 2.0#upgrading-leveldB]. + +[eLevelDB](https://github.com/basho/eleveldb) is an Erlang application +that encapsulates [LevelDB](http://code.google.com/p/leveldb/), an +open-source, on-disk key/value store created by Google Fellows Jeffrey +Dean and Sanjay Ghemawat. + +LevelDB is a relatively new entrant into the growing list of key/value +database libraries, but it has some very interesting qualities that we +believe make it an ideal candidate for use in Riak. LevelDB's storage +architecture is more like +[BigTable's](http://en.wikipedia.org/wiki/BigTable) memtable/sstable +model than it is like Bitcask. This design and implementation provide +the possibility of a storage engine without Bitcask's RAM limitation. + +> **Note:** Riak uses a fork of LevelDB. The code can be found +[on Github](https://github.com/basho/leveldb). + +A number of changes have been introduced in the LevelDB backend in Riak +2.0: + +* There is now only _one_ performance-related setting that Riak users + need to define---`leveldb.total_mem_percent`---as LevelDB now + dynamically sizes the file cache and block sizes based upon active + [vnodes][glossary vnode] assigned to the node. +* The LevelDB backend in Riak 2.0 utilizes a new, faster threading model + for background compaction work on `.sst` table files. The new model + has increased throughput by at least 10% in all test scenarios. +* Delete operations now receive priority handling in compaction + selection, which means more aggressive reclaiming of disk space than + in previous versions of Riak's LevelDB backend. +* Nodes storing massive key datasets (e.g. in the billions of keys) now + receive increased throughput due to automatic management of LevelDB's + block size parameter. This parameter is slowly raised to increase the + number of files that can open simultaneously, improving random read + performance. + +## Strengths + +1. **License** --- The LevelDB and eLevelDB licenses are the [New BSD + License](http://www.opensource.org/licenses/bsd-license.php) and the + [Apache 2.0 License](http://www.apache.org/licenses/LICENSE-2.0.html), + respectively. We'd like to thank Google and the authors of LevelDB at + Google for choosing a completely FLOSS license so that everyone can + benefit from this innovative storage engine. +2. **Data compression** --- LevelDB provides two compression algorithms + to reduce storage size and increase efficient use of storage bandwidth: + * Google's [Snappy](https://code.google.com/p/snappy/) data compression + * [LZ4](https://en.wikipedia.org/wiki/LZ4_(compression_algorithm)) data + compression + + Enabling compression means more CPU usage but less disk space. Compression + is especially good for text data, including raw text, Base64, JSON, etc. + +## Weaknesses + +1. Read access can be slow when there are many levels to search +2. LevelDB may have to do a few disk seeks to satisfy a read; one disk + seek per level and, if 10% of the database fits in memory, one seek + for the last level (since all of the earlier levels should end up + cached in the OS buffer cache for most filesystems) whereas if 1% + fits in memory, LevelDB will need two seeks. + +## Installing eLevelDB + +Riak ships with eLevelDB included within the distribution, so there is +no separate installation required. However, Riak is configured to use +the Bitcask storage engine by default. To switch to eLevelDB, set the +`storage_backend` variable in [`riak.conf`][config reference] to +`leveldb`: + +```riakconf +storage_backend = leveldb +``` + +```appconfig +{riak_kv, [ + %% ... + {storage_backend, riak_kv_eleveldb_backend}, + %% ... + ]} +``` + +## Configuring eLevelDB + +eLevelDb's default behavior can be modified by adding/changing +parameters in the `eleveldb` section of the [`riak.conf`][config reference]. The section below details the parameters you'll use to modify eLevelDB. + +The configuration values that can be set in your +[`riak.conf`][config reference] for eLevelDB are as follows: + +Config | Description | Default +:------|:------------|:------- +`leveldb.data_root` | LevelDB data root | `./data/leveldb` +`leveldb.maximum_memory.percent` | Defines the percentage (between 1 and 100) of total server memory to assign to LevelDB. LevelDB will dynamically adjust its internal cache sizes as Riak activates/inactivates [vnodes][glossary vnode] on this server to stay within this size. | `70` + +If you are using the older, `app.config`-based system, the equivalent to +the `leveldb.data_root` is the `data_root` setting, as in the following +example: + +```appconfig +{eleveldb, [ + {data_root, "/path/to/leveldb"}, + + %% Other eleveldb-specific settings +]} +``` + +The `leveldb.maximum_memory.percent` setting is only available in the +newer configuration system. + +### Recommended Settings + +Below are **general** configuration recommendations for Linux +distributions. Individual users may need to tailor these settings for +their application. + +#### sysctl + +For production environments, please see [System Performance Tuning][perf index] +for the recommended `/etc/sysctl.conf` settings. + +#### Block Device Scheduler + +Beginning with the 2.6 kernel, Linux gives you a choice of four I/O +[elevator models](http://www.gnutoolbox.com/linux-io-elevator/). We +recommend using the NOOP elevator. You can do this by changing the +scheduler on the Linux boot line: `elevator=noop`. + +#### ext4 Options + +The ext4 filesystem defaults include two options that increase integrity +but slow performance. Because Riak's integrity is based on multiple +nodes holding the same data, these two options can be changed to boost +LevelDB's performance. We recommend setting: `barrier`=0 and +`data`=writeback. + +#### CPU Throttling + +If CPU throttling is enabled, disabling it can boost LevelDB performance +in some cases. + +#### No Entropy + +If you are using https protocol, the 2.6 kernel is widely known for +stalling programs waiting for SSL entropy bits. If you are using https, +we recommend installing the +[HAVEGE](http://www.irisa.fr/caps/projects/hipsor/) package for +pseudorandom number generation. + +#### clocksource + +We recommend setting `clocksource=hpet` on your Linux kernel's `boot` +line. The TSC clocksource has been identified to cause issues on +machines with multiple physical processors and/or CPU throttling. + +#### swappiness + +We recommend setting `vm.swappiness=0` in `/etc/sysctl.conf`. The +`vm.swappiness` default is 60, which is aimed toward laptop users with +application windows. This was a key change for MySQL servers and is +often referenced in database performance literature. + +## Implementation Details + +[LevelDB](http://leveldb.googlecode.com/svn/trunk/doc/impl.html) is a +Google-sponsored open source project that has been incorporated into an +Erlang application and integrated into Riak for storage of key/value +information on disk. The implementation of LevelDB is similar in spirit +to the representation of a single Bigtable tablet (section 5.3). + +### How Levels Are Managed + +LevelDB is a memtable/sstable design. The set of sorted tables is +organized into a sequence of levels. Each level stores approximately ten +times as much data as the level before it. The sorted table generated +from a flush is placed in a special young level (also called level-0). +When the number of young files exceeds a certain threshold (currently +four), all of the young files are merged together with all of the +overlapping level-1 files to produce a sequence of new level-1 files (a +new level-1 file is created for every 2MB of data.) + +Files in the young level may contain overlapping keys. However files in +other levels have distinct non-overlapping key ranges. Consider level +number L where L >= 1. When the combined size of files in level-L +exceeds (10^L) MB (i.e. 10MB for level-1, 100MB for level-2, ...), one +file in level-L, and all of the overlapping files in level-(L+1) are +merged to form a set of new files for level-(L+1). These merges have the +effect of gradually migrating new updates from the young level to the +largest level using only bulk reads and writes (i.e., minimizing +expensive disk seeks). + +When the size of level L exceeds its limit, LevelDB will compact it in a +background thread. The compaction picks a file from level L and all +overlapping files from the next level L+1. Note that if a level-L file +overlaps only part of a level-(L+1) file, the entire file at level-(L+1) +is used as an input to the compaction and will be discarded after the +compaction. Compactions from level-0 to level-1 are treated specially +because level-0 is special (files in it may overlap each other). A +level-0 compaction may pick more than one level-0 file in case some of +these files overlap each other. + +A compaction merges the contents of the picked files to produce a +sequence of level-(L+1) files. LevelDB will switch to producing a new +level-(L+1) file after the current output file has reached the target +file size (2MB). LevelDB will also switch to a new output file when the +key range of the current output file has grown enough to overlap more +then ten level-(L+2) files. This last rule ensures that a later +compaction of a level-(L+1) file will not pick up too much data from +level-(L+2). + +Compactions for a particular level rotate through the key space. In more +detail, for each level L, LevelDB remembers the ending key of the last +compaction at level L. The next compaction for level L will pick the +first file that starts after this key (wrapping around to the beginning +of the key space if there is no such file). + +Level-0 compactions will read up to four 1MB files from level-0, and at +worst all the level-1 files (10MB) (i.e., LevelDB will read 14MB and +write 14MB in that case). + +Other than the special level-0 compactions, LevelDB will pick one 2MB +file from level L. In the worst case, this will overlap with +approximately 12 files from level L+1 (10 because level-(L+1) is ten +times the size of level-L, and another two at the boundaries since the +file ranges at level-L will usually not be aligned with the file ranges +at level-L+1). The compaction will therefore read 26MB, write 26MB. +Assuming a disk IO rate of 100MB/s, the worst compaction cost will be +approximately 0.5 second. + +If we throttle the background writing to a reasonably slow rate, for +instance 10% of the full 100MB/s speed, a compaction may take up to 5 +seconds. If the user is writing at 10MB/s, LevelDB might build up lots +of level-0 files (~50 to hold the 5*10MB). This may significantly +increase the cost of reads due to the overhead of merging more files +together on every read. + +### Compaction + +Levels are compacted into ordered data files over time. Compaction first +computes a score for each level as the ratio of bytes in that level to +desired bytes. For level 0, it computes files / desired files instead. +The level with the highest score is compacted. + +When compacting L0 the only special case to consider is that after +picking the primary L0 file to compact, it will check other L0 files to +determine the degree to which they overlap. This is an attempt to avoid +some I/O, we can expect L0 compactions to usually if not always be "all +L0 files". + +See the PickCompaction routine in +[1](https://github.com/basho/leveldb/blob/develop/db/version_set.cc) +for all the details. + +### Comparison of eLevelDB and Bitcask + +LevelDB is a persistent ordered map; Bitcask is a persistent hash table +(no ordered iteration). Bitcask stores keys in memory, so for databases +with large number of keys it may exhaust available physical memory and +then swap into virtual memory causing a severe slow down in performance. +Bitcask guarantees at most one disk seek per look-up. LevelDB may have +to do a small number of disk seeks. For instance, a read needs one disk +seek per level. If 10% of the database fits in memory, LevelDB will need +to do one seek (for the last level since all of the earlier levels +should end up cached in the OS buffer cache). If 1% fits in memory, +LevelDB will need two seeks. + +## Recovery + +LevelDB never writes in place: it always appends to a log file, or +merges existing files together to produce new ones. So an OS crash will +cause a partially written log record (or a few partially written log +records). LevelDB recovery code uses checksums to detect this and will +skip the incomplete records. + +### eLevelDB Database Files + +Below are two directory listings showing what you would expect to find +on disk when using eLevelDB. In this example, we use a 64-partition ring +which results in 64 separate directories, each with their own LevelDB +database: + +```bash +leveldb/ +|-- 0 +| |-- 000003.log +| |-- CURRENT +| |-- LOCK +| |-- LOG +| `-- MANIFEST-000002 +|-- 1004782375664995756265033322492444576013453623296 +| |-- 000005.log +| |-- CURRENT +| |-- LOCK +| |-- LOG +| |-- LOG.old +| `-- MANIFEST-000004 +|-- 1027618338748291114361965898003636498195577569280 +| |-- 000005.log +| |-- CURRENT +| |-- LOCK +| |-- LOG +| |-- LOG.old +| `-- MANIFEST-000004 + +... etc ... + +`-- 9819464125817003981681007469812.2.63831329677312 + |-- 000005.log + |-- CURRENT + |-- LOCK + |-- LOG + |-- LOG.old + `-- MANIFEST-000004 + +64 directories, 378 files +``` + +After performing a large number of PUT (write) operations, the Riak +cluster running eLevelDB will look something like this: + +```bash +tree leveldb +``` + +The result should look something like this: + +``` +├── 0 +│   ├── 000003.log +│   ├── CURRENT +│   ├── LOCK +│   ├── LOG +│   ├── MANIFEST-000002 +│   ├── sst_0 +│   ├── sst_1 +│   ├── sst_2 +│   ├── sst_3 +│   ├── sst_4 +│   ├── sst_5 +│   └── sst_6 +├── 1004782375664995756265033322492444576013453623296 +│   ├── 000003.log +│   ├── CURRENT +│   ├── LOCK +│   ├── LOG +│   ├── MANIFEST-000002 +│   ├── sst_0 +│   ├── sst_1 +│   ├── sst_2 +│   ├── sst_3 +│   ├── sst_4 +│   ├── sst_5 +│   └── sst_6 + +... etc ... +``` + +## Tiered Storage + +Google's original LevelDB implemented stored all `.sst` table files in a +single database directory. In Riak 1.3, the original LevelDB code was +modified to store `.sst` files in subdirectories representing each +"level" of the file, e.g. `sst_0` or `sst_1`, in the name of speeding up +database repair operations. + +An additional advantage of this approach is that it enables Riak +operators to mount alternative storage devices at each level of a +LevelDB database. This can be an effective strategy because LevelDB is +write intensive in lower levels, with the write intensity declining as +the level number increases. This is due to LevelDB's storage strategy, +which places more frequently updated data in lower levels. + +Because write intensity differs by level, performance can be improved by +mounting faster, more expensive storage arrays in lower levels and +slower, less expensive arrays at higher levels. Tiered storage enables +you to configure the level at which LevelDB switches from a faster array +to a slower array. + +> **Note on write throttling** +> +> High-volume, sustained write operations can occasionally fill the +higher-speed storage arrays before LevelDB has had the opportunity to +move data to the low-speed arrays. LevelDB's write throttle will slow +incoming write operations to allow compactions to catch up, as would be +the case when using a single storage array. + +### Configuring Tiered Storage + +If you are using the newer, `riak.conf`-based configuration system, the +following parameters can be used to configure LevelDB tiered storage: + +Parameter | Description +:---------|:----------- +`leveldb.tiered` | The level number at which data should switch to the slower array. The default is `0`, which disables the feature. +`leveldb.tiered.path.fast` | The path prefix for `.sst` files below the level set by `leveldb.tiered` +`leveldb.tiered.path.slow` | The path prefix for `.sst` files at and above the level set by `leveldb.tiered` + +If you are using the older, `app.config`-based system, the example below +will show you the equivalents of the settings listed in the table above. + +#### Example + +The following example LevelDB tiered storage +[configuration][config reference] for Riak 2.0 sets the level for +switching storage arrays to 4 and the file path prefix to `fast_raid` +for the faster array and `slow_raid` for the slower array: + +```riakconf +leveldb.tiered = 4 +leveldb.tiered.path.fast = /mnt/fast_raid +leveldb.tiered.path.slow = /mnt/slow_raid +``` + +```appconfig +{eleveldb, [ + {tiered_slow_level, 4}, + {tiered_fast_prefix, "/mnt/fast_raid"}, + {tiered_slow_prefix, "/mnt/slow_raid"} +]} +``` + +With this configuration, level directories `sst_0` through `sst_3` will +be stored in `/mnt/fast_raid`, while directories `sst_4` and `sst_6` +will be stored in `/mnt/slow_raid`. + +### Selecting a Level + +LevelDB will perform optimally when as much data as possible is stored +in the faster array. The amount of data that can be stored in the faster +array depends on the size of your array and the total number of LevelDB +databases (i.e. the total number of Riak [vnodes][glossary vnode]) +in your cluster. The following table shows approximate sizes (in +megabytes) for each of the following sizes: the amount of raw data +stored in the level, the cumulative size of all levels up to the +specified level, and the cumulative size including active anti-entropy +data. + +Level | Level Size | Cumulative Size | Cumulative with AAE +:-----|:-----------|:----------------|:------------------- +0 | 360 | 360 | 720 +1 | 2,160 | 2,520 | 5,040 +2 | 2,940 | 5,460 | 10,920 +3 | 6,144 | 11,604 | 23,208 +4 | 122,880 | 134,484 | 268,968 +5 | 2,362,232 | 2,496,716 | 4,993,432 +6 | not limited | not limited | not limited + +To select the appropriate value for `leveldb.tiered`, use the following +steps: + +* Determine the value of (ring size) / (N - 1), where ring size is the + value of the `ring_size` configuration parameter and N is the number + of nodes in the cluster. For a `ring_size` of 128 and a cluster with + 10 nodes, the value would be 14. +* Select either the **Cumulative Size** or **Cumulative with AAE** + column from the table above. Select the third column if you are not + using active anti-entropy or the fourth column if you are (i.e. if the + `anti_entropy` [configuration parameter][config reference#aae] is set to `active`). +* Multiply the value from the first step by the cumulative column in + each row in the table. The first result that exceeds your fast storage + array capacity will provide the level number that should be used for + your `leveldb.tiered` setting. + +### Migrating from One Configuration to Another + +If you want to use tiered storage in a new Riak installation, you don't +need to take any steps beyond setting configuration. The rest is +automated. + +But if you'd like to use tiered storage in an existing installation that +is not currently using it, you will need to manually move your +installation's `.sst` files from one configuration to another. diff --git a/content/riak/kv/2.2.6/setup/planning/backend/memory.md b/content/riak/kv/2.2.6/setup/planning/backend/memory.md new file mode 100644 index 0000000000..1bcde520a3 --- /dev/null +++ b/content/riak/kv/2.2.6/setup/planning/backend/memory.md @@ -0,0 +1,143 @@ +--- +title: "Memory" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Memory" + identifier: "planning_backend_memory" + weight: 102 + parent: "planning_choose_backend" +toc: true +aliases: + - /riak-docs/riak/2.2.6/ops/advanced/backends/memory/ + - /riak-docs/riak/kv/2.2.6/ops/advanced/backends/memory/ +--- + +[config reference]: {{<baseurl>}}riak/kv/2.2.6/configuring/reference +[plan backend multi]: {{<baseurl>}}riak/kv/2.2.6/setup/planning/backend/multi +[glossary vnode]: {{<baseurl>}}riak/kv/2.2.6/learn/glossary/#vnode +[plan backend leveldb]: {{<baseurl>}}riak/kv/2.2.6/setup/planning/backend/leveldb + +The Memory storage backend uses in-memory tables to store all data. +This data is never persisted to disk or to any other storage mechanism. +The Memory storage engine is best used for testing Riak clusters or for +storing small amounts of transient state in production systems. + +Internally, the Memory backend uses Erlang Ets tables to manage data. +More information can be found in the +[official Erlang documentation](http://www.erlang.org/doc/man/ets.html). + +## Enabling the Memory Backend + +To enable the memory backend, edit your [configuration files][config reference] +for each Riak node and specify the Memory backend as shown in the following +example: + +```riakconf +storage_backend = memory +``` + +```appconfig +{riak_kv, [ + ..., + {storage_backend, riak_kv_memory_backend}, + ... + ]} +``` + +**Note**: If you *replace* the existing specified backend by removing it +or commenting it out as shown in the above example, data belonging to +the previously specified backend will still be preserved on the +filesystem but will no longer be accessible through Riak unless the +backend is enabled again. + +If you require multiple backends in your configuration, please consult +the [Multi backend documentation][plan backend multi]. + +## Configuring the Memory Backend + +The Memory backend enables you to configure two fundamental aspects of +object storage: maximum memory usage per [vnode][glossary vnode] +and object expiry. + +### Max Memory + +This setting specifies the maximum amount of memory consumed by the +Memory backend. It's important to note that this setting acts on a +*per-vnode basis*, not on a per-node or per-cluster basis. This should +be taken into account when planning for memory usage with the Memory +backend, as the total memory used will be max memory times the number +of vnodes in the cluster. + +When the threshold value that you set has been met in a particular +vnode, Riak will begin discarding objects, beginning with the oldest +object and proceeding until memory usage returns below the allowable +threshold. + +You can configure maximum memory using the +`memory_backend.max_memory_per_vnode` setting. You can specify +`max_memory_per_vnode` however you'd like, using kilobytes, megabytes, +or even gigabytes. + +The following are all possible settings: + +```riakconf +memory_backend.max_memory_per_vnode = 500KB +memory_backend.max_memory_per_vnode = 10MB +memory_backend.max_memory_per_vnode = 2GB +``` + +```appconfig +%% In the app.config-based system, the equivalent setting is max_memory, +%% which must be expressed in megabytes: + +{riak_kv, [ + %% storage_backend specifies the Erlang module defining the storage + %% mechanism that will be used on this node. + + {storage_backend, riak_kv_memory_backend}, + {memory_backend, [ + ..., + {max_memory, 4096}, %% 4GB in megabytes + ... + ]} +``` + +To determine an optimal max memory setting, we recommend consulting the +documentation on [LevelDB cache size][plan backend leveldb]. + +### TTL + +The time-to-live (TTL) parameter specifies the amount of time an object +remains in memory before it expires. The minimum time is one second. + +In the newer, `riak.conf`-based configuration system, you can specify +`ttl` in seconds, minutes, hours, days, etc. The following are all +possible settings: + +```riakconf +memory_backend.ttl = 1s +memory_backend.ttl = 10m +memory_backend.ttl = 3h +``` + +```appconfig +%% In the app.config-based system, the ttl setting must be expressed in +%% seconds: + +{memory_backend, [ + %% other settings + {ttl, 86400}, %% Set to 1 day + %% other settings + ]} +``` + +> **Dynamically Changing `ttl`** +> +> There is currently no way to dynamically change the `ttl` setting for a +bucket or bucket type. The current workaround would be to define +multiple Memory backends using the Multi backend, each with different +`ttl` values. For more information, consult the documentation on the +[Multi][plan backend multi] backend. diff --git a/content/riak/kv/2.2.6/setup/planning/backend/multi.md b/content/riak/kv/2.2.6/setup/planning/backend/multi.md new file mode 100644 index 0000000000..18626cb1d2 --- /dev/null +++ b/content/riak/kv/2.2.6/setup/planning/backend/multi.md @@ -0,0 +1,226 @@ +--- +title: "Multi-backend" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Multi-backend" + identifier: "planning_backend_multi" + weight: 103 + parent: "planning_choose_backend" +toc: true +aliases: + - /riak-docs/riak/2.2.6/ops/advanced/backends/multi/ + - /riak-docs/riak/kv/2.2.6/ops/advanced/backends/multi/ +--- + +[concept buckets]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/buckets +[plan backend bitcask]: {{<baseurl>}}riak/kv/2.2.6/setup/planning/backend/bitcask +[plan backend leveldb]: {{<baseurl>}}riak/kv/2.2.6/setup/planning/backend/leveldb +[plan backend memory]: {{<baseurl>}}riak/kv/2.2.6/setup/planning/backend/memory +[config reference]: {{<baseurl>}}riak/kv/2.2.6/configuring/reference +[usage bucket types]: {{<baseurl>}}riak/kv/2.2.6/developing/usage/bucket-types +[use admin riak-admin cli]: {{<baseurl>}}riak/kv/2.2.6/using/admin/riak-admin + +Riak allows you to run multiple backends within a single Riak cluster. +Selecting the Multi backend enables you to use different storage +backends for different [buckets][concept buckets]. Any combination of the three +available backends---[Bitcask][plan backend bitcask], [LevelDB][plan backend leveldb], and [Memory][plan backend memory]---can be used. + +## Configuring Multiple Backends + +You can set up your cluster to use the Multi backend using Riak's +[configuration files][config reference]. + +```riakconf +storage_backend = multi +``` + +```appconfig +{riak_kv, [ + %% ... + {storage_backend, riak_kv_multi_backend}, + %% ... +]}, +``` + +Remember that you must stop and then re-start each node when you change +storage backends or modify any other configuration. + +## Using Multiple Backends + +In Riak 2.0 and later, we recommend using multiple backends by applying +them to buckets [using bucket types][usage bucket types]. Assuming that the cluster has already been configured to use the `multi` backend, this process +involves three steps: + +1. Creating a bucket type that enables buckets of that type to use the + desired backends +2. Activating that bucket type +3. Setting up your application to use that type + +Let's say that we've set up our cluster to use the Multi backend and we +want to use [LevelDB][plan backend leveldb] and the [Memory][plan backend memory] backend for different sets of data. First, we need to create two bucket types, one which sets the `backend` bucket property to `leveldb` and the other which sets that property to `memory`. All bucket type-related activity is performed through the [`riak-admin`][use admin riak-admin cli] command interface. + +We'll call our bucket types `leveldb_backend` and `memory_backend`, but +you can use whichever names you wish. + +```bash +riak-admin bucket-type create leveldb_backend '{"props":{"backend":"leveldb"}}' +riak-admin bucket-type create memory_backend '{"props":{"backend":"memory"}}' +``` + +Then, we must activate those bucket types so that they can be used in +our cluster: + +```bash +riak-admin bucket-type activate leveldb_backend +riak-admin bucket-type activate memory_backend +``` + +Once those types have been activated, any objects stored in buckets +bearing the type `leveldb_backend` will be stored in LevelDB, whereas +all objects stored in buckets of the type `memory_backend` will be +stored in the Memory backend. + +More information can be found in our documentation on [using bucket types][usage bucket types]. + +## Configuring Multiple Backends + +Once you've set up your cluster to use multiple backends, you can +configure each backend on its own. All configuration options available +for LevelDB, Bitcask, and Memory are all available to you when using the +Multi backend. + +#### Using the Newer Configuration System + +If you are using the newer, `riak.conf`-based [configuration system][config reference], you can configure the backends by +prefacing each configuration with `multi_backend`. + +Here is an example of the general form for configuring multiple +backends: + +```riakconf +multi_backend.$name.$setting_name = setting +``` + +If you are using, for example, the LevelDB and Bitcask backends and wish +to set LevelDB's `bloomfilter` setting to `off` and the Bitcask +backend's `io_mode` setting to `nif`, you would do that as follows: + +```riakconf +multi_backend.leveldb.bloomfilter = off +multi_backend.bitcask.io_mode = nif +``` + +#### Using the Older Configuration System + +If you are using the older, `app.config`-based configuration system, +configuring multiple backends involves adding one or more backend- +specific sections to your `riak_kv` settings (in addition to setting +the `storage_backend` setting to `riak_kv_multi_backend`, as shown +above). + +> **Note**: If you are defining multiple file-based backends of the same +type, each of these must have a separate `data_root` directory defined. + +While all configuration parameters can be placed anywhere within the +`riak_kv` section of `app.config`, in general we recommend that you +place them in the section containing other backend-related settings to +keep the settings organized. + +Below is the general form for your `app.config` file: + +```appconfig +{riak_kv, [ + %% ... + {multi_backend_default, <<"bitcask_mult">>}, + {multi_backend, [ + %% Here's where you set the individual multiplexed backends + {<<"bitcask_mult">>, riak_kv_bitcask_backend, [ + %% bitcask configuration + {data_root, "/var/lib/riak/bitcask_mult/"}, + {config1, ConfigValue1}, + {config2, ConfigValue2} + ]}, + {<<"bitcask_expiry_mult">>, riak_kv_bitcask_backend, [ + %% bitcask configuration + {data_root, "/var/lib/riak/bitcask_expiry_mult/"}, + {expiry_secs, 86400}, + {config1, ConfigValue1}, + {config2, ConfigValue2} + ]}, + {<<"eleveldb_mult">>, riak_kv_eleveldb_backend, [ + %% eleveldb configuration + {config1, ConfigValue1}, + {config2, ConfigValue2} + ]}, + {<<"second_eleveldb_mult">>, riak_kv_eleveldb_backend, [ + %% eleveldb with a different configuration + {config1, ConfigValue1}, + {config2, ConfigValue2} + ]}, + {<<"memory_mult">>, riak_kv_memory_backend, [ + %% memory configuration + {config1, ConfigValue1}, + {config2, ConfigValue2} + ]} + ]}, + %% ... +]}, +``` + +Note that in each of the subsections of the `multi_backend` setting, the +name of each backend you wish to configure can be anything you would +like. Directly after naming the backend, you must specify which of the +backends corresponds to that name, i.e. `riak_kv_bitcask_backend`, +`riak_kv_eleveldb_backend`, or `riak_kv_memory_backend`. Once you have +done that, the various configurations for each named backend can be set +as objects in an Erlang list. + +## Example Configuration + +Imagine that you are using both Bitcask and LevelDB in your cluster, and +you would like storage to default to Bitcask. The following +configuration would create two backend configurations, named +`bitcask_mult` and `leveldb_mult`, respectively, while also setting the +data directory for each backend and specifying that `bitcask_mult` is +the default. + +```riakconf +storage_backend = multi + +multi_backend.bitcask_mult.storage_backend = bitcask +multi_backend.bitcask_mult.bitcask.data_root = /var/lib/riak/bitcask_mult + +multi_backend.leveldb_mult.storage_backend = leveldb +multi_backend.leveldb_mult.leveldb.data_root = /var/lib/riak/leveldb_mult + +multi_backend.default = bitcask_mult +``` + +```appconfig +{riak_kv, [ + %% ... + {multi_backend_default, <<"bitcask_mult">>}, + {multi_backend, [ + {<<"bitcask_mult", riak_kv_bitcask_backend, [ + {data_root, "/var/lib/riak/bitcask"} + ]}, + {<<"leveldb_mult", riak_kv_eleveldb_backend, [ + {data_root, "/var/lib/riak/leveldb"} + ]} + ]} + %% ... +]} +``` + +## Multi Backend Memory Use + +Each Riak storage backend has settings for configuring how much memory +the backend can use, e.g. caching for LevelDB or for the entire set of +data for the Memory backend. Each of these backends suggests allocating +up to 50% of available memory for this purpose. When using the Multi +backend, make sure that the sum of all backend memory use is at 50% +or less. For example, using three backends with each set to 50% memory +usage will inevitably lead to memory problems. diff --git a/content/riak/kv/2.2.6/setup/planning/best-practices.md b/content/riak/kv/2.2.6/setup/planning/best-practices.md new file mode 100644 index 0000000000..207aa93e66 --- /dev/null +++ b/content/riak/kv/2.2.6/setup/planning/best-practices.md @@ -0,0 +1,141 @@ +--- +title: "Scaling and Operating Riak Best Practices" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Best Practices" + identifier: "planning_best_practices" + weight: 105 + parent: "planning" +toc: true +aliases: + - /riak-docs/riak/2.2.6/ops/building/planning/best-practices + - /riak-docs/riak/kv/2.2.6/ops/building/planning/best-practices +--- + +[use ref handoff]: {{<baseurl>}}riak/kv/2.2.6/using/reference/handoff +[config mapreduce]: {{<baseurl>}}riak/kv/2.2.6/configuring/mapreduce +[glossary aae]: {{<baseurl>}}riak/kv/2.2.6/learn/glossary/#active-anti-entropy-aae +[cluster ops add remove node]: {{<baseurl>}}riak/kv/2.2.6/using/cluster-operations/adding-removing-nodes + +Riak KV is a database designed for easy operation and scaling. Below are some best practices that will enable you to improve performance and reliability at all stages in the life of your Riak cluster. + +## Disk Capacity + +Filling up disks is a serious problem in Riak. In general, you should +add capacity under the following conditions: + +* a disk becomes more than 80% full +* you have fewer than 10 days of capacity remaining at current rates of + growth + +## RAID Levels + +Riak provides resilience through its built-in redundancy. + +* RAID0 can be used to increase the performance at the expense of + single-node reliability +* RAID5/6 can be used to increase the reliability over RAID0 but still + offers higher performance than single disks +* You should choose a RAID level (or no RAID) that you’re comfortable + with + +## Disk Leeway + +* Adding new nodes instantly increases the total capacity of the + cluster, but you should allow enough internal network capacity that + [handing off][use ref handoff] existing data outpaces the arrival of new + data. +* Once you’ve reached a scale at which the amount of new data arriving + is a small fraction of the cluster's total capacity, you can add new + nodes when you need them. You should be aware, however, that adding + new nodes can actually _increase_ disk usage on existing nodes in the + short term as data is rebalanced within the cluster. +* If you are certain that you are likely to run out of capacity, we + recommend allowing a week or two of leeway so that you have plenty of + time to add nodes and for [handoff][use ref handoff] to occur before the disks reach + capacity +* For large volumes of storage it's usually prudent to add more capacity + once a disk is 80% full + +## CPU Capacity Leeway + +* In a steady state, your peak CPU utilization, ignoring other + processes, should be less than 30% +* If you provide sufficient CPU capacity leeway, you’ll have spare + capacity to handle other processes, such as backups, [handoff][use ref handoff], and [active anti-entropy][glossary aae] + +## Network Capacity Leeway + +* Network traffic tends to be “bursty,” i.e. it tends to vary both quite + a bit and quickly +* Your normal load, as averaged over a 10-minute period, should be no + more than 20% of maximum capacity +* Riak generates 3-5 times the amount of intra-node traffic as inbound + traffic, so you should allow for this in your network design + +## When to Add Nodes + +You should add more nodes in the following scenarios: + +* you have reached 80% of storage capacity +* you have less than 10 days of leeway before you expect the cluster to + fill up +* the current node's IO/CPU activity is higher than average for extended + period of time, especially for [MapReduce][config mapreduce] + operations + +An alternative to adding more nodes is to add more storage to existing +nodes. However, you should do this only if: + +* you’re confident that there is plenty of spare network and CPU + capacity, _and_ +* you can upgrade storage _equally across all nodes_. If storage is + added in an unbalanced fashion, Riak will continue storing data + equally across nodes, and the node with the smallest available storage + space is likely to fail first. Thus, if one node uses 1 TB but the + rest use 1.5 TB, Riak will overload the 1 TB node first. + +The recommendations above should be taken only as general guidelines +because the specifics of your cluster will matter a great deal when +making capacity decisions. The following considerations are worth +bearing in mind: + +* If your disks are 90% full but only filling up 1% per month, this + might be a perfectly "safe" scenario. In cases like this, the velocity + of adding new data is more important than any raw total. +* The burstiness of your write load is also an important consideration. + If writes tend to come in large batches that are unpredictably timed, + it can be more difficult to estimate when disks will become full, + which means that you should probably over-provision storage as a + precaution. +* If Riak shares disks with other processes or is on the system root + mount point, i.e. `/`, we recommend leaving a little extra disk space + in addition to the estimates discussed above, as other system + processes might use disk space unexpectedly. + +## How to Add Nodes + +* You should add as many additional nodes as you require in one + operation +* Don’t add nodes one at a time if you’re adding multiple nodes +* You can limit the transfer rate so that priority is given to live + customer traffic + +This process is explored in more detail in [Adding and Removing Nodes][cluster ops add remove node]. + +## Scaling + +* All large-scale systems are bound by the availability of some + resources +* From a stability point of view, the best state for a busy Riak cluster + to maintain is the following: + * New network connections are limited to ensure that existing network + connections consume most network bandwidth + * CPU at < 30% + * Disk IO at < 90% +* You should use HAProxy or your application servers to limit new + network connections to keep network and IO below 90% and CPU below + 30%. diff --git a/content/riak/kv/2.2.6/setup/planning/bitcask-capacity-calc.md b/content/riak/kv/2.2.6/setup/planning/bitcask-capacity-calc.md new file mode 100644 index 0000000000..77e8f469b0 --- /dev/null +++ b/content/riak/kv/2.2.6/setup/planning/bitcask-capacity-calc.md @@ -0,0 +1,100 @@ +--- +title: "Bitcask Capacity Calculator" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Bitcask Capacity Calculator" + identifier: "planning_cluster_bitcask_capacity" + weight: 104 + parent: "planning" +toc: true +aliases: + - /riak-docs/riak/2.2.6/ops/building/planning/bitcask + - /riak-docs/riak/kv/2.2.6/ops/building/planning/bitcask +--- + +[plan backend bitcask]: {{<baseurl>}}riak/kv/2.2.6/setup/planning/backend/bitcask + +These calculators will assist you in sizing your cluster if you plan to +use the default [Bitcask][plan backend bitcask] storage back end. + +This page is designed to give you a rough estimate when sizing your +cluster. The calculations are a _best guess_, and they tend to be a bit +on the conservative side. It's important to include a bit of head room +as well as room for unexpected growth so that if demand exceeds +expectations you'll be able to add more nodes to the cluster and stay +ahead of your requirements. + +<div id="node_info" class="calc_info"></div> +<div class="calculator"> + <ul> + <li> + <label for="n_total_keys">Total Number of Keys:</label> + <input id="n_total_keys" type="text" size="12" name="n_total_keys" value="" class="calc_input"> + <span class="error_span" id="n_total_keys_error"></span> + </li> + <li> + <label for="n_bucket_size">Average Bucket Size (Bytes):</label> + <input id="n_bucket_size"type="text" size="7" name="n_bucket_size" value="" class="calc_input"> + <span class="error_span"id="n_bucket_size_error"></span> + </li> + <li> + <label for="n_key_size">Average Key Size (Bytes):</label> + <input type="text" size="2" name="n_key_size" id="n_key_size" value="" class="calc_input"> + <span class="error_span" id="n_key_size_error"></span> + </li> + <li> + <label for="n_record_size">Average Value Size (Bytes):</label> + <input id="n_record_size"type="text" size="7" name="n_record_size" value="" class="calc_input"> + <span class="error_span"id="n_record_size_error"></span> + </li> + <li> + <label for="n_ram">RAM Per Node (in GB):</label> + <input type="text" size="4" name="n_ram" id="n_ram" value="" class="calc_input"> + <span class="error_span" id="n_ram_error"></span> + </li> + <li> + <label for="n_nval"><i>N</i> (Number of Write Copies):</label> + <input type="text" size="2" name="n_nval" id="n_nval" value="" class="calc_input"> + <span class="error_span" id="n_nval_error"></span> + </li> +</ul> +</div> + +## Recommendations + +<span id="recommend"></span> + +## Details on Bitcask RAM Calculation + +With the above information in mind, the following variables will factor +into your RAM calculation: + +Variable | Description +:--------|:----------- +Static Bitcask per-key overhead | 44.5 bytes per key +Estimated average bucket-plus-key length | The combined number of characters your bucket + keynames will require (on average). We'll assume 1 byte per character. +Estimated total objects | The total number of key/value pairs your cluster will have when started +Replication Value (`n_val`) | The number of times each key will be replicated when written to Riak (the default is 3) + +## The actual equation + +Approximate RAM Needed for Bitcask = (static bitcask per key overhead + +estimated average bucket+key length in bytes) * estimate total number of +keys * `n_val` + +Example: + +* 50,000,000 keys in your cluster to start +* approximately 30 bytes for each bucket+key name +* default `n_val` of 3 + +The amount of RAM you would need for Bitcask is about **9.78 GBs across +your entire cluster.** + +Additionally, Bitcask relies on your operating system's filesystem cache +to deliver high performance reads. So when sizing your cluster, take +this into account and plan on having several more gigabytes of RAM +available for your filesystem cache. diff --git a/content/riak/kv/2.2.6/setup/planning/cluster-capacity.md b/content/riak/kv/2.2.6/setup/planning/cluster-capacity.md new file mode 100644 index 0000000000..baf8c04d1e --- /dev/null +++ b/content/riak/kv/2.2.6/setup/planning/cluster-capacity.md @@ -0,0 +1,234 @@ +--- +title: "Cluster Capacity Planning" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Cluster Capacity" + identifier: "planning_cluster_capacity" + weight: 103 + parent: "planning" +toc: true +aliases: + - /riak-docs/riak/2.2.6/ops/building/planning/cluster + - /riak-docs/riak/kv/2.2.6/ops/building/planning/cluster +--- + +[plan backend leveldb]: {{<baseurl>}}riak/kv/2.2.6/setup/planning/backend/leveldb +[plan bitcask capacity]: {{<baseurl>}}riak/kv/2.2.6/setup/planning/bitcask-capacity-calc +[plan index]: {{<baseurl>}}riak/kv/2.2.6/setup/planning +[concept replication]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/replication +[use admin riak-admin#cluster]: {{<baseurl>}}riak/kv/2.2.6/using/admin/riak-admin/#cluster +[config reference]: {{<baseurl>}}riak/kv/2.2.6/configuring/reference +[perf benchmark]: {{<baseurl>}}riak/kv/2.2.6/using/performance/benchmarking +[LVM]: http://en.wikipedia.org/wiki/Logical_Volume_Manager_(Linux) + + +This document outlines the various elements and variables to keep in mind when planning your Riak cluster. Your use case and environment variables will be specific to what you're building, but this document should set you on the right path when planning and launching a Riak cluster. + +## RAM + +[RAM](http://en.wikipedia.org/wiki/Random-access_memory) is the most important resource when sizing your Riak cluster. Memory keeps data closer to your users. Memory is essential for running complex MapReduce queries or caching data to provide low-latency request times. + +### Bitcask and Memory Requirements + +Your choice of local storage backend for Riak impacts your RAM +needs. Though Riak has pluggable backend storage, Bitcask is the +default. Why? Because it's built for: + +* low-latency request times +* high throughput +* the ability to handle data sets much larger than RAM w/o degradation + +Bitcask's one major requirement, however, is that it must keep the +entire **keydir** in memory. The keydir is a hash table that maps each +concatenated bucket + key name in a Bitcask (“a Bitcask” is the name for +each file contained within each Bitcask backend) to a fixed-size +structure giving the file, offset, and size of the most recently written +entry for that bucket + key on disk. + +To learn about Bitcask see [Hello Bitcask](http://basho.com/hello-bitcask/) on the Basho blog as well as the [Introduction to Bitcask](http://basho.com/assets/bitcask-intro.pdf) paper. + +If your calculated RAM needs will exceed your hardware resources--in other words, if you can't afford the RAM to use Bitcask---we recommend that you use LevelDB. + +Check out [Bitcask Capacity Planning][plan bitcask capacity] for more details on designing a Bitcask-backed cluster. + +### LevelDB + +If RAM requirements for Bitcask are prohibitive, we recommend use of +the LevelDB backend. While LevelDB doesn't require a large amount of RAM +to operate, supplying it with the maximum amount of memory available leads to higher performance. + +For more information see [LevelDB][plan backend leveldb]. + +## Disk + +Now that you have an idea of how much RAM you'll need, it's time to think about disk space. Disk space needs are much easier to calculate. Below is an equation to help you calculate disk space needs: + +#### Estimated Total Objects * Average Object Size * n_val + +For example: + +* 50,000,000 objects +* an average object size of two kilobytes (2,048 bytes) +* the default `n_val` of 3 + +Then you would need just over approximately **286 GB** of disk space in the entire cluster to accommodate your data. + +We believe that databases should be durable out of the box. When we +built Riak, we did so in a way that you could write to disk while +keeping response times below your users' expectations. So this +calculation assumes that you'll be keeping the entire data set on disk. + +Many of the considerations taken when configuring a machine to serve a +database apply to configuring a node for Riak as well. Mounting +disks with noatime and having separate disks for your OS and Riak data +lead to much better performance. See [Planning for a +Riak System](../start) for more information. + +### Disk Space Planning and Ownership Handoff + +When Riak nodes fail or leave the cluster, other nodes in the cluster start the **ownership handoff** process. Ownership handoff is when remaining nodes take ownership of the data partitions handled by an absent node. One side effect of this process is that the other nodes require more intensive disk space usage; in rare cases filling the disk of one or more of those nodes. + +When making disk space planning decisions, we recommend that you: + +* assume that one or more nodes may be down at any time +* monitor your disk space usage and add additional space when usage + exceeds 50-60% of available space. + +Another possibility worth considering is using Riak with a filesystem +that allows for growth, for example +[LVM], +[RAID](http://en.wikipedia.org/wiki/RAID), or +[ZFS](http://en.wikipedia.org/wiki/ZFS). + +## Read/Write Profile + +Read/write ratios, as well as the distribution of key access, should +influence the configuration and design of your cluster. If your use case +is write heavy, you will need less RAM for caching, and if only a +certain portion of keys is accessed regularly, such as a [Pareto +distribution](http://en.wikipedia.org/wiki/Pareto_distribution), you +won't need as much RAM available to cache those keys' values. + +## Number of Nodes + +The number of nodes (i.e. physical servers) in your Riak Cluster depends +on the number of times data is [replicated][concept replication] across the +cluster. To ensure that the cluster is always available to respond to +read and write requests, we recommend a "sane default" of N=3 +replicas. This requirement can be met with a 3 or 4-node +cluster. + +For production deployments, however, we recommend using no fewer than 5 +nodes, as node failures in smaller clusters can compromise the +fault-tolerance of the system. Additionally, in clusters smaller than 5 +nodes, a high percentage of the nodes (75-100% of them) will need to +respond to each request, putting undue load on the cluster that may +degrade performance. For more details on this recommendation, see our +blog post on [Why Your Riak Cluster Should Have at Least Five +Nodes](http://basho.com/posts/technical/Why-Your-Riak-Cluster-Should-Have-At-Least-Five-Nodes/). + +## Scaling + +Riak can be scaled in two ways: vertically, via improved hardware, and +horizontally, by adding more nodes. Both ways can provide performance +and capacity benefits, but should be used in different circumstances. +The [riak-admin cluster command][use admin riak-admin#cluster] can +assist scaling in both directions. + +#### Vertical Scaling + +Vertical scaling, or improving the capabilities of a node/server, +provides greater capacity to the node but does not decrease the overall +load on existing members of the cluster. That is, the ability of the +improved node to handle existing load is increased but the load itself +is unchanged. Reasons to scale vertically include increasing IOPS (I/O +Operations Per Second), increasing CPU/RAM capacity, and increasing disk +capacity. + +#### Horizontal Scaling + +Horizontal scaling, or increasing the number of nodes in the cluster, +reduces the responsibilities of each member node by reducing the number +of partitions and providing additional endpoints for client connections. +That is, the capacity of each individual node does not change but its +load is decreased. Reasons to scale horizontally include increasing I/O +concurrency, reducing the load on existing nodes, and increasing disk +capacity. + +> **Note on horizontal scaling** +> +> When scaling horizontally, it's best to add all planned nodes at once +with multiple `riak-admin cluster join` commands followed by +a `riak-admin cluster plan` and `riak-admin cluster commit`. This will help reduce the amount of data transferred between nodes in the cluster. + +#### Reducing Horizontal Scale + +If a Riak cluster is over provisioned, or in response to seasonal usage decreases, the horizontal scale of a Riak cluster can be decreased using the `riak-admin cluster leave` command. + +## Ring Size/Number of Partitions + +Ring size is the number of partitions that make up your Riak cluster. Ring sizes must be a power of 2. Ring size is configured before your cluster is started, and is set in your [configuration files][config reference]. + +The default number of partitions in a Riak cluster is 64. This works for smaller clusters, but if you plan to grow your cluster past 5 nodes we recommend a larger ring size. + +The minimum number of partitions recommended per node is 10. You can determine the number of partitions allocated per node by dividing the number of partitions by the number of nodes. + +There are no absolute rules for the ideal partitions-per-node ratio. This depends on your particular use case and what features the Riak cluster uses. We recommend between 10 and 50 data partitions per node. + +So if you're running a 3-node development cluster, a ring size of 64 or 128 should work just fine. While a 10-node cluster should work well with a ring size of 128 or 256 (64 is too small while 512 is likely too large). + +The table below provides some suggested combinations: + +Number of nodes | Number of data partitions +:---------------|:------------------------- +3, 4, 5 | 64, 128 +5 | 64, 128 +6 | 64, 128, 256 +7, 8, 9, 10 | 128, 256 +11, 12 | 128, 256, 512 + +By extension, a ring size of 1024 is advisable only in clusters with +more than 20 nodes, 2048 in clusters with more than 40 nodes, etc. + +If you're unsure about the best number of partitions to use, consult the +[Riak mailing +list](http://lists.basho.com/mailman/listinfo/riak-users_lists.basho.com) +for suggestions from the Riak community. + +## Other Factors + +Riak is built to run in a clustered environment, and while it will +compensate for network partitions, they do cause increased load on the +system. In addition, running in a virtualized environment that lacks +low-latency IO access can drastically decrease performance. Before +putting your Riak cluster in production is recommended that you gain a +full understanding of your environment's behavior so that you know how +your cluster performs under load for an extended period of time. Doing +so will help you size your cluster for future growth and lead to optimal +performance. + +We recommend using [Basho Bench][perf benchmark] for benchmarking the performance of your cluster. + +### Bandwidth + +Riak uses Erlang's built-in distribution capabilities to provide +reliable access to data. A Riak cluster can be deployed in many +different network environments. We recommend that you produce as +little latency between nodes as possible, as high latency leads to +sub-optimal performance. + +Deploying a single Riak cluster across two datacenters is not recommended. If your use case requires this capability, Riak offers a [Multi Data Center Replication: Architecture](../../../using/reference/v3-multi-datacenter/architecture) option that is built to keep multiple Riak clusters in +sync across several geographically diverse deployments. + +### I/O + +In general, the biggest bottleneck for Riak will be the amount of I/O +available to it, especially in the case of write-heavy workloads. Riak +functions much like any other database and the design of your disk +access should take this into account. Because Riak is clustered and your +data is stored on multiple physical nodes, you should consider forgoing +a traditional RAID setup for redundancy and focus on providing the least +latency possible using SATA Drives or SSDs, for example. diff --git a/content/riak/kv/2.2.6/setup/planning/future.md b/content/riak/kv/2.2.6/setup/planning/future.md new file mode 100644 index 0000000000..13540869fe --- /dev/null +++ b/content/riak/kv/2.2.6/setup/planning/future.md @@ -0,0 +1,16 @@ +--- +draft: true +title: "Planning for the Future" +description: "" +project: "riak_kv" +project_version: "2.2.6" +#menu: +# riak_kv-2.2.6: +# name: "Planning for the Future" +# identifier: "planning_future" +# weight: 106 +# parent: "planning" +toc: true +--- + +**TODO: Add content** diff --git a/content/riak/kv/2.2.6/setup/planning/operating-system.md b/content/riak/kv/2.2.6/setup/planning/operating-system.md new file mode 100644 index 0000000000..13ad4afc48 --- /dev/null +++ b/content/riak/kv/2.2.6/setup/planning/operating-system.md @@ -0,0 +1,25 @@ +--- +title: "Operating System Support" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "OS Support" + identifier: "planning_os" + weight: 101 + parent: "planning" +toc: true +--- + +[downloads]: {{<baseurl>}}riak/kv/2.2.6/downloads/ + +We recommend deploying Riak KV on a mainstream Unix-like operating system. +Mainstream distributions have larger support communities, making +solutions to common problems easier to find. + +Basho provides [binary packages][downloads] of Riak KV for the following distributions: + +* **Red Hat based:** Red Hat Enterprise Linux, CentOS, Fedora Core +* **Debian based:** Debian, Ubuntu +* **Solaris based:** Sun Solaris, OpenSolaris diff --git a/content/riak/kv/2.2.6/setup/planning/start.md b/content/riak/kv/2.2.6/setup/planning/start.md new file mode 100644 index 0000000000..56910560e4 --- /dev/null +++ b/content/riak/kv/2.2.6/setup/planning/start.md @@ -0,0 +1,57 @@ +--- +title: "Start Planning" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Start Planning" + identifier: "planning_start" + weight: 100 + parent: "planning" +toc: true +aliases: + - /riak-docs/riak/2.2.6/ops/building/planning/system-planning + - /riak-docs/riak/kv/2.2.6/ops/building/planning/system-planning +--- + +[plan backend]: {{<baseurl>}}riak/kv/2.2.6/setup/planning/backend +[plan cluster capacity]: {{<baseurl>}}riak/kv/2.2.6/setup/planning/cluster-capacity +[plan backend bitcask]: {{<baseurl>}}riak/kv/2.2.6/setup/planning/backend/bitcask +[plan bitcask capacity]: {{<baseurl>}}riak/kv/2.2.6/setup/planning/bitcask-capacity-calc + +Here are some steps and recommendations designing and configuring your +Riak cluster. + +## Backend + +Backends are what Riak KV uses to persist data. Different backends have +strengths and weaknesses, so if you are unsure of which backend you +need, read through the [Choosing a Backend][plan backend] tutorial. + +## Capacity + +[Cluster Capacity Planning][plan cluster capacity] outlines the various elements and variables that should be considered when planning your Riak cluster. + +If you have chosen [Bitcask][plan backend bitcask] as your backend, you will also want to run through [Bitcask Capacity Planning][plan bitcask capacity] to help you calculate a reasonable capacity. + +## Network Configuration / Load Balancing + +There are at least two acceptable strategies for load-balancing requests +across your Riak cluster: **virtual IPs** and **reverse-proxy**. + +For **virtual IPs**, we recommend using any of the various VIP +implementations. We don't recommend VRRP behavior for the VIP because +you'll lose the benefit of spreading client query load to all nodes in a +ring. + +For **reverse-proxy** configurations (HTTP interface), any one of the +following should work adequately: + +* haproxy +* squid +* varnish +* nginx +* lighttpd +* Apache + diff --git a/content/riak/kv/2.2.6/setup/search.md b/content/riak/kv/2.2.6/setup/search.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/content/riak/kv/2.2.6/setup/upgrading.md b/content/riak/kv/2.2.6/setup/upgrading.md new file mode 100644 index 0000000000..639d175fcc --- /dev/null +++ b/content/riak/kv/2.2.6/setup/upgrading.md @@ -0,0 +1,33 @@ +--- +title: "Upgrading Riak KV" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Upgrading" + identifier: "upgrading" + weight: 102 + parent: "setup_index" +toc: true +--- + +[upgrade checklist]: ./checklist +[upgrade version]: ./version +[upgrade cluster]: ./cluster +[upgrade mdc]: ./multi-datacenter +[upgrade search]: ./search + +## In This Section + +### [Production Checklist][upgrade checklist] + +An overview of what to consider before upgrading Riak KV in a production environment. + +[Learn More >>][upgrade checklist] + +### [Upgrading to Riak KV 2.2.6][upgrade version] + +A tutorial on updating to Riak KV 2.2.6 + +[Learn More >>][upgrade version] \ No newline at end of file diff --git a/content/riak/kv/2.2.6/setup/upgrading/checklist.md b/content/riak/kv/2.2.6/setup/upgrading/checklist.md new file mode 100644 index 0000000000..49ae21064b --- /dev/null +++ b/content/riak/kv/2.2.6/setup/upgrading/checklist.md @@ -0,0 +1,220 @@ +--- +title: "Production Checklist" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Production Checklist" + identifier: "upgrading_checklist" + weight: 100 + parent: "upgrading" +toc: true +aliases: + - /riak-docs/riak/2.2.6/ops/upgrading/production-checklist/ + - /riak-docs/riak/kv/2.2.6/ops/upgrading/production-checklist/ +--- + +[perf open files]: {{<baseurl>}}riak/kv/2.2.6/using/performance/open-files-limit +[perf index]: {{<baseurl>}}riak/kv/2.2.6/using/performance +[ntp]: http://www.ntp.org/ +[security basics]: {{<baseurl>}}riak/kv/2.2.6/using/security/basics +[cluster ops load balance]: {{<baseurl>}}riak/kv/2.2.6/configuring/load-balancing-proxy +[config reference]: {{<baseurl>}}riak/kv/2.2.6/configuring/reference +[config backend]: {{<baseurl>}}riak/kv/2.2.6/configuring/backend +[usage search]: {{<baseurl>}}riak/kv/2.2.6/developing/usage/search +[usage conflict resolution]: {{<baseurl>}}riak/kv/2.2.6/developing/usage/conflict-resolution +[concept eventual consistency]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/eventual-consistency +[apps replication properties]: {{<baseurl>}}riak/kv/2.2.6/developing/app-guide/replication-properties +[concept strong consistency]: {{<baseurl>}}riak/kv/2.2.6/using/reference/strong-consistency +[cluster ops bucket types]: {{<baseurl>}}riak/kv/2.2.6/using/cluster-operations/bucket-types +[use admin commands]: {{<baseurl>}}riak/kv/2.2.6/using/admin/commands +[use admin riak control]: {{<baseurl>}}riak/kv/2.2.6/using/admin/riak-control +[cluster ops inspect node]: {{<baseurl>}}riak/kv/2.2.6/using/cluster-operations/inspecting-node +[troubleshoot http]: {{<baseurl>}}riak/kv/2.2.6/using/troubleshooting/http-204 +[use admin riak-admin]: {{<baseurl>}}riak/kv/2.2.6/using/admin/riak-admin +[SANs]: http://en.wikipedia.org/wiki/Storage_area_network + +Deploying Riak KV to a realtime production environment from a development or testing environment can be a complex process. While the specifics of that process will always depend on your environment and practices, there are some basics for you to consider and a few questions that you will want to ask while making this transition. + +We've compiled these considerations and questions into separate categories for you to look over. + +## System + +* Are all systems in your cluster as close to identical as possible in + terms of both hardware and software? +* Have you set appropriate [open files limits][perf open files] on all + of your systems? +* Have you applied the [Riak KV performance improvement recommendations][perf index]? + +## Network + +* Are all systems using the same [NTP servers][ntp] to + synchronize clocks? +* Are you sure that your NTP clients' configuration is monotonic (i.e. + that your clocks will not roll back)? +* Is DNS correctly configured for all systems' production deployments? +* Are connections correctly routed between all Riak nodes? +* Are connections correctly set up in your load balancer? +* Are your [firewalls][security basics] correctly configured? +* Check that network latency and throughput are as expected for all of the + following (we suggest using [iperf][ntp] to verify): + - between nodes in the cluster + - between the load balancer and all nodes in the cluster + - between application servers and the load balancer +* Do all Riak nodes appear in the load balancer's rotation? +* Is the load balancer configured to balance connections with roundrobin + or a similarly random [distribution scheme][cluster ops load balance]? + +## Riak KV + +* Check [configuration files][config reference]: + - Does each machine have the correct name and IP settings in + `riak.conf` (or in `app.config` if you're using the older + configuration files)? + - Are all [configurable settings][config reference] identical + across the cluster? + - Have all of the settings in your configuration file(s) that were + changed for debugging purposes been reverted back to production + settings? + - If you're using [multiple data backends][config backend], are all of your + bucket types configured to use the correct backend? + - If you are using Riak Security, have you checked off all items in + the [security checklist][security basics] and turned on security? + - If you're using [multiple data backends][config backend], do all machines' + config files agree on their configuration? + - Do all nodes agree on the value of the [`allow_mult`][config basic] setting? + - Do you have a [sibling resolution][usage conflict resolution] strategy in + place if `allow_mult` is set to `true`? + - Have you carefully weighed the [consistency trade-offs][concept eventual consistency] that must be made if `allow_mult` is set to `false`? + - Are all of your [apps replication properties][apps replication properties] configured correctly and uniformly across the cluster? + - If you are using [Riak Search][usage search], is it enabled on all + nodes? If you are not, has it been disabled on all nodes? + - If you are using [strong consistency][concept strong consistency] for some or all of your + data: + * Does your cluster consist of at least three nodes? If it does + not, you will not be able to use this feature, and you are + advised against enabling it. + * If your cluster does consist of at least three nodes, has the + strong consistency subsystem been [enabled][config strong consistency] on all nodes? + * Is the [`target_n_val`][config reference] that is set on each node higher than any `n_val` that you intend to use for strongly consistent bucket types (or any bucket types for that matter)? The default is 4, which will likely need to be raised if you are using strong consistency. + - Have all [bucket types][cluster ops bucket types] that you intend to use + been created and successfully activated? + - If you are using [`riak_control`][use admin riak control], is it enabled on the node(s) from which you intend to use it? +* Check data mount points: + - Is `/var/lib/riak` mounted? + - Can you grow that disk later when it starts filling up? + - Do all nodes have their own storage systems (i.e. no + [SANs]), or do you have a plan in place for switching to that configuration later? +* Are all Riak KV nodes up? + - Run `riak ping` on all nodes. You should get `pong` as a response. + - Run `riak-admin wait-for-service riak_kv <node_name>@<IP>` on each + node. You should get `riak_kv is up` as a response. + + The `<node_name>@<IP>` string should come from your [configuration + file(s)][configure reference]. +* Do all nodes agree on the ring state? + - Run `riak-admin ringready`. You should get `TRUE ALL nodes agree on + the ring [list_of_nodes]`. + - Run `riak-admin member-status`. All nodes should be valid (i.e. + listed as `Valid: 1`), and all nodes should appear in the list + - Run `riak-admin ring-status`. The ring should be ready (`Ring Ready: + true`), there should be no unreachable nodes (`All nodes are up and + reachable`), and there should be no pending changes to the ring + (`No pending changes`). + - Run `riak-admin transfers`. There should be no active transfers (`No + transfers active`). + +## Operations + +* Does your monitoring system ensure that [NTP][ntp] is + running? +* Are you collecting [time series data][cluster ops inspect node] on + the whole cluster? + - System metrics + + CPU load + + Memory used + + Network throughput + + Disk space used/available + + Disk input/output operations per second (IOPS) + - Riak metrics (from the [`/stats`][troubleshoot http] HTTP endpoint or + using [`riak-admin`][use admin riak-admin]) + + Latencies: `GET` and `PUT` (mean/median/95th/99th/100th) + + Vnode stats: `GET`s, `PUT`s, `GET` totals, `PUT` totals + + Node stats: `GET`s, `PUT`s, `GET` totals, `PUT` totals + + Finite state machine (FSM) stats: + * `GET`/`PUT` FSM `objsize` (99th and 100th percentile) + * `GET`/`PUT` FSM `times` (mean/median/95th/99th/100th) + + Protocol buffer connection stats + * `pbc_connects` + * `pbc_active` + * `pbc_connects_total` +* Are the following being graphed (at least the key metrics)? + - Basic system status + - Median and 95th and 99th percentile latencies (as these tend to be + leading indicators of trouble) + +## Application and Load + +* Have you benchmarked your cluster with simulated load to confirm that + your configuration will meet your performance needs? +* Are the [develop client libraries] in use in your application up to date? +* Do the client libraries that you're using support the version of Riak KV + that you're deploying? + +## Confirming Configuration with Riaknostic + +Recent versions of Riak KV ship with Riaknostic, a diagnostic utility that +can be invoked by running `riak-admin diag <check>`, where `check` is +one of the following: + +* `disk` +* `dumps` +* `memory_use` +* `nodes_connected` +* `ring_membership` +* `ring_preflists` +* `ring_size` +* `search` +* `sysctl` + +Running `riak-admin diag` with no additional arguments will run all +checks and report the findings. This is a good way of verifying that +you've gotten at least some of the configurations mentioned above +correct, that all nodes in your cluster are up, and that nothing is +grossly misconfigured. Any warnings produced by `riak-admin diag` should +be addressed before going to production. + +## Troubleshooting and Support + +* Does your team, including developing and operations, know how to open + support requests with Basho? +* Is your team familiar with Basho Support's Service-Level Agreement + (SLA) levels? + - Normal and Low are for issues not immediately impacting production + systems + - High is for problems that impact production or soon-to-be-production + systems, but where stability is not currently compromised + - Urgent is for problems causing production outages or for those + issues that are likely to turn into production outages very soon. + On-call engineers respond to urgent requests within 30 minutes, + 24 / 7. +* Does your team know how to gather `riak-debug` results from the whole + cluster when opening tickets? If not, that process goes something like + this: + - SSH into each machine, run `riak-debug`, and grab the resultant + `.tar.gz` file + - Attach all debug tarballs from the whole cluster each time you open + a new High- or Urgent-priority ticket + +## The Final Step: Taking it to Production + +Once you've been running in production for a month or so, look back at +the metrics gathered above. Based on the numbers you're seeing so far, +configure alerting thresholds on your latencies, disk consumption, and +memory. These are the places most likely to give you advance warning of +trouble. + +When you go to increase capacity down the line, having historic metrics +will give you very clear indicators of having resolved scaling problems, +as well as metrics for understanding what to upgrade and when. diff --git a/content/riak/kv/2.2.6/setup/upgrading/multi-datacenter.md b/content/riak/kv/2.2.6/setup/upgrading/multi-datacenter.md new file mode 100644 index 0000000000..617df4d78d --- /dev/null +++ b/content/riak/kv/2.2.6/setup/upgrading/multi-datacenter.md @@ -0,0 +1,18 @@ +--- +draft: true +title: "Upgrading Multi-Datacenter" +description: "" +project: "riak_kv" +project_version: "2.2.6" +#menu: +# riak_kv-2.2.6: +# name: "Upgrading Multi-Datacenter" +# identifier: "upgrading_multi_datacenter" +# weight: 103 +# parent: "upgrading" +toc: true +--- + +## TODO + +How to update to a new version with multi-datacenter. diff --git a/content/riak/kv/2.2.6/setup/upgrading/version.md b/content/riak/kv/2.2.6/setup/upgrading/version.md new file mode 100644 index 0000000000..e6dc312a57 --- /dev/null +++ b/content/riak/kv/2.2.6/setup/upgrading/version.md @@ -0,0 +1,249 @@ +--- +title: "Upgrading to Riak KV 2.2.6" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Upgrading to 2.2.6" + identifier: "upgrading_version" + weight: 101 + parent: "upgrading" +toc: true +aliases: + - /riak-docs/riak/2.2.6/upgrade-v20/ + - /riak-docs/riak/kv/2.2.6/ops/upgrading/rolling-upgrades/ + - /riak-docs/riak/kv/2.2.6/ops/upgrading/rolling-upgrades/ + - /riak-docs/riak/kv/2.2.6/setup/upgrading/cluster/ + +--- + + +[production checklist]: {{<baseurl>}}riak/kv/2.2.6/setup/upgrading/checklist +[use admin riak control]: {{<baseurl>}}riak/kv/2.2.6/using/admin/riak-control +[use admin commands]: {{<baseurl>}}riak/kv/2.2.6/using/admin/commands +[use admin riak-admin]: {{<baseurl>}}riak/kv/2.2.6/using/admin/riak-admin +[usage secondary-indexes]: {{<baseurl>}}riak/kv/2.2.6/developing/usage/secondary-indexes +[release notes]: {{<baseurl>}}riak/kv/2.2.6/release-notes +[riak enterprise]: http://basho.com/products/riak-kv/ +[cluster ops mdc]: {{<baseurl>}}riak/kv/2.2.6/using/cluster-operations/v3-multi-datacenter +[config v3 mdc]: {{<baseurl>}}riak/kv/2.2.6/configuring/v3-multi-datacenter +[jmx monitor]: {{<baseurl>}}riak/kv/2.2.6/using/reference/jmx +[snmp]: {{<baseurl>}}riak/kv/2.2.6/using/reference/snmp +[Release Notes]: {{<baseurl>}}riak/kv/2.2.6/release-notes + + +## Overview + +You can upgrade one node or your whole cluster to Riak KV 2.2.6 by following the instructions below. + +{{% note title="Tip" %}} KV nodes negotiate with each other to determine supported operating modes. This allows clusters containing mixed-versions of Riak KV to interoperate without special configuration, and simplifies rolling upgrades. +{{% /note %}} + + +### General Process + +For every node in the cluster: + +1. Stop Riak KV. +1. Back up the Riak /etc, /data, and /basho-patches directories. +1. Remove your /basho-patches directory. +1. Upgrade Riak KV. + * If you are upgrading from EE to OSS, uninstall your EE KV package before upgrading. +1. (Optional) If you would like to potentially downgrade at some point, update your advanced.config file to opt-out of the AAE updates. +1. If you're upgrading from EE to OSS, apply your customized settings to vm.args/riak.conf and app.config/advanced.config +1. If you're using MDC replication to clusters with versions less than 2.2.0, update your advanced.config file to over-ride the default bucket properties for compatibility. +1. Start Riak KV. +1. Verify Riak KV is running the upgraded version. +1. Wait for the `riak_kv` service to start. +1. Wait for any hinted handoffs to complete. + +Before starting the rolling upgrade process on your cluster, check out the [Upgrading Riak KV: Production Checklist][production checklist], which covers details and questions to consider before upgrading. + + +## Data File Format Changes + +[Riak KV 2.2][release notes] introduces on-disk data file format changes that can impact the upgrade/downgrade process: + +* Changes to active anti-entropy related to inconsistent hashing. +* Upgrading to Solr 4.10.4 for Riak search. + +{{% note %}} +You must have [Java version 7 or higher](http://www.oracle.com/technetwork/java/javase/downloads/index.html) in order to upgrade to Riak KV 2.2.6 only if you plan to use Riak search. +{{% /note %}} + + +### Components That Complicate Downgrades + +We do our best to make all features that change data formats on disk opt-in; however, some features may be introduced that we either believe are so important that we automatically opt-in users on upgrade or there is no way to provide direct backward compatibility. Downgrading environments with these features can require more effort or might not be possible. + +* **Automatic** features alter the data format on disk, but are considered important enough for users to be automatically opted-in. +* **Required** features must be accepted as a part of the upgrade. Internal Solr version upgrades that change the data format on disk are an example of a required feature upgrade. +* **One Way** features, when enabled, will make a clean downgrade of a cluster impossible. + +| Feature | Automatic | Required | One Way | Notes | +|:---|:---:|:---:|:---:|:--- | +|Migration to Solr 4.10.4 |✔ | ✔| | Applies to all clusters using Riak search. +| Active anti-entropy file format changes | ✔ | | | Can opt-out using a capability. +| LZ4 compression in LevelDB | | | ✔ | +| Global expiration in LevelDB | | | ✔ | +| HyperLogLog data type | | |✔| On downgrade data written in HLL format is unreadable.| + + +### When Downgrading is No Longer an Option + +If you decide to upgrade to version 2.2, you can still downgrade your cluster to an earlier version of Riak KV if you wish, unless you perform one of the following actions in your cluster: + +* Enable LZ4 Compression in LevelDB +* Enable Global Expiration in LevelDB + +If you use other new features, such as the HyperLogLog data type, you can still downgrade your cluster, but you will no longer be able to use those features or access data in new formats after the downgrade. + + +## Upgrading process + +1\. Stop Riak KV on the node you are going to upgrade: + +```bash +riak stop +``` + +2\. Back up your /etc (app.config and vm.args), /data, and /basho-patches directories. + +```RHEL/CentOS +sudo tar -czf riak_backup.tar.gz /var/lib/riak /etc/riak /usr/lib64/riak/lib/basho-patches +``` + +```Ubuntu +sudo tar -czf riak_backup.tar.gz /var/lib/riak /etc/riak /usr/lib/riak/lib/basho-patches +``` + +3\. Remove your /basho-patches directory: + +```RHEL/CentOS +sudo rm -rf /usr/lib64/riak/lib/basho-patches/* +``` + +```Ubuntu +sudo rm -rf /usr/lib/riak/lib/basho-patches* +``` + +4\. Upgrade Riak KV: + +{{% note title="Upgrading from KV Enterprise Edition" %}} +If you are upgrading from Riak KV EE to Riak KV OSS, you must uninstall your Riak KV EE package right now, before you can install the OSS version. +{{% /note %}} + + + +```RHEL/CentOS +sudo rpm -Uvh »riak_package_name«.rpm +``` + +```Ubuntu +sudo dpkg -i »riak_package_name«.deb +``` + +5.a\. (**Optional**) If you would like to keep your AAE trees in a format that will facilitate downgrading, the capability override should be in the `riak_kv` proplist of the advanced.config file: + + ```advanced.config + {riak_kv, [ + {override_capability, [ + {object_hash_version, [{use, legacy}] } + ]} + ]} + ``` + +5.b\. (**Optional**) If you would like to keep your leveldb compression in a format that will facilitate downgrading, the capability override should be in riak.conf: + + ```riak.conf + leveldb.compression.algorithm=snappy + ``` + +5.c\. (**OSS Only**)If you are upgrading from Riak KV OSS =< 2.2.3, you must perform the following steps before moving on: + +* A standard package uninstall should not have removed your data directories, but if it did, move your backup to where the data directory should be. +* Then copy any customizations from your backed-up vm.args/riak.conf to the newly installed vm.args/riak.conf file (these files may be identical). +* The advanced.config file from the newly installed version will be significantly different from your backed-up file. It will have many new sections along with the original ones. Copy the customizations from your original advanced.config file into the appropriate sections in the new one. Ensure that the following sections are present in advanced.conf: + * `riak_core` --- the `cluster_mgr` setting must be present. See [MDC v3 Configuration][config v3 mdc] for more information. + * `riak_repl` --- See [MDC v3 Configuration][config v3 mdc] for more information. + * `snmp` --- See [SNMP][snmp] for more information. + * There is a sample configuration included at the end of the [Release Notes][release notes] for reference purposes. + +5.d\. (**EE Only with MDC**)If you need to replicate to EE clusters with versions less than 2.2.0, the capability override for bucket properties should be in the `riak_repl` proplist of the advanced.config file: + + ```advanced.config + {riak_repl, [ + {override_capability, [ + {default_bucket_props_hash, [{use, [consistent, datatype, n_val, allow_mult, last_write_wins]}] } + ]} + ]} + ``` +Once all of the clusters have been upgraded to version 2.2.0 or greater, this override should be removed. + +5.e\. (**EE Only**)JMX is no longer present in Riak KV. You must remove or comment out all references to it in your riak.conf/advanced.config files for Riak to start successfully post-upgrade. + +6\. Restart Riak KV: + +{{% note %}} +You must have [Java version 7 or higher](http://www.oracle.com/technetwork/java/javase/downloads/index.html) in order to upgrade to Riak KV 2.2.6 if you wish to use Riak search. If you do not have it installed, please install it now. +{{% /note %}} + + + +```bash +riak start +``` + +7\. Verify that Riak KV is running the new version: + +```bash +riak version +``` + +8\. Wait for the `riak_kv` service to start: + +```bash +riak-admin wait-for-service riak_kv »target_node« +``` + +* `»target_node«` is the node which you have just upgraded (e.g. +riak@192.168.1.11) + +9\. Wait for any hinted handoff transfers to complete: + +```bash +riak-admin transfers +``` + +* While the node was offline, other nodes may have accepted writes on its behalf. This data is transferred to the node when it becomes available. + +10\. Repeat the process for the remaining nodes in the cluster. + + +### Basho Patches + +After upgrading, you should ensure that any custom patches contained in the `basho-patches` directory are examined to determine their application to the upgraded version. You can find this information in the [Release Notes]. + +If you find that patches no longer apply to the upgraded version, you should remove them from the `basho-patches` directory prior to operating the node in production. + +The following lists locations of the `basho-patches` directory for +each supported operating system: + +- CentOS & RHEL Linux: `/usr/lib64/riak/lib/basho-patches` +- Debian & Ubuntu Linux: `/usr/lib/riak/lib/basho-patches` +- FreeBSD: `/usr/local/lib/riak/lib/basho-patches` +- SmartOS: `/opt/local/lib/riak/lib/basho-patches` +- Solaris 10: `/opt/riak/lib/basho-patches` + +### Riaknostic + +It is a good idea to also verify some basic configuration and general health of the Riak KV node after upgrading by using Riak KV's built-in diagnostic utility Riaknostic. + +Ensure that Riak KV is running on the node, and issue the following command: + +```bash +riak-admin diag +``` + +Make the recommended changes from the command output to ensure optimal node operation. diff --git a/content/riak/kv/2.2.6/using.md b/content/riak/kv/2.2.6/using.md new file mode 100644 index 0000000000..cf713fdc9f --- /dev/null +++ b/content/riak/kv/2.2.6/using.md @@ -0,0 +1,72 @@ +--- +title: "Using Riak KV" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Using" + identifier: "managing" + weight: 201 + pre: database +toc: true +--- + +[use running cluster]: ../using/running-a-cluster +[use admin index]: ../using/admin/ +[cluster ops index]: ../using/cluster-operations +[repair recover index]: ../using/repair-recovery +[security index]: ../using/security +[perf index]: ../using/performance +[troubleshoot index]: ../using/troubleshooting +[use ref]: ../using/reference + +## In This Section + +#### [Running a Cluster][use running cluster] + +A guide on basic cluster setup. + +[Learn More >>][use running cluster] + +#### [Cluster Administration][use admin index] + +Tutorials and reference documentation on cluster administration commands as well as command-line tools. + +[Learn More >>][use admin index] + +#### [Cluster Operations][cluster ops index] + +Step-by-step tutorials on a range of cluster operations, such as adding & removing nodes, renaming nodes, and back-ups. + +[Learn More >>][cluster ops index] + +#### [Repair & Recovery][repair recover index] + +Contains documentation on repairing a cluster, recovering from failure, and common errors. + +[Learn More >>][repair recover index] + +#### [Security][security index] + +Information on securing your Riak KV cluster. + +[Learn More >>][security index] + +#### [Performance][perf index] + +Articles on benchmarking your Riak KV cluster and improving performance. + +[Learn More >>][perf index] + +#### [Troubleshooting][troubleshoot index] + +Guides on troubleshooting issues and current product advisories. + +[Learn More >>][troubleshoot index] + +#### [Reference][use ref] + +Articles providing background information and implementation details on topics such as logging, bucket types, and search. + +[Learn More >>][use ref] diff --git a/content/riak/kv/2.2.6/using/admin.md b/content/riak/kv/2.2.6/using/admin.md new file mode 100644 index 0000000000..99c492984e --- /dev/null +++ b/content/riak/kv/2.2.6/using/admin.md @@ -0,0 +1,47 @@ +--- +title: "Cluster Administration" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Cluster Administration" + identifier: "managing_cluster_admin" + weight: 202 + parent: "managing" +toc: true +aliases: + - /riak-docs/riak/2.2.6/ops/running/cluster-admin + - /riak-docs/riak/kv/2.2.6/ops/running/cluster-admin +--- + +[use admin commands]: ./commands/ +[use admin riak cli]: ./riak-cli/ +[use admin riak-admin]: ./riak-admin/ +[use admin riak control]: ./riak-control/ + +## In This Section + +#### [Cluster Admin Commands][use admin commands] + +Explains usage of the `riak-admin cluster` interface, which enables you to perform a wide variety of cluster-level actions. + +[Learn More >>][use admin commands] + +#### [riak-admin Command Line Interface][use admin riak cli] + +Details the `riak-admin` interface. + +[Learn More >>][use admin riak-admin] + +#### [riak Command Line Interface][use admin riak-admin] + +Covers the `riak` interface, which enables control of the processes associated with a Riak node. + +[Learn More >>][use admin riak cli] + +#### [Riak Control][use admin riak control] + +Overview of Riak Control, a web-based administrative console for Riak clusters. + +[Learn More >>][use admin riak control] diff --git a/content/riak/kv/2.2.6/using/admin/commands.md b/content/riak/kv/2.2.6/using/admin/commands.md new file mode 100644 index 0000000000..6aa20f0c45 --- /dev/null +++ b/content/riak/kv/2.2.6/using/admin/commands.md @@ -0,0 +1,374 @@ +--- +title: "Cluster Administration Commands" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Cluster Admin Commands" + identifier: "cluster_admin_commands" + weight: 100 + parent: "managing_cluster_admin" +toc: true +aliases: + - /riak-docs/riak/2.2.6/ops/running/cluster-admin + - /riak-docs/riak/kv/2.2.6/ops/running/cluster-admin +--- + +[use admin riak-admin#cluster]: {{<baseurl>}}riak/kv/2.2.6/using/admin/riak-admin/#cluster +[concept clusters]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/clusters +[cluster ops add remove node]: {{<baseurl>}}riak/kv/2.2.6/using/cluster-operations/adding-removing-nodes +[use admin riak-admin#cluster-plan]: {{<baseurl>}}riak/kv/2.2.6/using/admin/riak-admin/#cluster-plan +[use admin riak-admin#cluster-commit]: {{<baseurl>}}riak/kv/2.2.6/using/admin/riak-admin/#cluster-commit + + +This document explains usage of the [`riak-admin cluster`][use admin riak-admin#cluster] interface, which enables you to perform a wide +variety of cluster-level actions. + +## How Cluster Administration Works + +Riak provides a multi-phased approach to cluster administration that +enables you to stage and review cluster-level changes prior to +committing them. This allows you to group multiple changes together, +such as adding multiple nodes at once, adding some nodes and removing +others, etc. + +Enacting cluster-level changes typically follows this set of steps: + +1. Choose an action or set of actions, such as adding a node, removing +multiple nodes, etc. These actions will be **staged** rather than +executed immediately. +1. **Plan** the changes using the [`cluster plan`](#plan) command. This will return a list of staged +commands that you can review. +1. **Commit** the changes using the [`cluster commit`](#commit) command. This will execute the changes that +have been staged and reviewed. + +> **Note on command names** +> +> Many of the commands available through the `riak-admin cluster` +interface are also available as self-standing commands. The `riak-admin +member-status` command is now the `riak-admin cluster status` command, +`riak-admin join` is now `riak-admin cluster join`, etc. +> +> We recommend using the `riak-admin cluster` interface over the older, +deprecated commands. You will receive a deprecation warning if you use +the older commands. + +## status + +Displays a variety of information about the cluster. + +```bash +riak-admin cluster status +``` + +This will return output like the following in a 3-node cluster: + +``` +---- Cluster Status ---- +Ring ready: true + ++--------------------+------+-------+-----+-------+ +| node |status| avail |ring |pending| ++--------------------+------+-------+-----+-------+ +| (C) dev1@127.0.0.1 |valid | up | 34.4| -- | +| dev2@127.0.0.1 |valid | up | 32.8| -- | +| dev3@127.0.0.1 |valid | up | 32.8| -- | ++--------------------+------+-------+-----+-------+ +``` + +In the above output, `Ring ready` denotes whether or not the cluster +agrees on [the ring][concept clusters], i.e. whether the cluster is +ready to begin taking requests. + +The following information is then displayed for each node, by nodename +(in this case `dev1@127.0.0.1`, etc.): + +* `status` --- There are five possible values for status: + * `valid` --- The node has begun participating in cluster operations + * `leaving` --- The node is is currently unloading ownership of its + [data partitions][concept clusters] to other nodes + * `exiting` --- The node's ownership transfers are complete and it is + currently shutting down + * `joining` --- The node is in the process of joining the cluster but + but has not yet completed the join process + * `down` --- The node is not currently responding +* `avail` --- There are two possible values: `up` if the node is + available and taking requests and `down!` if the node is unavailable +* `ring` --- What percentage of the Riak [ring][concept clusters] the + node is responsible for +* `pending` --- The number of pending transfers to or from the node + +In addition, the cluster's [claimant node][cluster ops add remove node] node will have a `(C)` next +to it. + +## join + +Joins the current node to another node in the cluster. + +```bash +riak-admin cluster join <node> +``` + +You _must_ specify a node to join to by nodename. You can join to any +node in the cluster. The following would join the current node to +`riak1@127.0.0.1`: + +```bash +riak-admin cluster join riak1@127.0.0.1 +``` + +Once a node joins, all of the operations necessary to establish +communication with all other nodes proceeds automatically. + +> **Note**: As with all cluster-level actions, the changes made when you +run the `cluster join` command will take effect only after you have both +planned the changes by running [`riak-admin cluster plan`][use admin riak-admin#cluster-plan] and committed the changes by running +[`riak-admin cluster commit`][use admin riak-admin#cluster-commit]. +You can stage multiple joins before planning/committing. + +## leave + +Instructs the current node to hand off its +[data partitions][concept clusters], leave the cluster, and shut down. + +```bash +riak-admin cluster leave +``` + +You can also instruct another node (by nodename) to leave the cluster: + +```bash +riak-admin cluster leave <node> +``` + +> **Note**: As with all cluster-level actions, the changes made when you +run the `cluster leave` command will take effect only after you have +both planned the changes by running [`riak-admin cluster plan`][use admin riak-admin#cluster-plan] and committed the changes +by running [`riak-admin cluster commit`][use admin riak-admin#cluster-commit]. +You can stage multiple leave command before planning/committing. + +## force-remove + +Removes another node from the cluster (by nodename) _without_ first +handing off its [data partitions][concept clusters]. This command is +designed for crashed, unrecoverable nodes and should be used with +caution. + +```bash +riak-admin cluster force-remove <node> +``` + +> **Note**: As with all cluster-level actions, the changes made when you +run the `cluster force-remove` command will take effect only after you have +both planned the changes by running [`riak-admin cluster plan`][use admin riak-admin#cluster-plan] and committed the changes +by running [`riak-admin cluster commit`][use admin riak-admin#cluster-commit]. You can stage multiple force-remove actions +before planning/committing. + +## replace + +Instructs a node to transfer all of its [data partitions][concept clusters] to another node and then to leave the +cluster and shut down. + +```bash +riak-admin cluster replace <node1> <node2> +``` + +> **Note**: As with all cluster-level actions, the changes made when you +run the `cluster replace` command will take effect only after you have +both planned the changes by running [`riak-admin cluster plan`][use admin riak-admin#cluster-plan] and committed the changes +by running [`riak-admin cluster commit`][use admin riak-admin#cluster-commit]. You can stage multiple replace actions before +planning/committing. + +## force-replace + +Reassigns all [data partitions][concept clusters] owned by one node to +another node _without_ first handing off data. + +```bash +riak-admin force-replace <node_being_replaced> <replacement_node> +``` + +Once the data partitions have been reassigned, the node that is being +replaced will be removed from the cluster. + +> **Note**: As with all cluster-level actions, the changes made when you +run the `cluster force-replace` command will take effect only after you have +both planned the changes by running [`riak-admin cluster plan`][use admin riak-admin#cluster-plan] and committed the changes +by running [`riak-admin cluster commit`][use admin riak-admin#cluster-commit]. You can stage multiple force-replace actions +before planning/committing. + +## plan + +Displays the currently staged cluster changes. + +```bash +riak-admin cluster plan +``` + +`riak-admin cluster plan` is complex, depending on the staged changes. + +* If a `leave` operation has been staged, `riak-admin cluster plan` will undo the staged change and no node will be stopped. +* If a `join` operation has been staged, the joining node will be shut down after its ring has been cleared. When this node restarts, it will behave like a fresh unjoined node and can be joined again. +* If a `cluster clear` operation is staged on a node that remains in the cluster, running `riak-admin cluster plan` will leave the node unaffected. + +If there is no current cluster plan, the output will be `There are no +staged changes`. + +If there is a staged change (or changes), however, you +will see a detailed listing of what will take place upon commit, what +the cluster will look like afterward, etc. + +For example, if a `cluster leave` operation is staged in a 3-node cluster the output will look something like this: + +``` +=============================== Staged Changes ================================ +Action Details(s) +------------------------------------------------------------------------------- +leave 'dev2@127.0.0.1' +------------------------------------------------------------------------------- + + +NOTE: Applying these changes will result in 2 cluster transitions + +############################################################################### + After cluster transition 1/2 +############################################################################### + +================================= Membership ================================== +Status Ring Pending Node +------------------------------------------------------------------------------- +leaving 32.8% 0.0% 'dev2@127.0.0.1' +valid 34.4% 50.0% 'dev1@127.0.0.1' +valid 32.8% 50.0% 'dev3@127.0.0.1' +------------------------------------------------------------------------------- +Valid:2 / Leaving:1 / Exiting:0 / Joining:0 / Down:0 + +WARNING: Not all replicas will be on distinct nodes + +Transfers resulting from cluster changes: 38 + 6 transfers from 'dev1@127.0.0.1' to 'dev3@127.0.0.1' + 11 transfers from 'dev3@127.0.0.1' to 'dev1@127.0.0.1' + 5 transfers from 'dev2@127.0.0.1' to 'dev1@127.0.0.1' + 16 transfers from 'dev2@127.0.0.1' to 'dev3@127.0.0.1' + +############################################################################### + After cluster transition 2/2 +############################################################################### + +================================= Membership ================================== +Status Ring Pending Node +------------------------------------------------------------------------------- +valid 50.0% -- 'dev1@127.0.0.1' +valid 50.0% -- 'dev3@127.0.0.1' +------------------------------------------------------------------------------- +Valid:2 / Leaving:0 / Exiting:0 / Joining:0 / Down:0 + +WARNING: Not all replicas will be on distinct nodes +``` + +Notice that there are distinct sections of the output for each of the +transitions that the cluster will undergo, including warnings, planned +data transfers, etc. + +## commit + +Commits the currently staged cluster changes. Staged cluster changes +must be reviewed using [`riak-admin cluster plan`][use admin riak-admin#cluster-plan] prior to being committed. + +```bash +riak-admin cluster commit +``` + +## clear + +Clears the currently staged cluster changes. + +```bash +riak-admin cluster clear +``` + +## partitions + +Prints primary, secondary, and stopped partition indices and IDs either +for the current node or for another, specified node. The following +prints that information for the current node: + +```bash +riak-admin cluster partitions +``` + +This would print the partition information for a different node in the +cluster: + +```bash +riak-admin cluster partitions --node=<node> +``` + +Partition information is contained in a table like this: + +``` +Partitions owned by 'dev1@127.0.0.1': ++---------+-------------------------------------------------+--+ +| type | index |id| ++---------+-------------------------------------------------+--+ +| primary | 0 |0 | +| primary | 91343852333181432387730302044767688728495783936 |4 | +| primary |182687704666362864775460604089535377456991567872 |8 | +| ... | .... |..| +| primary |1438665674247607560106752257205091097473808596992|63| +|secondary| -- |--| +| stopped | -- |--| ++---------+-------------------------------------------------+--+ +``` + +## partition-count + +Displays the current partition count either for the whole cluster or for +a particular node. This would display the partition count for the +cluster: + +```bash +riak-admin cluster partition-count +``` + +This would display the count for a node: + +```bash +riak-admin cluster partition-count --node=<node> +``` + +When retrieving the partition count for a node, you'll see a table like +this: + +``` ++--------------+----------+-----+ +| node |partitions| pct | ++--------------+----------+-----+ +|dev1@127.0.0.1| 22 | 34.4| ++--------------+----------+-----+ +``` + +The `partitions` column displays the number of partitions claimed by the +node, while the `pct` column displays the percentage of the ring claimed. + +## partition + +The `cluster partition` command enables you to convert partition IDs to +indexes and vice versa using the `partition id` and `partition index` +commands, respectively. Let's say that you run the `riak-admin cluster +partitions` command and see that you have a variety of partitions, one +of which has an index of +`1004782375664995756265033322492444576013453623296`. You can convert +that index to an ID like this: + +```bash +riak-admin cluster partition index=1004782375664995756265033322492444576013453623296 +``` + +Conversely, if you have a partition with an ID of 20, you can retrieve +the corresponding index: + +```bash +riak-admin cluster partition id=20 +``` diff --git a/content/riak/kv/2.2.6/using/admin/riak-admin.md b/content/riak/kv/2.2.6/using/admin/riak-admin.md new file mode 100644 index 0000000000..dcd44177f3 --- /dev/null +++ b/content/riak/kv/2.2.6/using/admin/riak-admin.md @@ -0,0 +1,717 @@ +--- +title: "riak-admin Command Line Interface" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "riak-admin CLI" + identifier: "cluster_admin_cli" + weight: 101 + parent: "managing_cluster_admin" +toc: true +aliases: + - /riak-docs/riak/2.2.6/ops/running/tools/riak-admin + - /riak-docs/riak/kv/2.2.6/ops/running/tools/riak-admin +--- + +[config reference]: {{<baseurl>}}riak/kv/2.2.6/configuring/reference +[use admin commands]: {{<baseurl>}}riak/kv/2.2.6/using/admin/commands +[use admin commands#join]: {{<baseurl>}}riak/kv/2.2.6/using/admin/commands/#join +[use admin commands#leave]: {{<baseurl>}}riak/kv/2.2.6/using/admin/commands/#leave +[cluster ops backup]: {{<baseurl>}}riak/kv/2.2.6/using/cluster-operations/backing-up +[config reference#node-metadata]: {{<baseurl>}}riak/kv/2.2.6/configuring/reference/#node-metadata +[cluster ops change info]: {{<baseurl>}}riak/kv/2.2.6/using/cluster-operations/changing-cluster-info +[usage mapreduce]: {{<baseurl>}}riak/kv/2.2.6/developing/usage/mapreduce +[usage commit hooks]: {{<baseurl>}}riak/kv/2.2.6/developing/usage/commit-hooks +[config reference#ring]: {{<baseurl>}}riak/kv/2.2.6/configuring/reference/#ring +[cluster ops inspect node]: {{<baseurl>}}riak/kv/2.2.6/using/cluster-operations/inspecting-node +[use ref monitoring]: {{<baseurl>}}riak/kv/2.2.6/using/reference/statistics-monitoring +[downgrade]: {{<baseurl>}}riak/kv/2.2.6/setup/downgrade +[security index]: {{<baseurl>}}riak/kv/2.2.6/using/security/ +[security managing]: {{<baseurl>}}riak/kv/2.2.6/using/security/managing-sources +[cluster ops bucket types]: {{<baseurl>}}riak/kv/2.2.6/using/cluster-operations/bucket-types +[cluster ops 2i]: {{<baseurl>}}riak/kv/2.2.6/using/reference/secondary-indexes +[repair recover index]: {{<baseurl>}}riak/kv/2.2.6/using/repair-recovery +[cluster ops strong consistency]: {{<baseurl>}}riak/kv/2.2.6/using/cluster-operations/strong-consistency +[cluster ops handoff]: {{<baseurl>}}riak/kv/2.2.6/using/cluster-operations/handoff +[use admin riak-admin#stats]: {{<baseurl>}}riak/kv/2.2.6/using/admin/riak-admin/#stats + +## `riak-admin` + +The riak-admin command performs operations unrelated to node liveness, including: +node membership, backup, and basic status reporting. The node must be +running for most of these commands to work. Running `riak-admin` by itself will output a list of available commands: + +``` +Usage: riak-admin { cluster | join | leave | backup | restore | test | + reip | js-reload | erl-reload | wait-for-service | + ringready | transfers | force-remove | down | + cluster-info | member-status | ring-status | vnode-status | + aae-status | diag | stat | status | transfer-limit | reformat-indexes | + top [-interval N] [-sort reductions|memory|msg_q] [-lines N] | + downgrade-objects | security | bucket-type | repair-2i | + search | services | ensemble-status | handoff | set | + show | describe } +``` + +## Node Naming + +An important thing to bear in mind is that all Riak nodes have unique +names within the cluster that are used for a wide variety of operations. +The name for each node can be set and changed in each node's +[configuration files][config reference]. The examples below set the name of a node to +`riak_node_1@199.99.99.01` in the `riak.conf` file if you are using the +newer configuration system and in `vm.args` if you are using the older +system: + +```riakconf +nodename = riak_node_1@199.99.99.01 +``` + +```vmargs +-name riak_node_1@199.99.99.01 +``` + +The name prior to the `@` symbol can be whatever you'd like, e.g. +`riak1`, `dev`, `cluster1_node1`, or `spaghetti`. After the `@` you must +use a resolvable IP address or hostname. In general, we recommend using +hostnames over IP addresses when possible because this enables the node +to potentially live on different machines over the course of its +existence. + +## cluster + +Documentation for the `riak-admin cluster` command interface can be +found in [Cluster Administration][use admin commands]. + +## join + +> **Deprecation Notice** +> +>As of Riak version 1.2, the `riak-admin join` command has +been deprecated in favor of the [`riak-admin cluster join`][use admin commands#join] command. However, this command can still be +used by providing a `-f` option (which forces the command). + +Joins the running node to another running node so that they participate +in the same cluster. `<node>` is the other node to connect to. + +```bash +riak-admin join -f <node> +``` + +## leave + +> **Deprecation Notice** +> +> As of Riak version 1.2, the `riak-admin leave` command has +been deprecated in favor of the new [`riak-admin cluster leave`][use admin commands#leave] command. However, this +command can still be used by providing a `-f` option (which +forces the command). + +Causes the node to leave the cluster in which it participates. After +this is run, the node in question will hand-off all its replicas to +other nodes in the cluster before it completely exits. + +```bash +riak-admin leave -f +``` + +## backup + +> **Deprecation notice** +The `riak-admin backup` command has been deprecated. We recommend using +backend-specific backup procedures instead. Documentation can be found +in [Backing up Riak KV][cluster ops backup]. + +Backs up the data from the node or entire cluster into a file. + +```bash +riak-admin backup <node> <cookie> <filename> [node|all] +``` + +* `<node>` is the node from which to perform the backup. +* `<cookie>` is the Erlang cookie/shared secret used to connect to the + node. This is `riak` in the [default configuration][config reference#node-metadata]. +* `<filename>` is the file where the backup will be stored. _This should + be the full path to the file_. +* `[node|all]` specifies whether the data on this node or the entire + +## restore + +> **Deprecation notice** +> +> The `riak-admin restore` command has been deprecated. It was originally +intended to be used in conjunction with backups performed using the +`riak-admin backup` command, which is also deprecated. We recommend +using the backup and restore methods described in [Backing up Riak KV][cluster ops backup]. + +Restores data to the node or cluster from a previous backup. + +```bash +riak-admin restore <node> <cookie> <filename> +``` + +* `<node>` is the node which will perform the restore. +* `<cookie>` is the Erlang cookie/shared secret used to connect to the + node. This is `riak` in the [default configuration][config reference#node-metadata]. +* `<filename>` is the file where the backup is stored. _This should be + the full path to the file_. + +## test + +Runs a test of a few standard Riak operations against the running node. + +```bash +riak-admin test +``` + +If the test is successful, you should see output like the following: + +``` +Successfully completed 1 read/write cycle to 'dev1@127.0.0.1' +``` + +## reip + +Renames a node. This process backs up and edits the Riak ring, and +**must** be run while the node is stopped. Reip should only be run in +cases where `riak-admin cluster force-replace` cannot be used to +rename the nodes of a cluster. For more information, visit the +[Changing Cluster Information][cluster ops change info] document. + +```bash +riak-admin reip <old nodename> <new nodename> +``` + +{{% note title="Note about reip prior to Riak 2.0" %}} +Several bugs have been fixed related to reip in Riak 2.0. We recommend against +using reip prior to 2.0, if possible. +{{% /note %}} + + +## js-reload + +Forces the embedded Javascript virtual machines to be restarted. This is +useful when deploying custom built-in [MapReduce][usage mapreduce] +functions. + +**Note**: This needs to be run on _all nodes_ in the cluster. + +```bash +riak-admin js-reload +``` + +## erl-reload + +Reloads the Erlang `.beam` files used for [MapReduce][usage mapreduce] +jobs, [pre- and post-commit hooks][usage commit hooks], and other +purposes. + +> **Note**: This needs to be run on _all nodes_ in the cluster. + +```bash +riak-admin erl-reload +``` + +## wait-for-service + +Waits on a specific watchable service to be available (typically +`riak_kv`). This is useful when (re-)starting a node while the cluster +is under load. Use `riak-admin services` to see which services are +available on a running node. + +```bash +riak-admin wait-for-service <service> <nodename> +``` + +## ringready + +Checks whether all nodes in the cluster agree on the ring state. +Prints `FALSE` if the nodes do not agree. This is useful after changing +cluster membership to make sure that the ring state has settled. + +```bash +riak-admin ringready +``` + +## transfers + +Identifies nodes that are awaiting transfer of one or more partitions. +This usually occurs when partition ownership has changed (after adding +or removing a node) or after node recovery. + +```bash +riak-admin transfers +``` + +## transfer-limit + +Change the `handoff_concurrency` limit. The value set by running this +command will only persist while the node is running. If the node is +restarted, the `transfer-limit` will return to the default of `2` or the +value specified in the [`transfer_limit`][config reference#ring] setting in the `riak.conf` configuration file. + +Running this command with no arguments will display the current +transfer-limit for each node in the cluster. + +```bash +riak-admin transfer-limit <node> <limit> +``` + +## down + +Marks a node as down so that ring transitions can be performed before +the node is brought back online. + +```bash +riak-admin down <node> +``` + +## cluster-info + +Output system information from a Riak cluster. This command will collect +information from all nodes or a subset of nodes and output the data to a +single text file. + +```bash +riak-admin cluster-info <output file> [<node list>] +``` + +The following information is collected: + + * Current time and date + * VM statistics + * `erlang:memory()` summary + * Top 50 process memory hogs + * Registered process names + * Registered process name via `regs()` + * Non-zero mailbox sizes + * Ports + * Applications + * Timer status + * ETS summary + * Nodes summary + * `net_kernel` summary + * `inet_db` summary + * Alarm summary + * Global summary + * `erlang:system_info()` summary + * Loaded modules + * Riak Core config files + * Riak Core vnode modules + * Riak Core ring + * Riak Core latest ring file + * Riak Core active partitions + * Riak KV status + * Riak KV ringready + * Riak KV transfers + +#### Examples + +Output information from all nodes to `/tmp/cluster_info.txt`: + +```bash +riak-admin cluster_info /tmp/cluster_info.txt +``` + +Output information from the current nodeL + +```bash +riak-admin cluster_info /tmp/cluster_info.txt local +``` + +Output information from a subset of nodes: + +```bash +riak-admin cluster_info /tmp/cluster_info.txt riak@192.168.1.10 +riak@192.168.1.11 +``` + +## member-status + +Prints the current status of all cluster members. + +```bash +riak-admin member-status +``` + +## ring-status + +Outputs the current claimant, its status, ringready, pending ownership +handoffs, and a list of unreachable nodes. + +```bash +riak-admin ring-status +``` + +## vnode-status + +Outputs the status of all vnodes the are running on the local node. + +```bash +riak-admin vnode-status +``` + +## aae-status + +This command provides insight into operation of Riak's Active +Anti-Entropy (AAE) feature. + +```bash +riak-admin aae-status +``` + +The output contains information on AAE key/value partition exchanges, +entropy tree building, and key repairs which were triggered by AAE. + +* **Exchanges** + * The *Last* column lists when the most recent exchange between a + partition and one of its sibling replicas was performed. + * The *All* column shows how long it has been since a partition + exchanged with all of its sibling replicas. + +* **Entropy Trees** + * The *Built* column shows when the hash trees for a given partition + were created. + +* **Keys Repaired** + * The *Last* column shows the number of keys repaired during the most + recent key exchange. + * The *Mean* column shows the mean number of keys repaired during all + key exchanges since the last node restart. + * The *Max* column shows the maximum number of keys repaired during all + key exchanges since the last node restart. + +{{% note title="Note in AAE status information" %}} +All AAE status information is in-memory and is reset across a node restart. +Only tree build times are persistent (since trees themselves are persistent) +{{% /note %}} + +More details on the `aae-status` command are available in the [Riak +version 1.3 release notes](https://github.com/basho/riak/blob/1.3/RELEASE-NOTES.md#active-anti-entropy). + +## diag + +The `diag` command invokes the [Riaknostic](http://riaknostic.basho.com/) +diagnostic system. + +```bash +riak-admin diag +``` + +This command allows you to specify which diagnostic checks you would +like to run, which types of diagnostic messages you wish to see, and so +on. More comprehensive information can be found in the documentation on +[inspecting a node][cluster ops inspect node]. + +## stat + +Provides an interface for interacting with a variety of cluster-level +metrics and information. + +```bash +riak-admin stat +``` + +Full documentation of this command can be found in [Statistics and Monitoring][use ref monitoring]. + +## status + +Prints status information, including performance statistics, system +health information, and version numbers. Further information about the +output is available in the documentation on [inspecting a node][cluster ops inspect node]. + +```bash +riak-admin status +``` + +## reformat-indexes + +This command reformats integer indexes in Secondary Index data for +versions of Riak prior to 1.3.1 so that range queries over the indexes +will return correct results. + +``` +riak-admin reformat-indexes [<concurrency>] [<batch size>] --downgrade +``` + +The `concurrency` option defaults to `2` and controls how many +partitions are concurrently reformatted. + +The `batch size` option controls the number of simultaneous key +operations and defaults to `100`. + +This command can be executed while the node is serving requests, and +default values are recommended for most cases. You should only change +the default values after testing impact on cluster performance. + +Information is written to `console.log` upon completion of the process. + +A `--downgrade` switch can be specified when downgrading a node to a version +of Riak prior to version 1.3.1. + +Additional details are available in the [Riak 1.3.1 release +notes](https://github.com/basho/riak/blob/1.3/RELEASE-NOTES.md). + +## top + +Top uses Erlang's etop to provide information about what the Erlang +processes inside of Riak are doing. Top reports process reductions (an +indicator of CPU utilization), memory used, and message queue sizes. + +```bash +riak-admin top [-interval N] [-sort reductions|memory|msg_q] [-lines N] +``` + +Options: + +* `interval` specifies the number of seconds between each update of the + top output and defaults to `5` +* `sort` determines on which category `riak-admin top` sorts and + defaults to `reductions` +* `lines` specifies the number of processes to display in the top output + and defaults to `10` + +More information about Erlang's etop can be found in the [etop +documentation](http://www.erlang.org/doc/man/etop.html). + +## downgrade-objects + +This command is used when changing the format of Riak objects, usually +as part of a version downgrade. + +```bash +riak-admin downgrade-objects <kill-handoffs> [<concurrency>] +``` + +More detailed information can be found in [Rolling Downgrades][downgrade]. + +## security + +This command enables you to manage to manage Riak users, choose sources +of authentication, assign and revoke permissions to/from users and +groups, enable and disable Riak Security, and more. + +```bash +riak-admin security <command> +``` + +More comprehensive information on user management and can be found in +the [Authentication and Authorization][security index] guide. Detailed information on authentication sources can be found in [Managing Security Sources][security managing]. + +## bucket-type + +Bucket types are a means of managing bucket properties introduced in +Riak 2.0, as well as an additional namespace in Riak in addition to +buckets and keys. This command enables you to create and modify bucket +types, provide the status of currently available bucket types, and +activate created bucket types. + +```bash +riak-admin bucket-type <command> +``` + +More on bucket types can be found in [Using Bucket Types][cluster ops bucket types]. + +## repair-2i + +This command repairs [secondary indexes][cluster ops 2i] in a +specific partition or on a cluster-wide basis. Implementation details +can be found in [Repairing Indexes][repair recover index]. + +To repair secondary indexes throughout the entire cluster, run the +`repair-2i`command by itself, without a subcommand: + +```bash +riak-admin repair-2i +``` + +This will initiate the repair process. When you run this command, you +should see something like the following (where `<ring_size>` is the +number of partitions in your Riak cluster): + +``` +Will repair 2i data on <ring_size> partitions +Watch the logs for 2i repair progress reports +``` + +To repair secondary indexes in a specific partition, provide the ID of +the partition along with the `repair-2i` command: + +```bash +riak-admin repair-2i 593735040165679310520246963290989976735222595584 +``` + +You can check on the status of the repair process at any time: + +```bash +riak-admin repair-2i status +``` + +If the repair is already finished, the console will return `2i repair is +not running`. If the repair is still in progress, the console will +return a series of statistics like this: + +``` +2i repair status is running: + Total partitions: 64 + Finished partitions: 44 + Speed: 100 + Total 2i items scanned: 0 + Total tree objects: 0 + Total objects fixed: 0 +``` + +If you're concerned about the computational resources required to repair +secondary indexes, you can set the speed of the process to an integer +between 1 and 100 (with 100 being the fastest). This command would set +the speed to 90: + +```bash +riak-admin repair-2i --speed 90 +``` + +The repair process can be stopped at any moment using the `kill` +command: + +```bash +riak-admin repair-2i kill +``` + +## search + +The search command provides sub-commands for various administrative +work related to the new Riak Search. + +```bash +riak-admin search <command> +``` + +### aae-status + +```bash +riak-admin search aae-status +``` + +Output active anti-entropy (AAE) statistics for search. There are +three sections. Each section contains statistics for a specific aspect +of AAE for every partition owned by the local node. + +The first section provides information on exchanges. Exchange is the +process of comparing hash trees to determine divergences between KV +data and search indexes. The `Index` column contains the partition +number. The `Last (ago)` column is the amount of time that has passed +since the last exchange. The `All (ago)` column is the amount of time +that has passed since all preflists for that partition have been +exchanged. + +The second section lists how much time has passed since the hashtree +for that partition has been built from scratch. By default trees +expire after 1 week and are rebuilt from scratch. + +The third section presents statistics on repair operations that have +occurred. Repair is performed when AAE notices that the KV and search +hashtree don't match for a particular key. The `Last` column is the +number of keys repaired during the last exchange. The `Mean` column is +the average number of keys repaired for all exchange rounds since the +node has started. The `Max` column is the maximum number of keys +repaired for a given exchange round since the node has started. + +### switch-to-new-search + +{{% note title="Only For Legacy Migration" %}} +This is only needed when migrating from legacy riak search to the new Search +(Yokozuna). +{{% /note %}} + +```bash +riak-admin search switch-to-new-search +``` + +Switch handling of the HTTP `/solr/<index>/select` resource and +protocol buffer query messages from legacy Riak Search to new Search +(Yokozuna). + +## services + +Lists available services on the node (e.g. `riak_kv`). + +```bash +riak-admin services +``` + +## ensemble-status + +This command is used to provide insight into the current status of the +consensus subsystem undergirding Riak's [strong consistency][cluster ops strong consistency] feature. + +```bash +riak-admin ensemble-status +``` + +This command can also be used to check on the status of a specific +consensus group in your cluster: + +```bash +riak-admin ensemble-status <group id> +``` + +Complete documentation of this command can be found in [Managing Strong Consistency][cluster ops strong consistency]. + +## handoff + +Documentation for the `handoff` command can be found in [Handoff][cluster ops handoff]. + +## set + +Enables you to change the value of one of Riak's configuration +parameters on the fly, without needing to stop and restart the node. + +```bash +riak-admin set <variable>=<value> +``` + +The set command can only be used for the following +parameters: + +* `transfer_limit` +* `handoff.outbound` +* `handoff.inbound` +* `search.dist_query=off` will disable distributed query for the node +* `search.dist_query=on` will enable distributed query for the node +* `search.dist_query` will get the status of distributed query for the node + +The `search.dist_query` commands above are non-persistent. Any settings you have defined in your riak.conf configuration file will be used when Riak KV is restarted. + + +## show + +Whereas the [`riak-admin status`][use admin riak-admin#stats] command will display all currently available statistics for your Riak +cluster, the `show` command enables you to view only some of those +statistics. + +```bash +riak-admin show <variable> +``` + +## describe + +Provides a brief description of one of Riak's [configurable parameters][config reference]. + +```bash +riak-admin describe <variable> +``` + +If you want to know the meaning of the `nodename` parameter: + +```bash +riak-admin describe nodename +``` + +That will produce the following output: + +``` +nodename: + Name of the Erlang node +``` diff --git a/content/riak/kv/2.2.6/using/admin/riak-cli.md b/content/riak/kv/2.2.6/using/admin/riak-cli.md new file mode 100644 index 0000000000..6775a3db8c --- /dev/null +++ b/content/riak/kv/2.2.6/using/admin/riak-cli.md @@ -0,0 +1,200 @@ +--- +title: "riak Command Line Interface" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "riak CLI" + identifier: "cluster_admin_riak_cli" + weight: 102 + parent: "managing_cluster_admin" +toc: true +aliases: + - /riak-docs/riak/2.2.6/ops/running/tools/riak + - /riak-docs/riak/kv/2.2.6/ops/running/tools/riak +--- + +[configuration file]: {{<baseurl>}}riak/kv/2.2.6/configuring/reference/ +[escript]: http://www.erlang.org/doc/man/escript.html +[`riak-admin`]: {{<baseurl>}}riak/kv/2.2.6/using/admin/riak-admin/#top +[configuration]: {{<baseurl>}}riak/kv/2.2.6/configuring/reference/ + +## riak + +This is the primary script for controlling the processes associated with a Riak node. Running the `riak` command by itself will output a listing of available commands: + +```bash +Usage: riak «command» +where «command» is one of the following: + { help | start | stop | restart | ping | console | attach + attach-direct | ertspath | chkconfig | escript | version | getpid + top [-interval N] [-sort { reductions | memory | msg_q }] [-lines N] } | + config { generate | effective | describe VARIABLE } [-l debug] +``` + +## help + +Provides a brief description of all available commands. + +## start + +Starts the Riak node in the background. If the node is already started, you will receive the message `Node is already running!` If the node is not already running, no output will be given. + +```bash +riak start +``` + +## stop + +Stops the running Riak node. Prints `ok` when successful or `Node <nodename> not responding to pings.` when the node is already stopped or not responding. + +```bash +riak stop +``` + +## restart + +Stops and then starts the running Riak node without exiting the Erlang VM. +Prints `ok` when successful, `Node <nodename> not responding to pings.` when the node is already stopped or not responding. + +```bash +riak restart +``` + +## ping + +Checks that the Riak node is running. Prints `pong` when successful or `Node <nodename> not responding to pings.` when the node is stopped or not responding. + +```bash +riak ping +``` + +## console + +Starts the Riak node in the foreground, giving access to the Erlang shell and +runtime messages. Prints `Node is already running - use 'riak attach' instead` +when the node is running in the background. You can exit the shell by pressing **Ctrl-C** twice. + +```bash +riak console +``` + +## attach + +Attaches to the console of a Riak node running in the background, giving access to the Erlang shell and runtime messages. Prints `Node is not running!` when the node cannot be reached. + +```bash +riak attach +``` + +## attach-direct + +Attaches to the console of a Riak running in the background using a directly-connected first-in-first-out (FIFO), providing access to the Erlang shell and runtime messages. Prints `Node is not running!` when the node cannot be reached. You can exit the shell by pressing **Ctrl-D**. + +```bash +riak attach-direct +``` + +## ertspath + +Outputs the path of the Riak Erlang runtime environment: + +```bash +riak ertspath +``` + +## chkconfig + +Checks whether the [configuration file][configuration file] is valid. If so, `config is OK` will be included in the output. + +```bash +riak chkconfig +``` + +## escript + +Provides a means of calling [escript][escript] scripts using the Riak Erlang runtime environment: + +```bash +riak escript <filename> +``` + +## version + +Outputs the Riak version identifier: + +```bash +riak version +``` + +## getpid + +Outputs the process identifier for the currently-running instance of Riak: + +```bash +riak getpid +``` + +## top + +The `riak top` command is the direct equivalent of `riak-admin top`: + +```bash +riak top [-interval N] [-sort { reductions | memory | msg_q }] [-lines N] } +``` + +More detailed information can be found in the [`riak-admin`][`riak-admin`] documentation. + +## config + +Provides information about the current [configuration][configuration] of a Riak node, i.e. the parameters and values in the node's riak.conf configuration. + +```bash +riak config { generate | effective | describe VARIABLE } [-l debug] +``` + +* `generate` will cause the configuration files to be re-processed. This behavior happens automatically at node startup; however `riak config generate` can be used to test for configuration errors that would prevent the node from starting after modifying the riak.conf or advanced.config files. + The output of a successful run will show the paths to the newly generated configuration files. These configuration files will contain a timestamp to indicate when they were generated. For example: + + ``` + -config /var/lib/riak/generated.configs/app.2016.12.02.17.47.32.config -args_file /var/lib/riak/generated.configs/vm.2016.12.02.17.47.32.args -vm_args /var/lib/riak/generated.configs/vm.2016.12.02.17.47.32.args + ``` + + If you are using the legacy configuration file format (app.config/vm.args), you will receive the following message: + + ``` + -config /etc/riak/app.config -args_file /etc/riak/vm.args -vm_args /etc/riak/vm.args + ``` + +* `effective` prints the effective configuration in the following syntax: + + ``` + parameter1 = value1 + parameter2 = value2 + ``` + + If you are using the legacy configuration file format (app.config/vm.args), you will receive the following error: + + ``` + Disabling cuttlefish, legacy configuration files found: + /etc/riak/app.config + /etc/riak/vm.args + Effective config is only visible for cuttlefish conf files. + ``` + +* `describe VARIABLE` prints the setting specified by `VARIABLE`, along with documentation and other useful information, such as the affected location in the configuration file, the data type of the value, the default value, and the effective value. For example, running `riak config describe storage_backend` will return the following: + + ``` + Documentation for storage_backend + Specifies the storage engine used for Riak's key-value data + and secondary indexes (if supported). + + Valid Values: + - one of: bitcask, leveldb, memory, multi, prefix_multi + Default Value : bitcask + Set Value : bitcask + Internal key : riak_kv.storage_backend + ``` + +Adding the `-l debug` flag to any `riak config` command will produce additional debugging information that can be used in advanced troubleshooting of "cuttlefish", Riak's configuration subsystem. diff --git a/content/riak/kv/2.2.6/using/admin/riak-control.md b/content/riak/kv/2.2.6/using/admin/riak-control.md new file mode 100644 index 0000000000..685ad888f7 --- /dev/null +++ b/content/riak/kv/2.2.6/using/admin/riak-control.md @@ -0,0 +1,233 @@ +--- +title: "Riak Control" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Riak Control" + identifier: "cluster_admin_riak_control" + weight: 103 + parent: "managing_cluster_admin" +toc: true +aliases: + - /riak-docs/riak/2.2.6/ops/advanced/riak-control + - /riak-docs/riak/kv/2.2.6/ops/advanced/riak-control +--- + +[config reference]: {{<baseurl>}}riak/kv/2.2.6/configuring/reference + +Riak Control is a web-based administrative console for inspecting and +manipulating Riak clusters. + +## Requirements + +Though Riak Control [is maintained as a separate application](https://github.com/basho/riak_control), the necessary code for it ships with versions of Riak 1.1 and above and requires no additional installation steps. + +Before getting started, you should know the address and port of the HTTP (or +HTTPS) listeners for the cluster member(s) running Riak Control. You can obtain +this information from the configuration files as indicated here: + +```riakconf +listener.http.<name> = 127.0.0.1:8098 + +or + +listener.https.<name> = 127.0.0.1:8096 + +## *** The default listeners in the riak.conf file are +## named `internal`, so you would consult the value of +## `listener.http.internal` in your configuration. + +``` + +```appconfig + {riak_api, + [ + %% Other configs + ... if HTTP is configured ... + {http,[{"127.0.0.1",8098}]}, + ... if HTTPS is configured ... + {https,[{"127.0.0.1",8069}]}, + %% Other configs + ]}, + +%% *** This is a truncated configuration to illustrate the +%% pertinent items -- the `http` and `https` tuples within +%% the `riak_api` tuple's value list. +``` + +{{% note title="Note on SSL" %}} +We strongly recommend that you enable SSL for Riak Control. It is disabled by +default, and if you wish to enable it you must do so explicitly. More +information can be found in the document below. +{{% /note %}} + +## Enabling and Disabling Riak Control + +Riak Control is disabled by default, meaning that you should see the +following in your [configuration files][config reference]: + +```riakconf +riak_control = off +``` + +```appconfig +{riak_control, [ + %% Other configs + {enabled, false}, + %% Other configs + ]} +``` + +Enabling Riak Control is simple: + +```riakconf +riak_control = on +``` + +```appconfig +{riak_control, [ + %% Other configs + {enabled, true}, + %% Other configs + ]} +``` + +Make sure to restart the node once you have enabled Riak Control for the +change to take effect. + +After restarting the node, you should be able to access it by going +to `http://ip_address_of_listener:port/admin`. In the case of a development +cluster using the default configuration, you would access Riak Control at +<http://127.0.0.1:8098/admin></a> + +If you enabled authentication for Riak Control while performing the above +configuration, you will be unable to access Riak Control until you have enabled +and configured SSL and HTTPS. + +## Enabling SSL and HTTPS + +In order to use SSL in conjunction with Riak Control, SSL must be +enabled on each Riak node. For more information, see our [security documentation]({{<baseurl>}}riak/kv/2.2.6/using/security/basics#enabling-ssl). Once SSL is enabled, you can proceed to setting up [authentication](#authentication) for Riak Control. + +Please note that Riak Control will not work if you have enabled +authentication but SSL is not set up properly. + +## Authentication + +Riak Control provides you the option of requiring authentication (via +HTTP basic auth) for users of the web interface. It is disabled by +default. To enable authentication: + +```riakconf +riak_control.auth.mode = userlist +``` + +```appconfig +{riak_control, [ + %% Other configs + {auth, userlist}, %% The only other available option is "none" + %% Other configs + ]} +``` + +When authentication is enabled, you can specify as many +username/password pairs as you wish. The default pair is the username + +`user` and the password `pass`. We strongly recommend selecting +different credentials. The example below would set up three user-defined +pairs: + +```riakconf +riak_control.auth.user.bob.password = bob_is_the_coolest +riak_control.auth.user.polly.password = h4x0r123 +riak_control.auth.user.riakrocks.password = cap_theorem_4_life +``` + +```appconfig +{riak_control, [ + %% Other configs + {userlist, [ + {"bob", "bob_is_the_coolest"}, + {"polly", "h4x0r123"}, + {"riakrocks", "cap_theorem_4_life"} + ]} + %% Other configs +]} +``` + +## User Interface + +To begin using Riak Control, navigate to https://ip_address_of_https_listener:https_port/admin +For a default configuration, this will be <https://localhost:8069/admin>. + +If your browser warns you that it cannot authenticate the page, this may +be because you are using self-signed certificates. If you have +authentication enabled in your configuration, you will next be asked to +authenticate. Enter an appropriate username and password now. + +{{% note title="Note on browser TLS" %}} +Your browser needs to be support TLS v1.2 to use Riak Control over HTTPS. A +list of browsers that support TLS v1.2 can be found +[here](https://en.wikipedia.org/wiki/Transport_Layer_Security#Web_browsers). +TLS v1.2 may be disabled by default on your browser, for example if you are +using Firefox versions earlier than 27, Safari versions earlier than 7, Chrome +versions earlier than 30, or Internet Explorer versions earlier than 11. To +enable it, follow browser-specific instructions. +{{% /note %}} + +### Snapshot View + +When you first navigate to Riak Control, you will land on the Snapshot +view: + +[ ![Snapshot View]({{<baseurl>}}images/control_current_snapshot.png) ] (/images/control_current_snapshot.png) + +In this interface, the health of your cluster is made immediately +obvious. In the event that something isn't quite right (or has the +potential to cause problems in the near future), the green check mark +will turn into a red `X`. The red `X` is accompanied by a list of +reasons for concern. Each item in the list links to a page where you can +get more information about the issue. + +### Cluster Management View + +On the top right side of the admin panel are navigation tabs. If you +click the **Cluster** tab, you will be taken to the cluster management +page. + +On this page, you can see all of the nodes in your cluster, along with +their status, the percentage of the ring owned by that node, and memory +consumption. You can also stage and commit changes to the cluster, such +as adding, removing, and marking nodes as down. + +Staged changes to the cluster: + +[ ![Cluster Management Staged]({{<baseurl>}}images/control_cluster_management_staged.png) ] (/images/control_cluster_management_staged.png) + +Changes committed; transfers active: + +[ ![Cluster Management Transfers]({{<baseurl>}}images/control_cluster_management_transfers.png) ] (/images/control_cluster_management_transfers.png) + +Cluster stabilizes after changes: + +[ ![Cluster Management Stable]({{<baseurl>}}images/control_cluster_management_stable.png) ] (/images/control_cluster_management_stable.png) + +### Node Management View + +The node management view allows you to operate against the individual +nodes in the cluster. + +[ ![Node Management]({{<baseurl>}}images/control_node_management.png) ] (/images/control_node_management.png) + +### Ring View + +One level deeper than the cluster view is the ring view. This is where you can +see the health of each [vnode]({{<baseurl>}}riak/kv/2.2.6/learn/glossary/#vnode). + +[ ![Ring View]({{<baseurl>}}images/control_current_ring.png) ] (/images/control_current_ring.png) + +Most of the time, your ring will be too large to effectively manage from +the ring view. That said, with filters you can easily identify partition +ownership, unreachable primaries, and in-progress handoffs. diff --git a/content/riak/kv/2.2.6/using/cluster-operations.md b/content/riak/kv/2.2.6/using/cluster-operations.md new file mode 100644 index 0000000000..7baf2a48a0 --- /dev/null +++ b/content/riak/kv/2.2.6/using/cluster-operations.md @@ -0,0 +1,104 @@ +--- +title: "Cluster Operations" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Cluster Operations" + identifier: "managing_cluster_operations" + weight: 203 + parent: "managing" +toc: true +--- + +[ops add remove node]: ./adding-removing-nodes +[ops change info]: ./changing-cluster-info +[ops replace node]: ./replacing-node +[ops inspect node]: ./inspecting-node +[ops bucket types]: ./bucket-types +[ops log]: ./logging +[ops backup]: ./backing-up +[ops handoff]: ./handoff +[ops strong consistency]: ./strong-consistency +[ops v3 mdc]: ./v3-multi-datacenter +[ops v2 mdc]: ./v2-multi-datacenter + +## In This Section + + +#### [Adding / Removing Nodes][ops add remove node] + +Describes the process of adding and removing nodes to and from a Riak KV cluster. + +[Learn More >>][ops add remove node] + + +#### [Changing Cluster Information][ops change info] + +Details how to rename nodes and change the HTTP & Protocol Buffers binding address. + +[Learn More >>][ops change info] + + +#### [Replacing a Node][ops replace node] + +Tutorial on replacing a node in a Riak KV cluster. + +[Learn More >>][ops replace node] + + +#### [Inspecting a Node][ops inspect node] + +Guide on some of the available tools for inspecting a Riak KV node. + +[Learn More >>][ops inspect node] + + +#### [Bucket Types][ops bucket types] + +Brief tutorial on creating bucket types. + +[Learn More >>][ops bucket types] + + +#### [Enabling and Disabling Debug Logging][ops log] + +Details toggling debug logging on a single node or all nodes in a cluster. + +[Learn More >>][ops log] + + +#### [Backing Up][ops backup] + +Covers backing up Riak KV data. + +[Learn More >>][ops backup] + + +#### [Enabling and Disabling Handoff][ops handoff] + +Information on using the `riak-admin handoff` interface to enable and disable handoff. + +[Learn More >>][ops handoff] + + +#### [Monitoring Strong Consistency][ops strong consistency] + +Overview of the various statistics used in monitoring strong consistency. + +[Learn More >>][ops strong consistency] + + +#### [V3 Multi-Datacenter][ops v3 mdc] + +Explains how to manage V3 replication with the `riak-repl` command. + +[Learn More >>][ops v3 mdc] + + +#### [V2 Multi-Datacenter][ops v2 mdc] + +Explains how to manage V2 replication with the `riak-repl` command. + +[Learn More >>][ops v2 mdc] diff --git a/content/riak/kv/2.2.6/using/cluster-operations/active-anti-entropy.md b/content/riak/kv/2.2.6/using/cluster-operations/active-anti-entropy.md new file mode 100644 index 0000000000..684b56283c --- /dev/null +++ b/content/riak/kv/2.2.6/using/cluster-operations/active-anti-entropy.md @@ -0,0 +1,282 @@ +--- +title: "Managing Active Anti-Entropy" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Managing Active Anti-Entropy" + identifier: "cluster_operations_aae" + weight: 111 + parent: "managing_cluster_operations" +toc: true +aliases: + - /riak-docs/riak/kv/2.2.6/ops/advanced/aae/ + - /riak-docs/riak/2.2.6/ops/advanced/aae/ +--- + +[config search#throttledelay]: {{<baseurl>}}riak/kv/2.2.6/configuring/search/#search-anti-entropy-throttle-$tier-delay +[config search#throttle]: riak/kv/2.2.6/configuring/search/#search-anti-entropy-throttle + +Riak's [active anti-entropy](../../../learn/concepts/active-anti-entropy/) \(AAE) subsystem is a set of background processes that repair object inconsistencies stemming from missing or divergent object values across nodes. Riak operators can turn AAE on and off and configure and monitor its functioning. + +## Enabling Active Anti-Entropy + +Whether AAE is currently enabled in a node is determined by the value of +the `anti_entropy` parameter in the node's [configuration files](../../../configuring/reference/). + +In Riak versions 2.0 and later, AAE is turned on by default. + +```riakconf +anti_entropy = active +``` + +```appconfig +{riak_kv, [ + + {anti_entropy, {on, []}}, + + %% More riak_kv settings... +]} +``` + +For monitoring purposes, you can also activate AAE debugging, which +provides verbose debugging message output: + +```riakconf +anti_entropy = active-debug +``` + +```appconfig +{riak_kv, [ + + %% With debugging + {anti_entropy, {on, [debug]}}, + + %% More riak_kv settings... +]} +``` + +Remember that you will need to [restart the node](../../admin/riak-admin/#restart) for any configuration-related changes to take effect. + +## Disabling Active Anti-Entropy + +Alternatively, AAE can be switched off if you would like to repair +object inconsistencies using [read repair](../../../learn/concepts/active-anti-entropy/#read-repair) alone: + +```riakconf +anti_entropy = passive +``` + +```appconfig +{riak_kv, [ + + %% AAE turned off + {anti_entropy, {off, []}}, + + %% More riak_kv settings... +]} +``` + +If you would like to reclaim the disk space used by AAE operations, you +must manually delete the directory in which AAE-related data is stored +in each node. + +```bash +rm -Rf <path_to_riak_node>/data/anti_entropy/* +``` + +The default directory for AAE data is `./data/anti_entropy`, as in the +example above, but this can be changed. See the section below titled +**Data Directory**. + +Remember that you will need to [restart the node](../../admin/riak-admin/#restart) for any configuration-related changes to take effect. + +The directory deletion method above can also be used to force a +rebuilding of hash trees. + +## Monitoring AAE + +Riak's command-line interface includes a command that provides insight +into AAE-related processes and performance: + +```bash +riak-admin aae-status +``` + +When you run this command in a node, the output will look like this +(shortened for the sake of brevity): + +``` +================================== Exchanges ================================== +Index Last (ago) All (ago) +------------------------------------------------------------------------------- +0 19.0 min 20.3 min +22835963083295358096932575511191922182123945984 18.0 min 20.3 min +45671926166590716193865151022383844364247891968 17.3 min 19.8 min +68507889249886074290797726533575766546371837952 16.5 min 18.3 min +91343852333181432387730302044767688728495783936 15.8 min 17.3 min +... + +================================ Entropy Trees ================================ +Index Built (ago) +------------------------------------------------------------------------------- +0 5.7 d +22835963083295358096932575511191922182123945984 5.6 d +45671926166590716193865151022383844364247891968 5.5 d +68507889249886074290797726533575766546371837952 4.3 d +91343852333181432387730302044767688728495783936 4.8 d + +================================ Keys Repaired ================================ +Index Last Mean Max +------------------------------------------------------------------------------- +0 0 0 0 +22835963083295358096932575511191922182123945984 0 0 0 +45671926166590716193865151022383844364247891968 0 0 0 +68507889249886074290797726533575766546371837952 0 0 0 +91343852333181432387730302044767688728495783936 0 0 0 + +``` + +Each of these three tables contains information for each +[vnode](../../../learn/concepts/vnodes) in your cluster in these three categories: + +Category | Measures | Description +:--------|:---------|:----------- +**Exchanges** | `Last` | When the most recent exchange between a data partition and one of its replicas was performed + | `All` | How long it has been since a partition exchanged with all of its replicas +**Entropy Trees** | `Built` | When the hash trees for a given partition were created +**Keys Repaired** | `Last` | The number of keys repaired during all key exchanges since the last node restart + | `Mean` | The mean number of keys repaired during all key exchanges since the last node restart + | `Max` | The maximum number of keys repaired during all key exchanges since the last node restart + +All AAE status information obtainable using the `riak-admin aae-status` +command is stored in-memory and is reset when a node is restarted with +the exception of hash tree build information, which is persisted on disk +(because hash trees themselves are persisted on disk). + +## Configuring AAE + +Riak's [configuration files](../../../configuring/reference/) enable you not just to turn AAE on and +off but also to fine-tune your cluster's use of AAE, e.g. how +much memory AAE processes should consume, how frequently specific +processes should be run, etc. + +### Data Directory + +By default, data related to AAE operations is stored in the +`./data/anti_entropy` directory in each Riak node. This can be changed +by setting the `anti_entropy.data_dir` parameter to a different value. + +### Throttling + +AAE has a built-in throttling mechanism that can insert delays between +AAE repair operations when [vnode](../../../learn/concepts/vnodes) mailboxes reach the length +specified by the [`search.anti_entropy.throttle.$tier.delay`][config search#throttledelay] parameter (more on +that in the section below). Throttling can be switched on and off using +the [`search.anti_entropy.throttle`][config search#throttle] parameter. The default is `on`. + +#### Throttling Tiers + +If you activate AAE throttling, you can use *tiered throttling* to +establish a series of vnode mailbox-size thresholds past which a +user-specified time delay should be observed. This enables you to +establish, for example, that a delay of 10 milliseconds should be +observed if the mailbox of any vnode reaches 50 messages. + +The general form for setting tiered throttling is as follows: + +```riakconf +search.anti_entropy.throttle.$tier.delay +search.anti_entropy.throttle.$tier.solrq_queue_length +``` + +In the above example, `$tier` should be replaced with the desired +name for that tier (e.g. `tier1`, `large_mailbox_tier`, etc). If you +choose to set throttling tiers, you will need to set the mailbox size +for one of the tiers to 0. Both the `.solrq_queue_length` and `.delay` +parameters must be set for each tier. + +Below is an example configuration for three tiers, with mailbox sizes of +0, 50, and 100 and time delays of 5, 10, and 15 milliseconds, +respectively: + +```riakconf +search.anti_entropy.throttle.tier1.solrq_queue_length = 0 +search.anti_entropy.throttle.tier1.delay = 5ms +search.anti_entropy.throttle.tier2.solrq_queue_length = 50 +search.anti_entropy.throttle.tier2.delay = 10ms +search.anti_entropy.throttle.tier3.solrq_queue_length = 100 +search.anti_entropy.throttle.tier3.delay = 15ms +``` + +### Bloom Filters + +Bloom filters are mechanisms used to prevent reads that are destined to +fail because no object exists in the location that they're querying. +Using bloom filters can improve reaction time for some queries, but +entail a small general performance cost. You can switch bloom filters +on and off using the `anti_entropy.bloomfilter` parameter. + +### Trigger Interval + +The `anti_entropy.trigger_interval` setting determines how often Riak's +AAE subsystem looks for work to do, e.g. building or expiring hash +trees, triggering information exchanges between nodes, etc. The default +is every 15 seconds (`15s`). Raising this value may save resources, but +at a slightly higher risk of data corruption. + +### Hash Trees + +As a fallback measure in addition to the normal operation of AAE on-disk +hash trees, Riak periodically clears and regenerates all hash trees +stored on disk to ensure that hash trees correspond to the key/value +data stored in Riak. This enables Riak to detect silent data corruption +resulting from disk failure or faulty hardware. The +`anti_entropy.tree.expiry` setting enables you to determine how often +that takes place. The default is once a week (`1w`). You can set up this +process to run once a day (`1d`), twice a day (`12h`), once a month +(`4w`), and so on. + +In addition to specifying how often Riak expires hash trees after they +are built, you can also specify how quickly and how many hash trees are +built. You can set the frequency using the +`anti_entropy.tree.build_limit.per_timespan` parameter, for which the +default is every hour (`1h`); the number of hash tree builds is +specified by `anti_entropy.tree.build_limit.number`, for which the +default is 1. + +### Write Buffer Size + +While you are free to choose the backend for data storage in Riak, +background AAE processes use [LevelDB](../../../setup/planning/backend/leveldb). You can adjust the size of the +write buffer used by LevelDB for hash tree generation using the +`anti_entropy.write_buffer_size` parameter. The default is `4MB`. + +### Open Files and Concurrency Limits + +The `anti_entropy.concurrency_limit` parameter determines how many AAE +cross-node information exchanges or hash tree builds can happen +concurrently. The default is `2`. + +The `anti_entropy.max_open_files` parameter sets an open-files limit for +AAE-related background tasks, analogous to [open files limit](../../performance/open-files-limit) settings used in operating systems. The default is `20`. + +## AAE and Riak Search + +Riak's AAE subsystem works to repair object inconsistencies both with +for normal key/value objects as well as data related to [Riak Search](../../../developing/usage/search). In particular, AAE acts on indexes stored in +[Solr](http://lucene.apache.org/solr/), the search platform that drives +Riak Search. Implementation details for AAE and Search can be found in +the [Search Details](../../reference/search/#active-anti-entropy-aae) +documentation. + +You can check on the status of Search-related AAE using the following +command: + +```bash +riak-admin search aae-status +``` + +The output from that command can be interpreted just like the output +discussed in the section on [monitoring](#monitoring-aae) above. diff --git a/content/riak/kv/2.2.6/using/cluster-operations/adding-removing-nodes.md b/content/riak/kv/2.2.6/using/cluster-operations/adding-removing-nodes.md new file mode 100644 index 0000000000..237e674f37 --- /dev/null +++ b/content/riak/kv/2.2.6/using/cluster-operations/adding-removing-nodes.md @@ -0,0 +1,194 @@ +--- +title: "Adding / Removing Nodes" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Adding/Removing Nodes" + identifier: "cluster_operations_add_remove_nodes" + weight: 100 + parent: "managing_cluster_operations" +toc: true +aliases: + - /riak-docs/riak/2.2.6/ops/running/nodes/adding-removing + - /riak-docs/riak/kv/2.2.6/ops/running/nodes/adding-removing +--- + +[use running cluster]: {{<baseurl>}}riak/kv/2.2.6/using/running-a-cluster + +This page describes the process of adding and removing nodes to and from +a Riak KV cluster. For information on creating a cluster check out [Running a Cluster][use running cluster]. + +## Start the Node + +Just like the initial configuration steps, this step has to be repeated +for every node in your cluster. Before a node can join an existing +cluster it needs to be started. Depending on your mode of installation, +use either the init scripts installed by the Riak binary packages or +simply the script [`riak`]({{<baseurl>}}riak/kv/2.2.6/using/admin/riak-cli/): + +```bash +/etc/init.d/riak start +``` + +or + +```bash +bin/riak start +``` + +When the node starts, it will look for a cluster description, known as +the **ring file**, in its data directory. If a ring file does not exist, +it will create a new ring file based on the initially configured +`ring_size` (or `ring_creation_size` if you're using the older, +`app.config`-based configuration system), claiming all partitions for +itself. Once this process completes, the node will be ready to serve +requests. + +## Add a Node to an Existing Cluster + +Once the node is running, it can be added to an existing cluster. Note +that this step isn't necessary for the first node; it's necessary only +for nodes that you want to add later. + +To join the node to an existing cluster, use the `cluster join` command: + +```bash +bin/riak-admin cluster join <node_in_cluster> +``` + +The `<node_in_cluster>` in the example above can be _any_ node in the +cluster you want to join to. So if the existing cluster consists of +nodes `A`, `B`, and `C`, any of the following commands would join the +new node: + +```bash +bin/riak-admin cluster join A +bin/riak-admin cluster join B +bin/riak-admin cluster join C +``` + +To give a more realistic example, let's say that you have an isolated +node named `riak@192.168.2.5` and you want to join it to an existing +cluster that contains a node named `riak@192.168.2.2`. This command +would stage a join to that cluster: + +```bash +bin/riak-admin cluster join riak@192.168.2.2 +``` + +If the join request is successful, you should see the following: + +``` +Success: staged join request for 'riak@192.168.2.5' to 'riak@192.168.2.2' +``` + +If you have multiple nodes that you would like to join to an existing +cluster, repeat this process for each of them. + +## Joining Nodes to Form a Cluster + +The process of joining a cluster involves several steps, including +staging the proposed cluster nodes, reviewing the cluster plan, and +committing the changes. + +After staging each of the cluster nodes with `riak-admin cluster join` +commands, as in the section above, the next step in forming a cluster is +to review the proposed plan of changes. This can be done with the +`riak-admin cluster plan` command, which is shown in the example below. + +``` +=============================== Staged Changes ================================ +Action Nodes(s) +------------------------------------------------------------------------------- +join 'riak@192.168.2.2' +join 'riak@192.168.2.2' +join 'riak@192.168.2.2' +join 'riak@192.168.2.2' +------------------------------------------------------------------------------- + + +NOTE: Applying these changes will result in 1 cluster transition + +############################################################################### + After cluster transition 1/1 +############################################################################### + +================================= Membership ================================== +Status Ring Pending Node +------------------------------------------------------------------------------- +valid 100.0% 20.3% 'riak@192.168.2.2' +valid 0.0% 20.3% 'riak@192.168.2.3' +valid 0.0% 20.3% 'riak@192.168.2.4' +valid 0.0% 20.3% 'riak@192.168.2.5' +valid 0.0% 18.8% 'riak@192.168.2.6' +------------------------------------------------------------------------------- +Valid:5 / Leaving:0 / Exiting:0 / Joining:0 / Down:0 + +Transfers resulting from cluster changes: 51 + 12 transfers from 'riak@192.168.2.2' to 'riak@192.168.2.3' + 13 transfers from 'riak@192.168.2.2' to 'riak@192.168.2.4' + 13 transfers from 'riak@192.168.2.2' to 'riak@192.168.2.5' + 13 transfers from 'riak@192.168.2.2' to 'riak@192.168.2.6' +``` + +If the plan is to your liking, submit the changes by running `riak-admin +cluster commit`. + +{{% note title="Note on ring changes" %}} +The algorithm that distributes partitions across the cluster during membership +changes is non-deterministic. As a result, there is no optimal ring. In the +event that a plan results in a slightly uneven distribution of partitions, the +plan can be cleared. Clearing a cluster plan with `riak-admin cluster clear` +and running `riak-admin cluster plan` again will produce a slightly different +ring. +{{% /note %}} + +## Removing a Node From a Cluster + +A node can be removed from the cluster in two ways. One assumes that a +node is decommissioned, for example, because its added capacity is not +needed anymore or because it's explicitly replaced with a new one. The +second is relevant for failure scenarios in which a node has crashed and +is irrecoverable and thus must be removed from the cluster from another +node. + +The command to remove a running node is `riak-admin cluster leave`. This +command must be executed on the node that you intend to removed from the +cluster. + +Similarly to joining a node, after executing `riak-admin cluster leave` +the cluster plan must be reviewed with `riak-admin cluster plan` and +the changes committed with `riak-admin cluster commit`. + +The other command is `riak-admin cluster leave <node>`, where `<node>` +is the node name as specified in the node's configuration files: + +```bash +riak-admin cluster leave riak@192.168.2.1 +``` + +This command can be run from any other node in the cluster. + +Under the hood, both commands do basically the same thing. Running +`riak-admin cluster leave <node>` selects the current node for you +automatically. + +As with `riak-admin cluster leave`, the plan to have a node leave the +cluster must be first reviewed with `riak-admin cluster plan` and +committed with `riak-admin cluster commit` before any changes will +actually take place. + + +## Pausing a `join` or `leave` + +{{% note title="Warning" %}} +Pausing may impact cluster health and is not recommended for more than a short period of time. +{{% /note %}} + +To pause during `riak-admin cluster join` or `riak-admin cluster leave`, set the node's transfer-limit to 0: + +```bash +riak-admin transfer-limit <node> 0 +``` diff --git a/content/riak/kv/2.2.6/using/cluster-operations/backend.md b/content/riak/kv/2.2.6/using/cluster-operations/backend.md new file mode 100644 index 0000000000..3adae4ee50 --- /dev/null +++ b/content/riak/kv/2.2.6/using/cluster-operations/backend.md @@ -0,0 +1,16 @@ +--- +draft: true +title: "Backend" +description: "" +project: "riak_kv" +project_version: "2.2.6" +#menu: +# riak_kv-2.2.6: +# name: "Backend" +# identifier: "cluster_operations_backend" +# weight: 112 +# parent: "managing_cluster_operations" +toc: true +--- + +**TODO: Add content** diff --git a/content/riak/kv/2.2.6/using/cluster-operations/backing-up.md b/content/riak/kv/2.2.6/using/cluster-operations/backing-up.md new file mode 100644 index 0000000000..9a60c2be28 --- /dev/null +++ b/content/riak/kv/2.2.6/using/cluster-operations/backing-up.md @@ -0,0 +1,267 @@ +--- +title: "Backing Up" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Backing Up" + identifier: "cluster_operations_backing_up" + weight: 106 + parent: "managing_cluster_operations" +toc: true +aliases: + - /riak-docs/riak/2.2.6/ops/running/backups + - /riak-docs/riak/kv/2.2.6/ops/running/backups +--- + +[concept clusters]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/clusters +[config reference]: {{<baseurl>}}riak/kv/2.2.6/configuring/reference +[plan backend leveldb]: {{<baseurl>}}riak/kv/2.2.6/setup/planning/backend/leveldb +[plan backend bitcask]: {{<baseurl>}}riak/kv/2.2.6/setup/planning/backend/bitcask +[use ref strong consistency]: {{<baseurl>}}riak/kv/2.2.6/using/reference/strong-consistency +[concept aae]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/active-anti-entropy/ +[aae read repair]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/active-anti-entropy/#read-repair-vs-active-anti-entropy + +Riak KV is a [clustered][concept clusters] system built to survive a wide range of failure scenarios, including the loss of nodes due to network or hardware failure. Although this is one of Riak KV's core strengths, it cannot withstand all failure scenarios. + +Backing up data (duplicating the database on a different long-term storage system) is a common approach to mitigating potential failure scenarios. + +This page covers how to perform backups of Riak KV data. + +## Overview + +Riak KV backups can be performed using operating system features or filesystems that support snapshots, such as LVM or ZFS, or by using tools like rsync or tar. + +Choosing your Riak KV backup strategy will depend on your already-established backup methodologies and the backend configuration of your nodes. + +The basic process for getting a backup of Riak KV from a node is as follows: + +1. Stop Riak KV with `riak stop`. +2. Backup the appropriate data, ring, and configuration directories. +3. Start Riak KV. + +Downtime of a node can be significantly reduced by using an OS feature or filesystem that supports snapshotting. + +{{% note title="Backups and eventual consistency" %}} +Due to Riak KV's eventually consistent nature, backups can become slightly inconsistent from node to node. + +Data could exist on some nodes and not others at the exact time a backup is made. Any inconsistency will be corrected once a backup is restored, either by Riak's [active anti-entropy]({{<baseurl>}}riak/kv/2.2.6/learn/concepts/active-anti-entropy/) processes or when the object is read, via [read repair]({{<baseurl>}}riak/kv/2.2.6/learn/concepts/active-anti-entropy/#read-repair-vs-active-anti-entropy). +{{% /note %}} + +## OS-Specific Directory Locations + +The default Riak KV data, ring, and configuration directories for each of the supported operating systems is as follows: + +#### Debian and Ubuntu + +Data | Directory +:----|:--------- +Bitcask | `/var/lib/riak/bitcask` +LevelDB | `/var/lib/riak/leveldb` +Ring | `/var/lib/riak/ring` +Configuration | `/etc/riak` +Cluster Metadata | `/var/lib/riak/cluster_meta` +Search | `/var/lib/riak/yz` +Strong consistency | `/var/lib/riak/ensembles` + +#### Fedora and RHEL + +Data | Directory +:----|:--------- +Bitcask | `/var/lib/riak/bitcask` +LevelDB | `/var/lib/riak/leveldb` +Ring | `/var/lib/riak/ring` +Configuration | `/etc/riak` +Cluster Metadata | `/var/lib/riak/cluster_meta` +Search | `/var/lib/riak/yz` +Strong consistency | `/var/lib/riak/ensembles` + +#### FreeBSD + +Data | Directory +:----|:--------- +Bitcask | `/var/db/riak/bitcask` +LevelDB | `/var/db/riak/leveldb` +Ring | `/var/db/riak/ring` +Configuration | `/usr/local/etc/riak` +Cluster Metadata | `/var/db/riak/cluster_meta` +Search | `/var/db/riak/yz` +Strong consistency | `/var/db/riak/ensembles` + +#### OS X + +Data | Directory +:----|:--------- +Bitcask | `./data/bitcask` +LevelDB | `./data/leveldb` +Ring | `./data/riak/ring` +Configuration | `./etc` +Cluster Metadata | `./data/riak/cluster_meta` +Search | `./data/riak/yz` +Strong consistency | `./data/ensembles` + +**Note**: OS X paths are relative to the directory in which the package +was extracted. + +#### SmartOS + +Data | Directory +:----|:--------- +Bitcask | `/var/db/riak/bitcask` +LevelDB | `/var/db/riak/leveldb` +Ring | `/var/db/riak/ring` +Configuration | `/opt/local/etc/riak` +Cluster Metadata | `/var/db/riak/cluster_meta` +Search | `/var/db/riak/yz` +Strong consistency | `/var/db/riak/ensembles` + +#### Solaris + +Data | Directory +:----|:--------- +Bitcask | `/opt/riak/data/bitcask` +LevelDB | `/opt/riak/data/leveldb` +Ring | `/opt/riak/ring` +Configuration | `/opt/riak/etc` +Cluster Metadata | `/opt/riak/cluster_meta` +Search | `/opt/riak/yz` +Strong consistency | `/opt/riak/data/ensembles` + +## Performing Backups + +{{% note title="Deprecation notice" %}} +In previous versions of Riak KV, there was a [`riak-admin backup`]({{<baseurl>}}riak/kv/2.2.6/using/admin/riak-admin/#backup) command commonly used for +backups. This functionality is now deprecated. We strongly recommend using the backup procedure documented below instead. +{{% /note %}} + +Backups can be accomplished through a variety of common methods. Standard utilities such `cp`, `rsync`, and `tar` can be used, as well as any backup system already in place in your environment. + +A simple shell command, like those in the following examples, are sufficient for creating a backup of your Bitcask or LevelDB data, ring, and Riak KV configuration directories for a binary package-based Riak KV Linux +installation. + +The following examples use `tar`: + +{{% note %}} +Backups must be performed on while Riak KV is stopped to prevent data loss. +{{% /note %}} + +### Bitcask + +```bash +tar -czf /mnt/riak_backups/riak_data_`date +%Y%m%d_%H%M`.tar.gz \ + /var/lib/riak/bitcask /var/lib/riak/ring /etc/riak +``` + +### LevelDB + +```bash +tar -czf /mnt/riak_backups/riak_data_`date +%Y%m%d_%H%M`.tar.gz \ + /var/lib/riak/leveldb /var/lib/riak/ring /etc/riak +``` + +### Cluster Metadata + +```bash +tar -czf /mnt/riak_backups/riak_data_`date +%Y%m%d_%H%M`.tar.gz \ + /var/lib/riak/cluster_meta +``` + +### Search / Solr Data + +```bash +tar -czf /mnt/riak_backups/riak_data_`date +%Y%m%d_%H%M`.tar.gz \ + /var/lib/riak/yz +``` + +### Strong Consistency Data + +Persistently stored data used by Riak's [strong consistency][use ref strong consistency] feature +can be stored in an analogous fashion: + +```bash +tar -czf /mnt/riak_backups/riak_data_`date +%Y%m%d_%H%M`.tar.gz \ + /var/lib/riak/ensembles +``` + +## Restoring a Node + +The method you use to restore a node will differ depending on a combination of factors, including node name changes and your network environment. + +If you are replacing a node with a new node that has the same node name (typically a fully qualified domain name or IP address), then restoring the node is a simple process: + +1. Install Riak on the new node. +2. Restore your old node's configuration files, data directory, and ring + directory. +3. Start the node and verify proper operation with `riak ping`, + `riak-admin status`, and other methods you use to check node health. + +If the node name of a restored node (`-name` argument in `vm.args` or +`nodename` parameter in `riak.conf`) is different than the name of the +node that the restored backup was taken from, you will need to +additionally: + +1. Mark the original instance down in the cluster using + [`riak-admin down <node>`]({{<baseurl>}}riak/kv/2.2.6/using/admin/riak-admin/#down) +2. Join the restored node to the cluster using + [`riak-admin cluster join <node>`]({{<baseurl>}}riak/kv/2.2.6/using/admin/riak-admin/#cluster-join) +3. Replace the original instance with the renamed instance with + [`riak-admin cluster force-replace <node1> <node2>`]({{<baseurl>}}riak/kv/2.2.6/using/admin/riak-admin/#cluster-force-replace) +4. Plan the changes to the cluster with `riak-admin cluster plan` +5. Finally, commit the cluster changes with `riak-admin cluster commit` + +{{% note %}} +For more information on the `riak-admin cluster` commands, refer to our documentation on [cluster administration]({{<baseurl>}}riak/kv/2.2.6/using/admin/). +{{% /note %}} + +For example, if there are five nodes in the cluster with the original node names `riak1.example.com` through `riak5.example.com` and you wish to restore `riak1.example.com` as `riak6.example.com`, you would execute the following commands on `riak6.example.com`. + +1. Join to any existing cluster node. + + ```bash + riak-admin cluster join riak@riak2.example.com + ``` + +2. Mark the old instance down. + + ```bash + riak-admin down riak@riak1.example.com + ``` + +3. Force-replace the original instance with the new one. + + ```bash + riak-admin cluster force-replace \ + riak@riak1.example.com riak@riak6.example.com + ``` + +4. Display and review the cluster change plan. + + ```bash + riak-admin cluster plan + ``` + +5. Commit the changes to the cluster. + + ```bash + riak-admin cluster commit + ``` + +Your [configuration files][config reference] should also be changed to match the new name in addition to running the commands (the `-name` setting in `vm.args` in the older config system, and the `nodename` setting in `riak.conf` in the newer system). + +If the IP address of any node has changed, verify that the changes are reflected in your configuration files to ensure that the HTTP and Protocol Buffers interfaces are binding to the correct addresses. + +A robust DNS configuration can simplify the restore process if the IP addresses of the nodes change, but the hostnames are used for the node names and the hostnames stay the same. Additionally, if the HTTP and Protocol Buffers interface settings are configured to bind to all IP interfaces (0.0.0.0), then no changes will need to be made to your configuration files. + +When performing restore operations involving `riak-admin cluster force-replace`, we recommend that you start only one node at a time and verify that each node that is started has the correct name for itself +and for any other nodes whose names have changed: + +1. Verify that the correct name is present your configuration file. +2. Once the node is started, run `riak attach` to connect to the node. The prompt obtained should contain the correct node name. + - (It may be necessary to enter an Erlang atom by typing `x.` and pressing Enter) +3. Disconnect from the attached session with **Ctrl-G + q**. +4. Finally, run `riak-admin member_status` to list all of the nodes and verify that all nodes listed have the correct names. + +## Restoring a Cluster + +Restoring a cluster from backups is documented [on its own page]({{<baseurl>}}riak/kv/2.2.6/using/repair-recovery/failure-recovery/#cluster-recovery-from-backups). diff --git a/content/riak/kv/2.2.6/using/cluster-operations/bucket-types.md b/content/riak/kv/2.2.6/using/cluster-operations/bucket-types.md new file mode 100644 index 0000000000..62f7544a35 --- /dev/null +++ b/content/riak/kv/2.2.6/using/cluster-operations/bucket-types.md @@ -0,0 +1,58 @@ +--- +title: "Bucket Types" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Bucket Types" + identifier: "cluster_operations_bucket_types" + weight: 104 + parent: "managing_cluster_operations" +toc: true +--- + +Buckets are essentially a flat namespace in Riak. They allow the same +key name to exist in multiple buckets and enable you to apply +configurations across keys. + +{{% note title="How Many Buckets Can I Have?" %}} +Buckets come with virtually no cost _except for when you modify the default +bucket properties_. Modified bucket properties are gossiped around the cluster +and therefore add to the amount of data sent around the network. In other +words, buckets using the `default` bucket type are free. More on that in the +next section. +{{% /note %}} + +In Riak versions 2.0 and later, Basho suggests that you [use bucket types]({{<baseurl>}}riak/kv/2.2.6/developing/usage/bucket-types) to namespace and configure all buckets you use. Bucket types have a lower overhead within the cluster than the +default bucket namespace but require an additional setup step on the +command line. + +## Creating a Bucket Type + +When creating a new bucket type, you can create a bucket type without +any properties and set individual buckets to be indexed. The step below +creates and activates the bucket type: + +```bash +riak-admin bucket-type create animals '{"props":{}}' +riak-admin bucket-type activate animals +``` + +And this step applies the index to the `cats` bucket, which bears the +`animals` bucket type we just created and activated: + +```curl +curl -XPUT $RIAK_HOST/types/animals/buckets/cats/props \ + -H 'Content-Type: application/json' \ + -d '{"props":{"search_index":"famous"}}' +``` + +Another possibility is to set the `search_index` as a default property +of the bucket type. This means _any_ bucket under that type will +inherit that setting and have its values indexed. + +```bash +riak-admin bucket-type create animals '{"props":{"search_index":"famous"}}' +riak-admin bucket-type activate animals +``` diff --git a/content/riak/kv/2.2.6/using/cluster-operations/changing-cluster-info.md b/content/riak/kv/2.2.6/using/cluster-operations/changing-cluster-info.md new file mode 100644 index 0000000000..d874b00b66 --- /dev/null +++ b/content/riak/kv/2.2.6/using/cluster-operations/changing-cluster-info.md @@ -0,0 +1,454 @@ +--- +title: "Changing Cluster Information" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Changing Cluster Info" + identifier: "cluster_operations_change_info" + weight: 101 + parent: "managing_cluster_operations" +toc: true +aliases: + - /riak-docs/riak/2.2.6/ops/running/nodes/renaming + - /riak-docs/riak/kv/2.2.6/ops/running/nodes/renaming +--- + +[config reference]: {{<baseurl>}}riak/kv/2.2.6/configuring/reference + +## Change the Node Name + +The node name is an important setting for the Erlang VM, especially when +you want to build a cluster of nodes, as the node name identifies both +the Erlang application and the host name on the network. All nodes in +the Riak cluster need these node names to communicate and coordinate +with each other. + +In your configuration files, the node name defaults to `riak@127.0.0.1`. +To change the node name, change the following line: + +```riakconf +nodename = riak@127.0.0.1 +``` + +```vmargs +-name riak@127.0.0.1 +``` + +Change it to something that corresponds to either the IP address or a +resolvable host name for this particular node, like so: + +```riakconf +nodename = riak@192.168.1.10 +``` + +```vmargs +-name riak@192.168.1.10 +``` + +## Change the HTTP and Protocol Buffers binding address + +By default, Riak's HTTP and Protocol Buffers services are bound to the +local interface, i.e. 127.0.0.1, and are therefore unable to serve +requests from the outside network. The relevant setting is in your +[configuration files][config reference]: + +```riakconf +# For HTTP +listener.http.internal = 127.0.0.1:8098 + +# For Protocol Buffers +listener.protobuf.internal = 127.0.0.1:8087 +``` + +```appconfig +% In the riak_api section + +% For HTTP +{http, [ {"127.0.0.1", 8098 } ]}, + +% For Protocol Buffers +{pb, [ {"127.0.0.1", 8087} ] }, +``` + +Either change it to use an IP address that corresponds to one of the +server's network interfaces, or 0.0.0.0 to allow access from all +interfaces and networks, e.g.: + +```riakconf +listener.http.internal = 0.0.0.0:8098 +``` + +```appconfig +% In the riak_core section +{http, [ {"0.0.0.0", 8098 } ]}, +``` + +The same configuration should be changed for the Protocol Buffers +interface if you intend on using it (which we recommend). Change the +following line: + +```riakconf +listener.protobuf.internal = 0.0.0.0:8087 +``` + +```appconfig +% In the riak_core section +{pb, [ {"0.0.0.0", 8087} ] }, +``` + +## Rename Single Node Clusters + +To rename a single-node development cluster: + +1. Stop the node with `riak stop`. + +2. Change the node's `nodename` parameter in `riak.conf`, or `-name` parameter in `vm.args` to the new name. + +3. Change any IP addresses in `riak.conf` or `app.config` if necessary. Specifically: `listener.protobuf.$name`, `listener.http.$name`, and `listener.https.$name` in `riak.conf`, and `pb_ip`, `http`, `https`, and `cluster_mgr` in `app.config`. + +4. Delete the contents of the node's `ring` directory. The location of the ring directory is the value for the `ring.state_dir` in `riak.conf`, or `ring_state_dir` in `app.config`. + +5. Start Riak on the node with `riak start`. + + +## Rename Multi-Node Clusters + +For multi-node clusters, a rename is a slightly more complex procedure; however, it is very similar to the process for renaming a single node. + +Previous to Riak version 1.2, a cluster node's name could only be changed with the [`riak-admin reip`]({{<baseurl>}}riak/kv/2.2.6/using/admin/riak-admin/#reip) command, which involves downtime for the entire cluster. As of Riak version 1.2, that method has been superseded by [`riak-admin cluster force-replace`]({{<baseurl>}}riak/kv/2.2.6/using/admin/riak-admin/#cluster-force-replace), which is safer and does not require cluster wide downtime. + +There still exist scenarios that require nodes to be renamed while stopped, such as seeding a cluster with backups from another cluster that does not share the same node names. Please see the [Clusters from Backups](#clusters-from-backups) section for more details on renaming in this scenario. + +The following example describes reconfiguring node names with the new `riak-admin cluster force-replace` method. + +### Example Scenario + +For this example scenario, Riak is operating in a cluster of 5 nodes with the following network configuration: + +* `riak@10.1.42.11` on `node1.localdomain` → IP address changing to 192.168.17.11 +* `riak@10.1.42.12` on `node2.localdomain` → IP address changing to 192.168.17.12 +* `riak@10.1.42.13` on `node3.localdomain` → IP address changing to 192.168.17.13 +* `riak@10.1.42.14` on `node4.localdomain` → IP address changing to 192.168.17.14 +* `riak@10.1.42.15` on `node5.localdomain` → IP address changing to 192.168.17.15 + +The above list shows the network configuration details for our 5 nodes, including the Erlang node name value, the node's fully qualified domain name, and the new IP address each node will be configured to use. + +The nodes in our example cluster are currently configured to use the *10.1.42.* private subnetwork range. Our goal for this example will be to configure the nodes to instead use the *192.168.17.* private subnetwork range and do so in a rolling fashion without interrupting cluster operation. + +### Process + +This process can be accomplished in three phases. The details and steps required of each phase are presented in the following section. + +1. [Down the node to be reconfigured](#down) +2. [Reconfigure node to use new address](#reconfigure) +3. [Repeat previous steps on each node](#repeat) + + +<a id="down"></a> +#### Down the Node + +1. Stop Riak on `node1.localdomain`: + + ```bash + riak stop + ``` + + The output should look like this: + + ``` + Attempting to restart script through sudo -H -u riak + ok + ``` + +2. From the `node2.localdomain` node, mark `riak@10.1.42.11` down: + + ```bash + riak-admin down riak@10.1.42.11 + ``` + + Successfully marking the node down should produce output like this: + + ```bash + Attempting to restart script through sudo -H -u riak + Success: "riak@10.1.42.11" marked as down + ``` + + This step informs the cluster that `riak@10.1.42.11` is offline and ring-state transitions should be allowed. While we're executing the `riak-admin down` command from `node2.localdomain` in this example, the command can be executed from any currently running node. + +<a id="reconfigure"></a> +#### Reconfigure Node to Use New Address + +Reconfigure `node1.localdomain` to listen on the new private IP address *192.168.17.11* by following these steps: + +1. Change the node's `nodename` parameter in `riak.conf`, or `-name` parameter in `vm.args`, to reflect the new node name. For example: + + `riak.conf`: `nodename = riak@192.168.17.11` + `vm.args` : `-name riak@192.168.17.11` + +2. Change any IP addresses to *192.168.17.11* in `riak.conf` or `app.config` as previously described in step 3 of [Single Node Clusters](#single-node-clusters). + +3. Rename the node's `ring` directory, the location of which is described in step 4 of [Single Node Clusters](#single-node-clusters). You may rename it to whatever you like, as it will only be used as a backup during the node renaming process. + +4. Start Riak on `node1.localdomain`. + + ```bash + riak start + ``` + +5. Join the node back into the cluster. + + ```bash + riak-admin cluster join riak@10.1.42.12 + ``` + + Successful staging of the join request should have output like this: + + ```bash + Attempting to restart script through sudo -H -u riak + Success: staged join request for 'riak@192.168.17.11' to 'riak@10.1.42.12' + ``` + +6. Use `riak-admin cluster force-replace` to change all ownership references from `riak@10.1.42.11` to `riak@192.168.17.11`: + + ```bash + riak-admin cluster force-replace riak@10.1.42.11 riak@192.168.17.11 + ``` + + Successful force replacement staging output looks like this: + + ```bash + Attempting to restart script through sudo -H -u riak + Success: staged forced replacement of 'riak@10.1.42.11' with 'riak@192.168.17.11' + ``` + +7. Review the new changes with `riak-admin cluster plan:` + + ```bash + riak-admin cluster plan + ``` + + Example output: + + ```bash + Attempting to restart script through sudo -H -u riak + =========================== Staged Changes ============================ + Action Nodes(s) + ----------------------------------------------------------------------- + join 'riak@192.168.17.11' + force-replace 'riak@10.1.42.11' with 'riak@192.168.17.11' + ----------------------------------------------------------------------- + + WARNING: All of 'riak@10.1.42.11' replicas will be lost + + NOTE: Applying these changes will result in 1 cluster transition + + ####################################################################### + After cluster transition 1/1 + ####################################################################### + + ============================= Membership ============================== + Status Ring Pending Node + ----------------------------------------------------------------------- + valid 20.3% -- 'riak@192.168.17.11' + valid 20.3% -- 'riak@10.1.42.12' + valid 20.3% -- 'riak@10.1.42.13' + valid 20.3% -- 'riak@10.1.42.14' + valid 18.8% -- 'riak@10.1.42.15' + ----------------------------------------------------------------------- + Valid:5 / Leaving:0 / Exiting:0 / Joining:0 / Down:0 + + Partitions reassigned from cluster changes: 13 + 13 reassigned from 'riak@10.1.42.11' to 'riak@192.168.17.11' + ``` + +8. Commit the new changes to the cluster with `riak-admin cluster commit`: + + ```bash + riak-admin cluster commit + ``` + + Output from the command should resemble this example: + + ```bash + Attempting to restart script through sudo -H -u riak + Cluster changes committed + ``` + +9. Check that the node is participating in the cluster and functioning as expected: + + ```bash + riak-admin member-status + ``` + + Output should resemble this example: + + ```bash + Attempting to restart script through sudo -H -u riak + ============================= Membership ============================== + Status Ring Pending Node + ----------------------------------------------------------------------- + valid 20.3% -- 'riak@192.168.17.11' + valid 20.3% -- 'riak@10.1.42.12' + valid 20.3% -- 'riak@10.1.42.13' + valid 20.3% -- 'riak@10.1.42.14' + valid 18.8% -- 'riak@10.1.42.15' + ----------------------------------------------------------------------- + Valid:5 / Leaving:0 / Exiting:0 / Joining:0 / Down:0 + ``` + +10. Monitor hinted handoff transfers to ensure they have finished with the `riak-admin transfers` command. + +11. Clean up by deleting the renamed `ring` directory once all previous steps have been successfully completed. + +{{% note title="Note" %}} +When using the `riak-admin force-replace` command, you will always get a +warning message like: `WARNING: All of 'riak@10.1.42.11' replicas will be +lost`. Since we didn't delete any data files and we are replacing the node +with itself under a new name, we will not lose any replicas. +{{% /note %}} + +<a id="repeat"></a> +#### Repeat previous steps on each node + +Repeat the steps above for each of the remaining nodes in the cluster. + +Use *riak@192.168.17.11* as the target node for further `riak-admin cluster join` commands issued from subsequently reconfigured nodes to join those nodes to the cluster. + +```bash +riak-admin cluster join riak@192.168.17.11 +``` + +A successful join request staging produces output similar to this example: + +```bash +Attempting to restart script through sudo -H -u riak +Success: staged join request for 'riak@192.168.17.12' to 'riak@192.168.17.11' +``` + +## Clusters from Backups + +The above steps describe a process for renaming nodes in a running cluster. When seeding a new cluster with backups where the nodes must have new names, typically done as a secondary cluster or in a disaster recovery scenario, a slightly different process must be used. This is because the node names must resolve to the new hosts in order for the nodes to start and communicate with each other. + +Expanding on the Example Scenario above, the below steps can be used to rename nodes in a cluster that is being restored from backups. The below steps assume every node is offline, and they will indicate when to bring each node online. + +#### Bringing Up the First Node + +In order to bring our first node online, we'll first need to use the `riak-admin reip` command on a single node. In this example, we'll use `riak@10.1.42.11` as our first node. + +1. In `riak.conf` change `nodename`, `-name` in `vm.args`, from `riak@10.1.42.11` to your new nodename, `riak@192.168.17.11`. + +2. On `node1.localdomain` run `riak-admin reip riak@10.1.42.11 riak@192.168.17.11`. This will change the name of `riak@10.1.42.11` to `riak@192.168.17.11` in the Riak ring. + +3. Start Riak on `node1.localdomain`. + +4. Once Riak is started on `node1.localdomain`, mark the rest of the nodes in the cluster down, using `riak-admin down`. For example, we would down `riak@10.1.42.12` with `riak-admin down riak@10.1.42.12`. + +5. Confirm every other node in the cluster is marked down by running `riak-admin member-status` on `node1.localdomain`: + + ```bash + ================================= Membership ================================== + Status Ring Pending Node + ------------------------------------------------------------------------------- + valid 20.3% -- 'riak@192.168.17.11' + down 20.3% -- 'riak@10.1.42.12' + down 20.3% -- 'riak@10.1.42.13' + down 20.3% -- 'riak@10.1.42.14' + down 18.8% -- 'riak@10.1.42.15' + ------------------------------------------------------------------------------- + Valid:1 / Leaving:0 / Exiting:0 / Joining:0 / Down:4 + + ``` + +6. Ensure `riak@192.168.17.11` is listed as the claimant by running `riak-admin ring-status` on `node1.localdomain`: + + ```bash + ================================== Claimant =================================== + Claimant: 'riak@192.168.17.11' + Status: up + Ring Ready: true + + ============================== Ownership Handoff ============================== + No pending changes. + + ============================== Unreachable Nodes ============================== + All nodes are up and reachable + ``` + +Once all nodes are marked as down and our first node is listed as the claimant, we can proceed with the rest of the nodes. + +#### Bringing Up the Remaining Nodes + +1. On each of the remaining nodes, change `nodename` in `riak.conf`, or `-name` in `vm.args` as described above. + +2. Move aside the ring directory. As in [Multi-Node Clusters](#multi-node-clusters), we will save this ring directory as a backup until were finished. + +3. Start each node. They will start as if they are each a member of their own cluster, but will retain their restored data. + +4. Join each node to our first node using `riak-admin cluster join riak@192.168.17.11`. + +5. Force replace each node with its old node name. For example, `riak-admin cluster force-replace riak@10.1.42.12 riak@192.168.17.12`. + +6. Once the above is complete for each node, run `riak-admin cluster plan` on any node. The output should look similar to below: + + ```bash + =============================== Staged Changes ================================ + Action Details(s) + ------------------------------------------------------------------------------- + force-replace 'riak@10.1.42.12' with 'riak@192.168.17.12' + force-replace 'riak@10.1.42.13' with 'riak@192.168.17.13' + force-replace 'riak@10.1.42.14' with 'riak@192.168.17.14' + force-replace 'riak@10.1.42.15' with 'riak@192.168.17.15' + join 'riak@192.168.17.12' + join 'riak@192.168.17.13' + join 'riak@192.168.17.14' + join 'riak@192.168.17.15' + ------------------------------------------------------------------------------- + + WARNING: All of 'riak@10.1.42.12' replicas will be lost + WARNING: All of 'riak@10.1.42.13' replicas will be lost + WARNING: All of 'riak@10.1.42.14' replicas will be lost + WARNING: All of 'riak@10.1.42.15' replicas will be lost + + NOTE: Applying these changes will result in 1 cluster transition + + ############################################################################### + After cluster transition 1/1 + ############################################################################### + + ================================= Membership ================================== + Status Ring Pending Node + ------------------------------------------------------------------------------- + valid 20.3% -- 'riak@192.168.17.11' + valid 20.3% -- 'riak@192.168.17.12' + valid 20.3% -- 'riak@192.168.17.13' + valid 20.3% -- 'riak@192.168.17.14' + valid 18.8% -- 'riak@192.168.17.15' + ------------------------------------------------------------------------------- + Valid:5 / Leaving:0 / Exiting:0 / Joining:0 / Down:0 + + Partitions reassigned from cluster changes: 51 + 13 reassigned from 'riak@10.1.42.12' to 'riak@192.168.17.12' + 13 reassigned from 'riak@10.1.42.13' to 'riak@192.168.17.13' + 13 reassigned from 'riak@10.1.42.14' to 'riak@192.168.17.14' + 12 reassigned from 'riak@10.1.42.15' to 'riak@192.168.17.15' + ``` + +7. If the above plan looks correct, commit the cluster changes with `riak-admin cluster commit`. + +8. Once the cluster transition has completed, all node names should be changed and be marked as valid in `riak-admin member-status` like below: + + ```bash + ================================= Membership ================================== + Status Ring Pending Node + ------------------------------------------------------------------------------- + valid 20.3% -- 'riak@192.168.17.11' + valid 20.3% -- 'riak@192.168.17.12' + valid 20.3% -- 'riak@192.168.17.13' + valid 20.3% -- 'riak@192.168.17.14' + valid 18.8% -- 'riak@192.168.17.15' + ------------------------------------------------------------------------------- + Valid:5 / Leaving:0 / Exiting:0 / Joining:0 / Down:0 + + ``` diff --git a/content/riak/kv/2.2.6/using/cluster-operations/handoff.md b/content/riak/kv/2.2.6/using/cluster-operations/handoff.md new file mode 100644 index 0000000000..6c5dd61491 --- /dev/null +++ b/content/riak/kv/2.2.6/using/cluster-operations/handoff.md @@ -0,0 +1,116 @@ +--- +title: "Enabling and Disabling Handoff" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Handoff" + identifier: "cluster_operations_handoff" + weight: 107 + parent: "managing_cluster_operations" +toc: true +aliases: + - /riak-docs/riak/2.2.6/ops/running/handoff + - /riak-docs/riak/kv/2.2.6/ops/running/handoff +--- + +Riak KV provides a command-line interface for enabling and disabling handoff on the fly, without needing to set your configuration and restart the node. To +enable handoff: + +```bash +riak-admin handoff enable <inbound|outbound|both> <nodename> +``` + +You must specify two things when enabling handoff: + +* whether you'd like to enable inbound handoff, outbound handoff, or + both +* the node to be targeted by the command (or all nodes) + +You can select a target node using either the `--node` or the `-n` flag. +You can select a direction by specifying `inbound`, `outbound`, or +`both`. The following equivalent commands would enable outbound handoff +on the node `riak3@100.0.0.1`: + +```bash +riak-admin handoff enable outbound --node riak3@100.0.0.1 +riak-admin handoff enable outbound -n riak3@100.0.0.1 +``` + +These two equivalent commands would enable inbound handoff on the node +`riak5@100.0.0.1`: + +```bash +riak-admin handoff enable inbound --node riak5@100.0.0.1 +riak-admin handoff enable inbound -n riak5@127.0.0.1 +``` + +Alternatively, you can enable handoff on all nodes at the same time +using either the `-a` or `--all` flag. This command would enable both +inbound and outbound handoff on all nodes: + +```bash +riak-admin handoff enable both --all +``` + +As for enabling handoff, the `riak-admin disable` command requires that +you specify both both a node or nodes to be targeted by the command and +whether you'd like to disable inbound handoff, outbound handoff, or +both. The `disable` command works just like `enable`. This command +would disable all forms of handoff on all nodes, to give just one +example: + +```bash +riak-admin handoff disable both --all +``` + +## Other Command-line Tools + +In addition to enabling and disabling handoff, the +[`riak-admin`]({{<baseurl>}}riak/kv/2.2.6/using/admin/riak-admin/) interface enables you to +retrieve a summary of handoff-related activity and other information. + +### summary + +The `summary` command provides high-level information about active +handoffs in a cluster. + +```bash +riak-admin handoff summary +``` + +This will return a table that will provide the following information +about each node in your cluster: + +Header | Description +:------|:----------- +`Node` | The name of the node +`Total` | Total number of active transfers throughout the entire cluster +`Ownership` | Total number of ownership exchanges +`Resize` | Total handoffs related to ring resizing operations (This should always be 0, as the Resize Ring feature has been deprecated) +`Hinted` | Total number of [hinted handoffs](../../reference/handoff#types-of-handoff) +`Repair` | Total repair-related handoffs. More information can be found [here](https://github.com/basho/riak_core/commit/036e409eb83903315dd43a37c7a93c9256863807). + +### details + +This command provides information only about active transfers. + +```bash +riak-admin handoff details +``` + +If no transfers are currently underway, this command will output `No +ongoing transfers`. Otherwise, you will something like this: + +### config + +This command displays the values for handoff-specific [configurable parameters]({{<baseurl>}}riak/kv/2.2.6/configuring/reference/#intra-cluster-handoff) on each node in +the cluster, including: + +* `transfer_limit` +* `handoff.outbound` +* `handoff.inbound` +* `handoff.port` + +Descriptions of those parameters can be found in the sections above. diff --git a/content/riak/kv/2.2.6/using/cluster-operations/inspecting-node.md b/content/riak/kv/2.2.6/using/cluster-operations/inspecting-node.md new file mode 100644 index 0000000000..3548336262 --- /dev/null +++ b/content/riak/kv/2.2.6/using/cluster-operations/inspecting-node.md @@ -0,0 +1,492 @@ +--- +title: "Inspecting a Node" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Inspecting a Node" + identifier: "cluster_operations_inspecting_node" + weight: 103 + parent: "managing_cluster_operations" +toc: true +aliases: + - /riak-docs/riak/2.2.6/ops/running/nodes/inspecting + - /riak-docs/riak/kv/2.2.6/ops/running/nodes/inspecting +--- + +When inspection of a Riak node to gather metrics on performance or +potential issues is desired, a number of tools are available to help, +and are either included with Riak itself or made available through the +Riak community. + +This guide provides starting points and details on some of the available +tools for inspecting a Riak node. + +## riak-admin status + +`riak-admin status` is a subcommand of the `riak-admin` command that is +included with every installation of Riak. The `status` subcommand +provides data related to the current operating status for a node. The +output of `riak-admin status` is categorized and detailed below. + +Please note, for some counters, such as `node_get_fsm_objsize`, a +minimum of 5 transactions is required for statistics to be generated. + +#### Performance + +We recommended checking stats every 90-120 seconds for best performance. + +Repeated runs of the `riak-admin status` command should not have a +negative performance impact as the statistics are cached internally in +Riak. + +### Active Stats + +Active Stats represent current activity on the node. + +Stat | Description +------------------------|--------------------------------------------------- +`pbc_active` | Number of active Protocol Buffers connections +`node_get_fsm_active` | Number of active GET FSMs +`node_put_fsm_active` | Number of active PUT FSMs +`index_fsm_active` | Number of active Secondary Index FSMs +`list_fsm_active` | Number of active Keylisting FSMs +`node_get_fsm_rejected` | Number of GET FSMs actively being rejected by Sidejob's overload protection +`node_put_fsm_rejected` | Number of PUT FSMs actively being rejected by Sidejob's overload protection + +### Average Stats + +Average Stats represent an average calculated as (total occurrences / +number of samples) since this node was started. In the below stats the +sample time is 1s, giving us a per-second average. Currently, the only +Average Stats are reported by Sidejob - an Erlang library that +implements a parallel, capacity-limited request pool. + +Stat | Description +------------------------|--------------------------------------------------- +`node_get_fsm_in_rate` | Average number of GET FSMs enqueued by Sidejob +`node_get_fsm_out_rate` | Average number of GET FSMs dequeued by Sidejob +`node_put_fsm_in_rate` | Average number of PUT FSMs enqueued by Sidejob +`node_put_fsm_out_rate` | Average number of PUT FSMs dequeued by Sidejob + +### One-Minute Stats + +One-Minute Stats represent the number of times a particular activity has +occurred within the last minute on this node. + +#### General One-Minute Stats + +Stat | Description +--------------------------------------|--------------------------------------------------- +`node_gets` | Number of GETs coordinated by this node, including GETs to non-local vnodes in the last minute +`node_puts` | Number of PUTs coordinated by this node, where a PUT is sent to a local vnode in the last minute +`vnode_gets` | Number of GET operations coordinated by local vnodes on this node in the last minute +`vnode_puts` | Number of PUT operations coordinated by local vnodes on this node in the last minute +`vnode_index_refreshes` | Number of secondary indexes refreshed on this node during secondary index anti-entropy in the last minute +`vnode_index_reads` | Number of local replicas participating in secondary index reads in the last minute +`vnode_index_writes` | Number of local replicas participating in secondary index writes in the last minute +`vnode_index_writes_postings` | Number of individual secondary index values written in the last minute +`vnode_index_deletes` | Number of local replicas participating in secondary index deletes in the last minute +`vnode_index_deletes_postings` | Number of individual secondary index values deleted in the last minute +`pbc_connects` | Number of Protocol Buffers connections made in the last minute +`node_get_fsm_active_60s` | Number of GET FSMs active in the last minute +`node_put_fsm_active_60s` | Number of PUT FSMs active in the last minute +`node_get_fsm_rejected_60s` | Number of GET FSMs rejected by Sidejob's overload protection in the last minute +`node_put_fsm_rejected_60s` | Number of PUT FSMs rejected by Sidejob's overload protection in the last minute +`index_fsm_create` | Number of Secondary Index query FSMs created in the last minute +`index_fsm_create_error` | Number of Secondary Index query FSM creation errors in the last minute +`list_fsm_create` | Number of Keylisting FSMs created in the last minute +`list_fsm_create_error` | Number of Keylisting FSM creation errors in the last minute +`read_repairs` | Number of read repair operations this node has coordinated in the last minute +`read_repairs_primary_outofdate_one` | Number of read repair operations performed on primary vnodes in the last minute due to stale replicas +`read_repairs_primary_notfound_one` | Number of read repair operations performed on primary vnodes in the last minute due to missing replicas +`read_repairs_fallback_outofdate_one` | Number of read repair operations performed on fallback vnodes in the last minute due to stale replicas +`read_repairs_fallback_notfound_one` | Number of read repair operations performed on fallback vnodes in the last minute due to missing replicas + +#### FSM Time + +FSM Time Stats represent the amount of time in microseconds required to +traverse the GET or PUT Finite State Machine code, offering a picture of +general node health. From your application's perspective, FSM Time +effectively represents experienced latency. Mean, Median, and 95th-, +99th-, and 100th-percentile (Max) counters are displayed. These are +one-minute stats. + +Stat | Description +---------------------------|--------------------------------------------------- +`node_get_fsm_time_mean` | Mean time between reception of client GET request and subsequent response to client +`node_get_fsm_time_median` | Median time between reception of client GET request and subsequent response to client +`node_get_fsm_time_95` | 95th percentile time between reception of client GET request and subsequent response to client +`node_get_fsm_time_99` | 99th percentile time between reception of client GET request and subsequent response to client +`node_get_fsm_time_100` | 100th percentile time between reception of client GET request and subsequent response to client +`node_put_fsm_time_mean` | Mean time between reception of client PUT request and subsequent response to client +`node_put_fsm_time_median` | Median time between reception of client PUT request and subsequent response to client +`node_put_fsm_time_95` | 95th percentile time between reception of client PUT request and subsequent response to client +`node_put_fsm_time_99` | 99th percentile time between reception of client PUT request and subsequent response to client +`node_put_fsm_time_100` | 100th percentile time between reception of client PUT request and subsequent response to client + +#### GET FSM Siblings + +GET FSM Sibling Stats offer a count of the number of siblings +encountered by this node on the occasion of a GET request. These are +one-minute stats. + +Stat | Description +-------------------------------|--------------------------------------------------- +`node_get_fsm_siblings_mean` | Mean number of siblings encountered during all GET operations by this node within the last minute +`node_get_fsm_siblings_median` | Median number of siblings encountered during all GET operations by this node within the last minute +`node_get_fsm_siblings_95` | 95th percentile of siblings encountered during all GET operations by this node within the last minute +`node_get_fsm_siblings_99` | 99th percentile of siblings encountered during all GET operations by this node within the last minute +`node_get_fsm_siblings_100` | 100th percentile of siblings encountered during all GET operations by this node within the last minute + +#### GET FSM Objsize + +GET FSM Objsize Stats represent a view of the sizes of objects flowing +through this node's GET FSMs. The size of an object is obtained by +summing the length of the bucket name, key, serialized vector clock, +value, and serialized metadata of each sibling. GET FSM Objsize and GET +FSM Siblings are inextricably linked. These are one-minute stats. + +Stat | Description +------------------------------|--------------------------------------------------- +`node_get_fsm_objsize_mean` | Mean object size (bytes) encountered by this node within the last minute +`node_get_fsm_objsize_median` | Median object size (bytes) encountered by this node within the last minute +`node_get_fsm_objsize_95` | 95th percentile object size (bytes) encountered by this node within the last minute +`node_get_fsm_objsize_99` | 99th percentile object size (bytes) encountered by this node within the last minute +`node_get_fsm_objsize_100` | 100th percentile object size (bytes) encountered by this node within the last minute + +### Total Stats + +Total Stats represent the total number of times a particular activity +has occurred since this node was started. + +Stat | Description +---------------------------------------|--------------------------------------------------- +`node_gets_total` | Total number of GETs coordinated by this node, including GETs to non-local vnodes +`node_puts_total` | Total number of PUTs coordinated by this node, including PUTs to non-local vnodes +`vnode_gets_total` | Total number of GETs coordinated by local vnodes +`vnode_puts_total` | Total number of PUTS coordinated by local vnodes +`read_repairs_total` | Total number of Read Repairs this node has coordinated +`coord_redirs_total` | Total number of requests this node has redirected to other nodes for coordination +`vnode_index_refreshes_total` | Total number of indexes refreshed during secondary index anti-entropy +`vnode_index_reads_total` | Total number of local replicas participating in secondary index reads +`vnode_index_writes_total` | Total number of local replicas participating in secondary index writes +`vnode_index_writes_postings_total` | Total number of individual secondary index values written +`vnode_index_deletes_total` | Total number of local replicas participating in secondary index deletes +`vnode_index_deletes_postings_total` | Total number of individual secondary index values deleted +`pbc_connects_total` | Total number of Protocol Buffers connections made +`precommit_fail` | Total number of pre-commit hook failures +`postcommit_fail` | Total number of post-commit hook failures +`node_get_fsm_rejected_total` | Total number of GET FSMs rejected by Sidejob's overload protection +`node_put_fsm_rejected_total` | Total number of PUT FSMs rejected by Sidejob's overload protection +`read_repairs_primary_outofdate_count` | Total number of read repair operations performed on primary vnodes due to stale replicas +`read_repairs_primary_notfound_count` | Total number of read repair operations performed on primary vnodes due to missing replicas +`read_repairs_fallback_outofdate_count`| Total number of read repair operations performed on fallback vnodes due to stale replicas +`read_repairs_fallback_notfound_count` | Total number of read repair operations performed on fallback vnodes due to missing replicas + +### Timestamps + +Some of the Erlang applications that Riak is comprised of contribute +statistics to `riak-admin status`. The below timestamps record, in +Epoch time, the last time statistics for that application were +generated. + +Stat | Description +--------------------|--------------------------------------------------- +`riak_kv_stat_ts` | The last time Riak KV stats were generated. +`riak_pipe_stat_ts` | The last time Riak Pipe stats were generated. + +### Ring + +General ring information is reported in `riak-admin status`. + +Stat | Description +---------------------|--------------------------------------------------- +`ring_members` | List of nodes that are members of the ring +`ring_num_partitions`| The number of partitions in the ring +`ring_ownership` | List of all nodes in the ring and their associated partition ownership +`ring_creation_size` | Ring size this cluster was created with + +### CPU and Memory + +CPU statistics are taken directly from Erlang’s cpu_sup module. +Documentation for which can be found at [ErlDocs: +cpu_sup](http://erlang.org/doc/man/cpu_sup.html). + +Stat | Description +-------------|--------------------------------------------------- +`cpu_nprocs` | Number of operating system processes +`cpu_avg1` | The average number of active processes for the last 1 minute (equivalent to top(1) command’s load average when divided by 256()) +`cpu_avg5` | The average number of active processes for the last 5 minutes (equivalent to top(1) command’s load average when divided by 256()) +`cpu_avg15` | The average number of active processes for the last 15 minutes (equivalent to top(1) command’s load average when divided by 256()) + +Memory statistics are taken directly from the Erlang virtual machine. +Documentation for which can be found at [ErlDocs: +Memory](http://erlang.org/doc/man/erlang.html#memory-0#memory/0). + +Stat | Description +------------------------|--------------------------------------------------- +`memory_total` | Total allocated memory (sum of processes and system) +`memory_processes` | Total amount of memory allocated for Erlang processes +`memory_processes_used` | Total amount of memory used by Erlang processes +`memory_system` | Total allocated memory that is not directly related to an Erlang process +`memory_atom` | Total amount of memory currently allocated for atom storage +`memory_atom_used` | Total amount of memory currently used for atom storage +`memory_binary` | Total amount of memory used for binaries +`memory_code` | Total amount of memory allocated for Erlang code +`memory_ets` | Total memory allocated for Erlang Term Storage +`mem_total` | Total available system memory +`mem_allocated` | Total memory allocated for this node + +### Erlang VM + +The below statistics describe properties of the Erlang VM. + +Stat | Description +--------------------------|--------------------------------------------------- +`nodename` | The name this node uses to identify itself +`connected_nodes` | A list of the nodes that this node is aware of at this time +`sys_driver_version` | String representing the Erlang driver version in use by the runtime system +`sys_global_heaps_size` | Current size of the shared global heap +`sys_heap_type` | String representing the heap type in use (one of private, shared, hybrid) +`sys_logical_processors` | Number of logical processors available on the system +`sys_otp_release` | Erlang OTP release version in use on the node +`sys_process_count` | Number of processes currently running in the Erlang VM +`sys_smp_support` | Boolean value representing whether symmetric multi-processing (SMP) is available +`sys_system_version` | Detailed Erlang version information +`sys_system_architecture` | The node operating system and hardware architecture +`sys_threads_enabled` | Boolean value representing whether threads are enabled +`sys_thread_pool_size` | Number of threads in the asynchronous thread pool +`sys_wordsize` | Size of Erlang term words in bytes as an integer, for examples, on 32-bit architectures 4 is returned and on 64-bit architectures 8 is returned + +### Miscellaneous Information + +Miscellaneous Information provide additional details particular to this +node. + +Stat | Description +---------------------------|--------------------------------------------------- +`leveldb_read_block_error` | The number of LevelDB read block errors. Will read as undefined if LevelDB is not being used. +`disk` | Information about the disk, taken from Erlang's disksup module. Reported as [{"ID",KBytes_Used,Percent_Util}]. +`storage_backend` | The storage backend currently in use. + +### Pipeline Metrics + +The following metrics from from riak_pipe are generated during MapReduce +operations. + +Stat | Description +--------------------------------|--------------------------------------------------- +`pipeline_active` | The number of pipelines active in the last 60 seconds +`pipeline_create_count` | The total number of pipelines created since the node was started +`pipeline_create_error_count` | The total number of pipeline creation errors since the node was started +`pipeline_create_error_one` | The number of pipeline creation errors in the last 60 seconds +`pipeline_create_one` | The number of pipelines created in the last 60 seconds + +### Application and Subsystem Versions + +The specific version of each Erlang application and subsystem which +makes up a Riak node is present in the `riak-admin status` output. Each +application is linked below next to it's version identifier. + +Stat | Description +------------------------|--------------------------------------------------- +`erlydtl_version` | [ErlyDTL](http://github.com/erlydtl/erlydtl) +`riak_control_version` | [Riak Control](http://github.com/basho/riak_control) +`cluster_info_version` | [Cluster Information](http://github.com/basho/cluster_info) +`riak_search_version` | [Riak Search](http://github.com/basho/riak_search) +`merge_index_version` | [Merge Index](http://github.com/basho/merge_index) +`riak_kv_version` | [Riak KV](http://github.com/basho/riak_kv) +`sidejob_version` | [Sidejob](http://github.com/basho/sidejob) +`riak_api_version` | [Riak API](http://github.com/basho/riak_api) +`riak_pipe_version` | [Riak Pipe](http://github.com/basho/riak_pipe) +`riak_core_version` | [Riak Core](http://github.com/basho/riak_core) +`bitcask_version` | [Bitcask](http://github.com/basho/bitcask) +`basho_stats_version` | [Basho Stats](http://github.com/basho/basho_stats) + `webmachine_version` | [Webmachine](http://github.com/basho/webmachine) +`mochiweb_version` | [MochiWeb](http://github.com/basho/mochiweb) +`inets_version` | [inets](http://erlang.org/doc/apps/inets/) +`erlang_js_version` | [Erlang JS](http://github.com/basho/erlang_js) +`runtime_tools_version` | [Erlang Runtime Tools](http://erlang.org/doc/apps/runtime_tools/) +`os_mon_version` | [Erlang Operating System Monitor](http://erlang.org/doc/apps/os_mon/) +`riak_sysmon_version` | [Riak System Monitor](http://github.com/basho/riak_sysmon) +`ssl_version` | [Erlang Secure Sockets Layer (SSL)](http://erlang.org/doc/apps/ssl/) +`public_key_version` | [Erlang Public Key](http://erlang.org/doc/apps/public_key/) +`crypto_version` | [Erlang crypto](http://erlang.org/doc/apps/crypto/) +`sasl_version` | [SASL](http://erlang.org/doc/apps/sasl/) +`lager_version` | [Lager](http://github.com/DeadZen/lager) +`goldrush_version` | [Goldrush](http://github.com/DeadZen/goldrush) +`compiler_version` | [Erlang Compiler](http://erlang.org/doc/apps/compiler/) +`syntax_tools_version` | [Erlang Syntax Tools](http://www.erlang.org/doc/apps/syntax_tools/) +`stdlib_version` | [Standard Library](http://erlang.org/doc/apps/stdlib/) +`kernel_version` | [Kernel](http://erlang.org/doc/apps/kernel/) + +### Riak Search Statistics + +The following statistics related to Riak Search message queues are +available. + +Stat | Description +-----------------------------|--------------------------------------------------- +`riak_search_vnodeq_max` | Maximum number of unprocessed messages all virtual node (vnode) message queues in the Riak Search subsystem have received on this node in the last minute +`riak_search_vnodeq_mean` | Mean number of unprocessed messages all vnode message queues in the Riak Search subsystem have received on this node in the last minute +`riak_search_vnodeq_median` | Median number of unprocessed messages all vnode message queues in the Riak Search subsystem have received on this node in the last minute +`riak_search_vnodeq_min` | Minimum number of unprocessed messages all vnode message queues in the Riak Search subsystem have received on this node in the last minute +`riak_search_vnodeq_total` | Total number of unprocessed messages all vnode message queues in the Riak Search subsystem have received on this node since it was started +`riak_search_vnodes_running` | Total number of vnodes currently running in the Riak Search subsystem + +Note that under ideal operation and with the exception of +`riak_search_vnodes_running` these statistics should contain low values +(e.g., 0-10). Presence of higher values could be indicative of an issue. + +## `riak-debug` + +The `riak-debug` command is used to identify and diagnose common problems with your Riak KV nodes. + +`riak-debug` also runs `riak-admin diag`, which runs a small suite of diagnostic checks against a Riak KV node to discover common problems. It often offers recommendations about how to resolve those problems as well. + +{{% note title="Warning about `riak-debug` and `riak-admin diag` usage" %}} +The `riak-debug` and `riak-admin diag` commands should only be used after a new installation or configuration change. It should not be used as part of regular monitoring. Overuse of `riak-debug` or `riak-admin diag` can eventually cause the node to crash from atom table exhaustion. +{{% /note %}} + +## Strong Consistency Stats + +Riak tabulates a variety of stats related to Riak's optional [strong consistency](../../reference/strong-consistency) feature. The table below lists those stats. + +### GET-related stats + +Stat | Description +:----|:----------- +`consistent_gets` | Number of strongly consistent GETs coordinated by this node in the last minute +`consistent_gets_total` | Total number of strongly consistent GETs coordinated by this node +`consistent_get_objsize_mean` | Mean object size (bytes) for strongly consistent GETs on this node in the last minute +`consistent_get_objsize_median` | Median object size (bytes) for strongly consistent GETs on this node in the last minute +`consistent_get_objsize_95` | 95th-percentile object size (bytes) for strongly consistent GETs on this node in the last minute +`consistent_get_objsize_99` | 99th-percentile object size (bytes) for strongly consistent GETs on this node in the last minute +`consistent_get_objsize_100` | 100th-percentile object size (bytes) for strongly consistent GETs on this node in the last minute +`consistent_get_time_mean` | Mean time between reception of client GETs to strongly consistent keys and subsequent response +`consistent_get_time_median` | Median time between reception of client GETs to strongly consistent keys and subsequent response +`consistent_get_time_95` | 95th-percentile time between reception of client GETs to strongly consistent keys and subsequent response +`consistent_get_time_99` | 99th-percentile time between reception of client GETs to strongly consistent keys and subsequent response +`consistent_get_time_100` | 100th-percentile time between reception of client GETs to strongly consistent keys and subsequent response + +### PUT-related stats + +Stat | Description +:----|:----------- +`consistent_puts` | Number of strongly consistent PUTs coordinated by this node in the last minute +`consistent_puts_total` | Total number of strongly consistent PUTs coordinated by this node +`consistent_put_objsize_mean` | Mean object size (bytes) for strongly consistent PUTs on this node in the last minute +`consistent_put_objsize_median` | Median object size (bytes) for strongly consistent PUTs on this node in the last minute +`consistent_put_objsize_95` | 95th-percentile object size (bytes) for strongly consistent PUTs on this node in the last minute +`consistent_put_objsize_99` | 99th-percentile object size (bytes) for strongly consistent PUTs on this node in the last minute +`consistent_put_objsize_100` | 100th-percentile object size (bytes) for strongly consistent PUTs on this node in the last minute +`consistent_put_time_mean` | Mean time between reception of client PUTs to strongly consistent keys and subsequent response +`consistent_put_time_median` | Median time between reception of client PUTs to strongly consistent keys and subsequent response +`consistent_put_time_95` | 95th-percentile time between reception of client PUTs to strongly consistent keys and subsequent response +`consistent_put_time_99` | 99th-percentile time between reception of client PUTs to strongly consistent keys and subsequent response +`consistent_put_time_100` | 100th-percentile time between reception of client PUTs to strongly consistent keys and subsequent response +## riak-admin diag + +Running `riak-admin diag` by itself will perform a check of all of the +data partitions in your cluster. It will return a listing of partitions +that have been checked, each of which looks something like this: + +``` +{1392993748081016843912887106182707253109560705024, % the partition checked + 'dev-rel@127.0.0.1'}, % that partition's nodename +``` + +At the end of that (potentially very long) listing of checked +partitions, it will print notices, warnings, and other pieces of +information about issues that it has found, including date/time, message +type, and a detailed description. Here's an example: + +``` +15:34:52.736 [warning] Riak crashed at Wed, 07 Dec 2011 21:47:50 GMT, leaving crash dump in /srv/riak/log/erl_crash.dump. Please inspect or remove the file. +15:34:52.736 [notice] Data directory /srv/riak/data/bitcask is not mounted with 'noatime'. Please remount its disk with the 'noatime' flag to improve performance. +``` + +Messages bear the following types (derived from +[syslog](http://en.wikipedia.org/wiki/Syslog) security levels): + +* `debug` +* `info` +* `notice` +* `warning` +* `error` +* `critical` +* `alert` +* `emergency` + +#### Command flags + +Attaching the `--help` flag will return a list of flags and commands +that can be used with Riaknostic: + +``` +Usage: riak-admin diag [-d <level>] [-l] [-h] [--export] [check_name ...] + +-h, --help Display help/usage dialogue +-d, --level Minimum message severity level (default: notice) +-l, --list Describe available diagnostic tasks +--export Package system info in '/export.zip' +check_name A specific check to run +``` + +Running `riak-admin diag` with the `--list` flag will return a list of +available diagnostic checks. The following checks are available: + +Check | Description +:-----|:----------- +`disk` | Data directory permissions and atime +`dumps` | Find crash dumps +`memory_use` | Measure memory usage +`nodes_connected` | Cluster node liveness +`ring_membership` | Cluster membership validity +`ring_preflists` | Check if the ring satisfies `n_val` +`ring_size` | Check if the ring size valid +`search` | Check whether Riak Search is enabled on all nodes + +The `--level` flag enables you to specify the log level and thus to +filter messages based on type. You can pass in any of the message types +listed above (`debug`, `info`, etc.). + +The `--level` flag can be used when running `riak-admin diag` with or +without specifying a diagnostic check. + +#### Contributing + +Do you have an idea that would help us improve Riaknostic? If so, fork +the [GitHub repository](https://github.com/basho/riaknostic) and send us +a pull request with your changes. The code is documented with +[edoc](http://riaknostic.basho.com/edoc/index.html), so give the API +Docs a read before you contribute. + +If you want to run the Riaknostic script while developing and you don't +have it hooked up to your local Riak installation, you can invoke it +directly like so: + +```bash +./riaknostic --etc ~/code/riak/rel/riak/etc --base ~/code/riak/rel/riak --user `whoami` [other options] +``` + +Those extra options are usually assigned by the `riak-admin` script for +you, but here's how to set them: + +* `--etc` --- The location of your Riak configuration directory (usually + `/etc`). In the example above, configuration is in the generated + directory of a source checkout of Riak. +* `--base` --- The "base" directory of Riak, usually the root of the + generated directory or `/usr/lib/riak` on Linux. Scan the + `riak-admin` script for how the `RUNNER_BASE_DIR` variable is + assigned on your platform. +* `--user` --- The user/UID as which the Riak node runs. In a source + checkout, it's the current user; on most systems, it's `riak`. + +## Related Resources + +* [The riak-admin configuration management tool](../../admin/riak-admin/) +* [Riaknostic](http://riaknostic.basho.com/) +* [HTTP API Status](../../../developing/api/http/status/) diff --git a/content/riak/kv/2.2.6/using/cluster-operations/load-balancing.md b/content/riak/kv/2.2.6/using/cluster-operations/load-balancing.md new file mode 100644 index 0000000000..94e906e1db --- /dev/null +++ b/content/riak/kv/2.2.6/using/cluster-operations/load-balancing.md @@ -0,0 +1,16 @@ +--- +draft: true +title: "Load Balancing" +description: "" +project: "riak_kv" +project_version: "2.2.6" +#menu: +# riak_kv-2.2.6: +# name: "Load Balancing" +# identifier: "cluster_operations_load_balancing" +# weight: 111 +# parent: "managing_cluster_operations" +toc: true +--- + +**TODO: Add content (not sure where this exists in docs)** diff --git a/content/riak/kv/2.2.6/using/cluster-operations/logging.md b/content/riak/kv/2.2.6/using/cluster-operations/logging.md new file mode 100644 index 0000000000..4abde19556 --- /dev/null +++ b/content/riak/kv/2.2.6/using/cluster-operations/logging.md @@ -0,0 +1,42 @@ +--- +title: "Enabling and Disabling Debug Logging" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Logging" + identifier: "cluster_operations_logging" + weight: 105 + parent: "managing_cluster_operations" +toc: true +--- + +If you'd like to enable debug logging on the current node, i.e. set the +console log level to `debug`, you can do so without restarting the node +by accessing the Erlang console directly using the [`riak attach`]({{<baseurl>}}riak/kv/2.2.6/using/admin/riak-cli/#attach) command. Once you run this command and drop into the console, enter the following: + +```erlang +lager:set_loglevel(lager_file_backend, "/var/log/riak/console.log", debug). +``` + +You should replace the file location above (`/var/log/riak/console.log`) +with your platform-specific location, e.g. `./log/console.log` for a +source installation. This location is specified by the +`log.console.file` parameter explained above. + +If you'd like to enable debug logging on _all_ nodes instead of just one +node, you can enter the Erlang console of any running by running `riak +attach` and enter the following: + +```erlang +rp(rpc:multicall(lager, set_loglevel, [lager_file_backend, "/var/log/riak/console.log", debug])). +``` + +As before, use the appropriate log file location for your cluster. + +At any time, you can set the log level back to `info`: + +```erlang +rp(rpc:multicall(lager, set_loglevel, [lager_file_backend, "/var/log/riak/console.log", info])). +``` diff --git a/content/riak/kv/2.2.6/using/cluster-operations/replacing-node.md b/content/riak/kv/2.2.6/using/cluster-operations/replacing-node.md new file mode 100644 index 0000000000..72c6bfc9cf --- /dev/null +++ b/content/riak/kv/2.2.6/using/cluster-operations/replacing-node.md @@ -0,0 +1,95 @@ +--- +title: "Replacing a Node" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Replacing a Node" + identifier: "cluster_operations_replace_node" + weight: 102 + parent: "managing_cluster_operations" +toc: true +--- + +At some point, for various reasons, you might need to replace a node in +your Riak cluster (which is different from [recovering a failed node]({{<baseurl>}}riak/kv/2.2.6/using/repair-recovery)). Here is the recommended way to go +about replacing a node. + +1. Back up your data directory on the node in question. In this example +scenario, we'll call the node `riak4`: + + ```bash + sudo tar -czf riak_backup.tar.gz /var/lib/riak /etc/riak + ``` + + If you have any unforeseen issues at any point in the node + replacement process, you can restore the node's data from this + backup. + +2. Download and install Riak on the new node you wish to bring into the +cluster and have it replace the `riak4` node. We'll call the new node +`riak7` for the purpose of this example. + +3. Start the new `riak7` node with [`riak start`]({{<baseurl>}}riak/kv/2.2.6/using/admin/riak-cli/#start): + + ```bash + riak start + ``` + +4. Plan the join of the new `riak7` node to an existing node already +participating in the cluster; for example `riak0` with the [`riak-admin cluster join`]({{<baseurl>}}riak/kv/2.2.6/using/admin/riak-admin/#cluster) command executed on the new `riak7` node: + + ```bash + riak-admin cluster join riak0 + ``` + +5. Plan the replacement of the existing `riak4` node with the new +`riak7` node using the [`riak-admin cluster replace`]({{<baseurl>}}riak/kv/2.2.6/using/admin/riak-admin/#cluster) command: + + ```bash + riak-admin cluster replace riak4 riak7 + ``` + + <div class=info> + <div class=title>Single Nodes</div> + If a node is started singly using default settings (as, for example, + you might do when you are building your first test environment), you + will need to remove the ring files from the data directory after you + edit `/etc/vm.args`. `riak-admin cluster replace` will not work as + the node has not been joined to a cluster. + </div> + +6. Examine the proposed cluster changes with the [`riak-admin cluster plan`]({{<baseurl>}}riak/kv/2.2.6/using/admin/riak-admin/#cluster) command executed on the new +`riak7` node: + + ```bash + riak-admin cluster plan + ``` + +7. If the changes are correct, you can commit them with the +[`riak-admin cluster commit`]({{<baseurl>}}riak/kv/2.2.6/using/admin/riak-admin/#cluster) command: + + ```bash + riak-admin cluster commit + ``` + + If you need to clear the proposed plan and start over, use [`riak-admin cluster clear`]({{<baseurl>}}riak/kv/2.2.6/using/admin/riak-admin/#cluster): + + ```bash + riak-admin cluster clear + ``` + +Once you have successfully replaced the node, it should begin leaving +the cluster. You can check on ring readiness after replacing the node +with the [`riak-admin ringready`]({{<baseurl>}}riak/kv/2.2.6/using/admin/riak-admin/#ringready) +and [`riak-admin member-status`]({{<baseurl>}}riak/kv/2.2.6/using/admin/riak-admin/#member-status) +commands. + +{{% note title="Ring Settling" %}} +You'll need to make sure that no other ring changes occur between the time +when you start the new node and the ring settles with the new IP info. + +The ring is considered settled when the new node reports `true` when you run +the `riak-admin ringready` command. +{{% /note %}} diff --git a/content/riak/kv/2.2.6/using/cluster-operations/secondary-indexes.md b/content/riak/kv/2.2.6/using/cluster-operations/secondary-indexes.md new file mode 100644 index 0000000000..7b136a1cd9 --- /dev/null +++ b/content/riak/kv/2.2.6/using/cluster-operations/secondary-indexes.md @@ -0,0 +1,80 @@ +--- +draft: true +title: "Secondary Indexes" +description: "" +project: "riak_kv" +project_version: "2.2.6" +#menu: +# riak_kv-2.2.6: +# name: "Secondary Indexes" +# identifier: "cluster_operations_2i" +# weight: 109 +# parent: "managing_cluster_operations" +toc: true +--- + +## Hanc capellae + +Lorem markdownum Byblida. Modo **etiam** litora mittat vellera infelix caeli. +Studiosius forte, potuit pectore. Puer undas dignior iam turpe sorores abesse. +Deae Saturnia levius viribus membra. + +## Iussorum ad fronti rutilasque tenuit cursu quae + +Nostros vovistis artes. **Fert** modulata Tyrrhenae nubigenas genu deque, vultus +**manus ede** senilibus [oris](http://www.youtube.com/watch?v=MghiBW3r65M) +transcurrere quem rarissima. Viderunt nutu quod, tumidaque, mihi mihi sacer pia. +Summis rediit pavidus tersere et at prosiluit natus Phaethon noxa. Singultibus +oblita **foedabis** orsa. + +- Fecere aliis postquam inviti caliginis ab inque +- Voverat dividuae et tardus huc magna non +- Sex barba ipsaque Caucason corpora sono ecce +- Non esse +- Sibi atris regna licuit Antium carituraque nubes + +## Omni levare gelidumque minanti + +Omnis adeunt ossibus gravis, Venus pinuque capit, et sereno viros ignara *plena +incaluere* percussit mellaque, vertere arte. Ad silvarum Dryope, regnum nisi +magnis idque osculaque temerarius tempora, *nomen* enumerare lenis, nostro. Ac +mutabit [arma](http://www.thesecretofinvisibility.com/) operiri saxum ratione, +crudelior feram, est usu tamen quod, hasta. Equos **sonant et deum**. Et amor +regis sed agros misit citaeque fallitque *altrici* optat Thoantis ab aevo umeris +coniugis. + +## Troiana quoque + +Equo uni Stygias trahunt, interea, in tela labores lumina, nam *Aganippe +sanctique meum*; est. [Gente inimica +premeret](http://en.wikipedia.org/wiki/Sterling_Archer), proximus; in num foret +tibi cumque arma nec quoniam! Contribuere mollis, tu dum parem viscera, tamen +ante. Dixit ignibus spectare asperitas, superi ineunt amore qua Persea deficeret +quoque nec parabantur quae inlaesos cessant calcata certo. Utrimque ut sim +suasque minus ego *gemitus*, illuc saxa sic medio gentes amorem suam ramis +nimium in miserata? + +1. `In naribus aequos aberant` +2. Naturae murmura te rimas suarum vulnus quod +3. Socios leto loquor timide +4. Ergo sub +5. Patrias mihi consumite breve + +## Ruit huic movit luminibus excubias arma + +> Loco humo tecum gurgite timui. Peragant tu regia ut umbras premit condit. Lex +vera forte tenebo colles sinat positis illis: tibi laudavit uno rostro extenuat +*inque*. Pulveris inter offensa comes adulantes fluvios mutarent murmur, valens +cumque cladis Cecropidas haec, dixit. Lucus cognomine **Achilles**: pastor nec. + +1. Hic causam et dilecte nudae nec corpus +2. Cor Si nive +3. Petis equos perosa tu perterrita exitus non +4. Per et et ire geminos parte +5. Aqua coniunx cecidisse sonum + +``` +Nominis haec lacrimis orba gloria obstipuere tu Ceyx tepebat fetus me equorum +potero! Iampridem illi; deducit [reor orbem](http://heeeeeeeey.com/), comes, et +nec rubebant pietas, ipsa. +``` diff --git a/content/riak/kv/2.2.6/using/cluster-operations/strong-consistency.md b/content/riak/kv/2.2.6/using/cluster-operations/strong-consistency.md new file mode 100644 index 0000000000..a650a31cb5 --- /dev/null +++ b/content/riak/kv/2.2.6/using/cluster-operations/strong-consistency.md @@ -0,0 +1,71 @@ +--- +title: "Monitoring Strong Consistency" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Monitoring Strong Consistency" + identifier: "cluster_operations_strong_consistency" + weight: 110 + parent: "managing_cluster_operations" +toc: true +--- + +{{% note title="Please Note:" %}} +Riak KV's strong consistency is an experimental feature and may be removed +from the product in the future. Strong consistency is not commercially +supported or production-ready. Strong consistency is incompatible with +Multi-Datacenter Replication, Riak Search, Bitcask Expiration, LevelDB +Secondary Indexes, Riak Data Types and Commit Hooks. We do not recommend its +usage in any production environment. +{{% /note %}} + +## Monitoring Strong Consistency + +Riak provides a wide variety of data related to the current operating +status of a node. This data is available by running the [`riak-admin status`]({{<baseurl>}}riak/kv/2.2.6/using/admin/riak-admin/#riak-admin-status) command. That data now +includes statistics specific to strongly consistent operations. + +A full listing of these stats is available in [Inspecting a Node]({{<baseurl>}}riak/kv/2.2.6/using/cluster-operations/inspecting-node). +All strong consistency-related stats are prefixed with `consistent_`, +e.g. `consistent_gets`, `consistent_puts`, etc. Many of these stats are +so-called "one-minute stats," meaning that they reflect node activity in +the last minute. + +Strong consistency stats fall into two categories: GET-related and +PUT-related stats. + +### GET-related stats + +Stat | Description +:----|:----------- +`consistent_gets` | Number of strongly consistent GETs coordinated by this node in the last minute +`consistent_gets_total` | Total number of strongly consistent GETs coordinated by this node +`consistent_get_objsize_mean` | Mean object size for strongly consistent GETs on this node in the last minute +`consistent_get_objsize_median` | Median object size for strongly consistent GETs on this node in the last minute +`consistent_get_objsize_95` | 95th-percentile object size for strongly consistent GETs on this node in the last minute +`consistent_get_objsize_99` | 99th-percentile object size for strongly consistent GETs on this node in the last minute +`consistent_get_objsize_100` | 100th-percentile object size for strongly consistent GETs on this node in the last minute +`consistent_get_time_mean` | Mean time between reception of client GETs to strongly consistent keys and subsequent response +`consistent_get_time_median` | Median time between reception of client GETs to strongly consistent keys and subsequent response +`consistent_get_time_95` | 95th-percentile time between reception of client GETs to strongly consistent keys and subsequent response +`consistent_get_time_99` | 99th-percentile time between reception of client GETs to strongly consistent keys and subsequent response +`consistent_get_time_100` | 100th-percentile time between reception of client GETs to strongly consistent keys and subsequent response + +### PUT-related stats + +Stat | Description +:----|:----------- +`consistent_puts` | Number of strongly consistent PUTs coordinated by this node in the last minute +`consistent_puts_total` | Total number of strongly consistent PUTs coordinated by this node +`consistent_put_objsize_mean` | Mean object size for strongly consistent PUTs on this node in the last minute +`consistent_put_objsize_median` | Median object size for strongly consistent PUTs on this node in the last minute +`consistent_put_objsize_95` | 95th-percentile object size for strongly consistent PUTs on this node in the last minute +`consistent_put_objsize_99` | 99th-percentile object size for strongly consistent PUTs on this node in the last minute +`consistent_put_objsize_100` | 100th-percentile object size for strongly consistent PUTs on this node in the last minute +`consistent_put_time_mean` | Mean time between reception of client PUTs to strongly consistent keys and subsequent response +`consistent_put_time_median` | Median time between reception of client PUTs to strongly consistent keys and subsequent response +`consistent_put_time_95` | 95th-percentile time between reception of client PUTs to strongly consistent keys and subsequent response +`consistent_put_time_99` | 99th-percentile time between reception of client PUTs to strongly consistent keys and subsequent response +`consistent_put_time_100` | 100th-percentile time between reception of client PUTs to strongly consistent keys and subsequent response diff --git a/content/riak/kv/2.2.6/using/cluster-operations/v2-multi-datacenter.md b/content/riak/kv/2.2.6/using/cluster-operations/v2-multi-datacenter.md new file mode 100644 index 0000000000..9e89c084dc --- /dev/null +++ b/content/riak/kv/2.2.6/using/cluster-operations/v2-multi-datacenter.md @@ -0,0 +1,259 @@ +--- +title_supertext: "V2 Multi-Datacenter" +title: "Replication Operations" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "V2 Multi-Datacenter" + identifier: "cluster_operations_v2" + weight: 115 + parent: "managing_cluster_operations" +toc: true +commercial_offering: true +aliases: + - /riak-docs/riak/2.2.6/ops/mdc/v2/operations + - /riak-docs/riak/kv/2.2.6/ops/mdc/v2/operations +--- + +{{% note title="Deprecation Warning" %}} +v2 Multi-Datacenter Replication is deprecated and will be removed in a future version. Please use [v3]({{<baseurl>}}riak/kv/2.2.6/using/cluster-operations/v3-multi-datacenter/) instead. +{{% /note %}} + +Riak's Multi-Datacenter Replication system is largely +controlled by the `riak-repl` command. The sections below detail the +available subcommands. + +## add-listener + +Adds a listener (primary) to the given node, IP address, and port. + +```bash +riak-repl add-listener <nodename> <listen_ip> <port> +``` + +Below is an example usage: + +```bash +riak-repl add-listener riak@10.0.1.156 10.0.1.156 9010 +``` + +## add-nat-listener + +Adds a NAT-aware listener (primary) to the given node, IP address, port, +NAT IP, and NAT port. If a non-NAT listener already exists with the same +internal IP and port, it is "upgraded” to a NAT Listener. + +```bash +riak-repl add-nat-listener <nodename> <internal_ip> <internal_port> <nat_ip> <nat_port> +``` + +Below is an example usage: + +```bash +riak-repl add-nat-listener riak@10.0.1.156 10.0.1.156 9010 50.16.238.123 9010 +``` + +## del-listener + +Removes and shuts down a listener (primary) on the given node, IP +address, and port. + +```bash +riak-repl del-listener <nodename> <listen_ip> <port> +``` + +Below is an example usage: + +```bash +riak-repl del-listener riak@10.0.1.156 10.0.1.156 9010 +``` + +## add-site + +Adds a site (secondary) to the local node, connecting to the specified +listener. + +```bash +riak-repl add-site <ipaddr> <portnum> <sitename> +``` + +Below is an example usage: + +```bash +riak-repl add-site 10.0.1.156 9010 newyork +``` + +## del-site + +Removes a site (secondary) from the local node by name. + +```bash +riak-repl del-site <sitename> +``` + +Below is an example usage: + +```bash +riak-repl del-site newyork +``` + +## status + +Obtains status information about replication. Reports counts on how much +data has been transmitted, transfer rates, message queue lengths of +clients and servers, number of fullsync operations, and connection +status. This command only displays useful information on the leader +node. + +```bash +riak-repl status +``` + +## start-fullsync + +Manually initiates a fullsync operation with connected sites. + +```bash +riak-repl start-fullsync +``` + +## cancel-fullsync + +Cancels any fullsync operations in progress. If a partition is in +progress, synchronization will stop after that partition completes. +During cancellation, `riak-repl status` will show `cancelled` in the +status. + +```bash +riak-repl cancel-fullsync +``` + +## pause-fullsync + +Pauses any fullsync operations in progress. If a partition is in +progress, synchronization will pause after that partition completes. +While paused, `riak-repl status` will show `paused` in the status +information. Fullsync may be cancelled while paused. + +```bash +riak-repl pause-fullsync +``` + +## resume-fullsync + +Resumes any fullsync operations that were paused. If a fullsync +operation was running at the time of the pause, the next partition will +be synchronized. If not, it will wait until the next `start-fullsync` +command or `fullsync_interval`. + +```bash +riak-repl resume-fullsync +``` + +## riak-repl Status Output + +The following definitions describe the output of the `riak-repl status` +command. Please note that many of these statistics will only appear on +the current leader node, and that all counts will be reset to 0 upon +restarting Riak. + +### Client + +Field | Description +:-----|:----------- +`client_stats` | See <a href="http://docs.basho.com/riak/kv/2.2.6/using/reference/multi-datacenter/statistics/#client-statistics">Client Statistics</a> +`client_bytes_recv` | The total number of bytes the client has received since the server has been started +`client_bytes_sent` | The total number of bytes sent to all connected sites +`client_connect_errors` | The number of TCP/IP connection errors +`client_connects` | A count of the number of site connections made to this node +`client_redirect` | If a client connects to a non-leader node, it will be redirected to a leader node +`client_rx_kbps` | A snapshot of the client (site)-received kilobits/second taken once a minute. The past 8 snapshots are stored in this list. Newest snapshots appear on the left side of the list. +`client_tx_kbps` | A snapshot of the client (site)-sent kilobits/second taken once a minute. The past 8 snapshots are stored in this list. Newest snapshots appear on the left side of the list. + +### Server + +Field | Description +:-----|:----------- +`server_bytes_recv` | The total number of bytes the server (listener) has received +`server_bytes_sent` | The total number of bytes the server (listener) has sent +`server_connect_errors` | The number of listener to site connection errors +`server_connects` | The number of times the listener connects to the client site +`server_fullsyncs` | The number of fullsync operations that have occurred since the server was started +`server_rx_kbps` | A snapshot of the server (listener) received kilobits/second taken once a minute. The past 8 snapshots are stored in this list. Newest snapshots appear on the left side of the list. +`server_tx_kbps` | A snapshot of the server (listener) sent kilobits/second taken once a minute. The past 8 snapshots are stored in this list. Newest snapshots appear on the left side of the list. +`server_stats` | See <a href="http://docs.basho.com/riak/kv/2.2.6/using/reference/multi-datacenter/statistics/#server-statistics">Server Statistics</a> + +### Elections and Objects + +Field | Description +:-----|:----------- +`elections_elected` | If the replication leader node becomes unresponsive or unavailable, a new leader node in the cluster will be elected +`elections_leader_changed` | The number of times a Riak node has surrendered leadership +`objects_dropped_no_clients` | If the realtime replication work queue is full and there aren't any clients to receive objects, then objects will be dropped from the queue. These objects will be synchronized during a fullsync operation. +`objects_dropped_no_leader` | If a client (site) cannot connect to a leader, objects will be dropped during realtime replication +`objects_forwarded` | The number of Riak objects forwarded to the leader the participate in replication. *Please note that this value will only be accurate on a non-leader node*. +`objects_sent` | The number of objects sent via realtime replication + +### Other + +Field | Description +:-----|:----------- +`listener_<nodeid>` | Defines a replication listener that is running on node `<nodeid>` +`[sitename]_ips` | Defines a replication site +`leader` | Which node is the current leader of the cluster +`local_leader_message_queue_len` | The length of the object queue on the leader +`local_leader_heap_size `| The amount of memory the leader is using + +## Client Statistics + +Field | Description +------|------------ +`node` | A unique ID for the Riak node on which the client (site) is running +`site` | The connected site name configured with `riak-repl add-site` +`strategy` | A replication strategy defines an implementation of the Riak Replication protocol. Valid values: `keylist`, `syncv1` +`fullsync_worker` | The Erlang process ID of the fullsync worker +`waiting_to_retry` | The listeners currently waiting to retry replication after a failure +`connected` | A list of connected clients<ul><li>`connected` --- The IP address and port of a connected client (site)</li><li>`cluster_name` --- The name of the connected client (site)</li><li>`connecting` --- The PID, IP address, and port of a client currently establishing a connection</li></ul> +`state` | State shows what the current replication strategy is currently processing. The following definitions appear in the status output if keylist strategy is being used. They can be used by Basho support to identify replication issues.<ul><li>`request_partition`</li><li>`wait_for_fullsync`</li><li>`send_keylist`</li><li>`wait_ack`</li></ul> + +## Bounded Queue + +The bounded queue is responsible for holding objects that are waiting to +participate in realtime replication. Please see the [Riak MDC Replication Configuration]({{<baseurl>}}riak/kv/2.2.6/configuring/v2-multi-datacenter/) guide for more information. + +Field | Description +------|------------ +`queue_pid` | The Erlang process ID of the bounded queue +`dropped_count` | The number of objects that failed to be enqueued in the bounded queue due to the queue being full. *These objects will be replicated during the next fullsync operation*. +`queue_length` | The number of Riak objects currently in the bounded queue +`queue_byte_size` | The size of all objects currently in the queue +`queue_max_size` | The number of bytes the queue can hold before objects are dropped. *These objects will be replicated during the next fullsync operation*. +`queue_percentage` | The percentage of the queue that is full +`queue_pending` | The current count of "in-flight" objects we've sent that the client has not acknowledged +`queue_max_pending` | The maximum number of objects that can be "in flight" before we refuse to send any more. + +## Server Statistics + +Field | Description +------|------------ +`node` | A unique ID for the Riak node on which the server (listener) is running +`site` | The connected site name configured with `riak-repl add-site` +`strategy` | A replication strategy defines an implementation of the Riak Replication protocol. Valid values: `keylist` or `syncv1`. +`fullsync_worker` | The Erlang process ID of the fullsync worker +`bounded_queue` | See the <a href="http://docs.basho.com/riak/kv/2.2.6/using/cluster-operations/v2-multi-datacenter/#bounded-queue">Bounded Queue</a> section above +`state` | State shows what the current replication strategy is processing. The following definitions appear in the status output if the keylist strategy is being used. They can be used by Basho support to identify replication issues.<ul><li>`wait_for_partition`</li><li>`build_keylist`</li><li>`wait_keylist`</li><li>`diff_bloom`</li><li>`diff_keylist`</li></ul>s +`message_queue_len` | The number of Erlang messages that are waiting to be processed by the server + +## Keylist Strategy + +These similar fields are under both the `keylist_server` and +`keylist_client` fields. Any differences are described in the table. + +Field | Description +------|------------ +`fullsync` | On the client, the number of partitions that remain to be processed. On the server, the partition currently being processed by fullsync replication. +`partition_start` | The number of elapsed seconds since replication has started on a given partition +`stage_start` | The number of elapsed seconds since replication has started on a given stage +`get_pool_size` | The number of Riak get finite state workers available to process requests diff --git a/content/riak/kv/2.2.6/using/cluster-operations/v3-multi-datacenter.md b/content/riak/kv/2.2.6/using/cluster-operations/v3-multi-datacenter.md new file mode 100644 index 0000000000..d1314d4e4e --- /dev/null +++ b/content/riak/kv/2.2.6/using/cluster-operations/v3-multi-datacenter.md @@ -0,0 +1,421 @@ +--- +title_supertext: "V3 Multi-Datacenter" +title: "Replication Operations" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "V3 Multi-Datacenter" + identifier: "cluster_operations_v3" + weight: 114 + parent: "managing_cluster_operations" +toc: true +commercial_offering: true +aliases: + - /riak-docs/riak/2.2.6/ops/mdc/v3/operations + - /riak-docs/riak/kv/2.2.6/ops/mdc/v3/operations +--- + +[config v3 mdc]: {{<baseurl>}}riak/kv/2.2.6/configuring/v3-multi-datacenter +[config v3 nat]: {{<baseurl>}}riak/kv/2.2.6/configuring/v3-multi-datacenter/nat +[config v3 quickstart]: {{<baseurl>}}riak/kv/2.2.6/configuring/v3-multi-datacenter/quick-start +[config v3 ssl]: {{<baseurl>}}riak/kv/2.2.6/configuring/v3-multi-datacenter/ssl +[ref v3 stats]: {{<baseurl>}}riak/kv/2.2.6/using/reference/multi-datacenter/statistics + +This document explains how to manage replication with the `riak-repl` +command. Some of these commands can be set or behavior altered by +setting appropriate [configuration][config v3 mdc] values. + +All commands need to be run only once on a single node of a cluster for +the changes to propagate to all other nodes. All changes will persist +across node restarts and will automatically take effect when nodes are +added to the cluster. + +## Cluster Connectivity + +#### clustername + +Set the `clustername` for all nodes in a Riak cluster. + +* Without a parameter, returns the current name of the cluster +* With a parameter, names the current cluster + +To **set** the `clustername`: + +* Syntax: `riak-repl clustername <clustername>` +* Example: `riak-repl clustername Boston` + +To **get** the `clustername`: + +* Syntax: `riak-repl clustername` +* Example: `riak-repl clustername` + +#### connect + +The `connect` command establishes communications from a source cluster +to a sink cluster of the same ring size. The `host:port` of the sink +cluster is used for this. The IP and port to connect to can be found in +the `advanced.config` of the remote cluster, under `riak_core` and +`cluster_mgr`. + +The `host` can be either an IP address + +* Syntax: `riak-repl connect <ip>:<port>` +* Example: `riak-repl connect 192.168.2.1:9080` + +...or a hostname that will resolve to an IP address. + +* Syntax: `riak-repl connect <host>:<port>` +* Example: `riak-repl connect Austin:9080` + +#### disconnect + +Disconnecting a source cluster from a sink cluster. + +You may define a `host:port` combination + +* Syntax: `riak-repl disconnect <host>:<port>` +* Example: `riak-repl disconnect 192.168.2.1:9080` + +...or use the *name* of the cluster. + +* Syntax: `riak-repl disconnect <sink_clustername>` +* Example: `riak-repl disconnect Austin` + +#### connections + +Display a list of connections between source and sink clusters. + +* Syntax: `riak-repl connections` +* Example: `riak-repl connections` + +#### clusterstats + +Displays current cluster stats using an optional `ip:port` as well as an +optional `protocol-id`. + +`protocol-id` can be one of the following: + +* `cluster_mgr` +* `rt_repl` +* `fs_repl` + +The `clusterstats` command in use: + +* Syntax: `riak-repl clusterstats <host>:<port> <protocol-id>` +* Example: `riak-repl clusterstats 192.168.2.1:9080` +* Example: `riak-repl clusterstats 192.168.2.1:9080 fs_repl` + + +## Realtime Replication Commands + +#### realtime enable + +Enable realtime replication from a source cluster to sink clusters. + +This will start queuing updates for replication. The cluster will still +require an invocation of `realtime start` for replication to occur. + +* Syntax: `riak-repl realtime enable <sink_clustername>` +* Example: `riak-repl realtime enable Austin` + +#### realtime disable + +Disable realtime replication from a source cluster to sink clusters. + +* Syntax: `riak-repl realtime disable <sink_clustername>` +* Example: `riak-repl realtime disable Austin` + + +#### realtime start + +Start realtime replication connections from a source cluster to sink +clusters. See also `realtime enable` (above). + +* Syntax: `riak-repl realtime start <sink_clustername>` +* Example: `riak-repl realtime start Austin` + +#### realtime stop + +Stop realtime replication from a source cluster to sink clusters. + +* Syntax `riak-repl realtime stop <sink_clustername>` +* Example `riak-repl realtime stop Austin` + + +## Fullsync Replication Commands + +These behaviors can be altered by using the `advanced.config` +`fullsync_on_connect` parameter. See the [Configuration Guide][config v3 mdc] for more information. + +#### fullsync enable + +Enable fullsync replication from a source cluster to sink clusters. By +default, a fullsync will begin as soon as a connection to the remote +cluster is established. + +* Syntax: `riak-repl fullsync enable <sink_clustername>` +* Example: `riak-repl fullsync enable Austin` + +#### fullsync disable + +Disables fullsync for a cluster. + +* Syntax: `riak-repl fullsync disable <sink_clustername>` +* Example: `riak-repl fullsync disable Austin` + +#### fullsync start + +Starts a fullsync. If the application configuration +`fullsync_on_connect` is set to `false`, a fullsync needs to be started +manually. This is also used to trigger a periodic fullsync using a cron +job. While a fullsync is in progress, a `start` command is ignored and a +message is logged. + +* Syntax: `riak-repl fullsync start <sink_clustername>` +* Example: `riak-repl fullsync start Austin` + +#### fullsync stop + +Stops a fullsync. + +* Syntax: `riak-repl fullsync stop <sink_clustername>` +* Example: `riak-repl fullsync stop Austin` + +## Cascading Realtime Writes + +#### realtime cascades + +Shows the current cascading realtime setting. + +* Syntax: `realtime cascades` +* Example: `riak-repl realtime cascades` + +#### realtime cascades always + +Enable realtime cascading writes. + +* Syntax: `realtime cascades always` +* Example: `riak-repl realtime cascades always` + +#### realtime cascades never + +Disable realtime cascading writes. + +* Syntax: `realtime cascades never` +* Example: `riak-repl realtime cascades never` + + +## NAT + +**Note**: See the [V3 Multi Data Center Replication With NAT][config v3 nat] for more information. + +#### nat-map show + +Show the current NAT mapping table. + +* Syntax: `nat-map show` +* Example: `riak-repl nat-map show` + +#### nat-map add + +Adds a NAT map from the external IP, with an optional port, to an +internal IP. + +* Syntax: `nat-map add <externalip>[:port] <internalip>` +* Example: `riak-repl nat-map add 128.205.106.1:5555 192.168.1.2` + +#### nat-map del + +Deletes a specific NAT map entry. + +* Syntax: `nat-map del <externalip>[:port] <internalip>` +* Example: `riak-repl nat-map del 128.205.106.1:5555 192.168.1.2` + +NAT changes will be applied once fullsync and/or realtime replication +has been stopped and started. + + +## Riak CS MDC Gets + +#### proxy-get enable + +Enable Riak CS `proxy_get` requests from a **sink** cluster (if +`proxy_get` has been enabled in `advanced.config`). + +* Syntax: `proxy-get enable <sink_clustername>` +* Example: `riak-repl proxy-get enable newyorkbackup` + +#### `proxy-get disable` + +Disable Riak CS `proxy_get` requests from a **sink** cluster (if +`proxy_get` has been enabled in `advanced.config`). + +* Syntax: `proxy-get disable <sink_clustername>` +* Example: `riak-repl proxy-get disable newyorkbackup` + +#### `add-block-provider-redirect` + +Provide a redirection to the `<to-cluster-id>` for `proxy_get` if the +`<from-cluster>` is going to be decommissioned. + +* Syntax: `riak-repl add-block-provider-redirect <from-cluster> <to-cluster>` +* Example: `riak-repl add-block-provider-redirect "{'dev1@127.0.0.1',{1391,544501,519016}}" "{'dev3@127.0.0.1',{1299,512501,511032}}"` + +#### `show-block-provider-redirect` +Show the mapping for a given cluster-id redirect. + +* Syntax: `riak-repl show-block-provider-redirect <from-cluster>` +* Example: `riak-repl show-block-provider-redirect "{'dev1@127.0.0.1',{1391,544501,519016}}"` + +#### `delete-block-provider-redirect` +Delete a existing redirect such that proxy_gets go again to the original +provider cluster id. + +* Syntax:* `riak-repl delete-block-provider-redirect <from-cluster>` +* Example:* `riak-repl delete-block-provider-redirect "{'dev1@127.0.0.1', {1391,544501,519016}}"` + +#### `show-local-cluster-id` + +Display this cluster's cluster-id tuple, for use with the +`*-block-provider-redirect` commands. + +**Note**: A cluster-id is surrounded by double quotes, which need to be +included when passed to `*-block-provider-redirect`. + +* Syntax: `riak-repl show-local-cluster-id` +* Example: + + ```bash + riak-repl show-local-cluster-id + ``` + + Possible output: + + ``` + local cluster id: "{'dev1@127.0.0.1',{1391,544501,519016}}" + ``` + +## `riak-repl` Status Output + +Details about the `riak-repl status` command can be found under +[Statistics][ref v3 stats]. + + +## Tuning + +These tuning values may also be set via the node's `advanced.config` file. +See the [Configuration Guide][config v3 mdc] for more information. + +#### `fullsync max_fssource_node` + +This limits the number of fullsync workers that will be running on each +individual node in a source cluster. This is a hard limit for *all* +fullsyncs that are enabled. Additional fullsync configurations will +*not* increase the number of fullsync workers allowed to run on any +node. This only affects nodes on the source cluster on which this +parameter is defined via the configuration file or command line. + +* Syntax: `riak-repl fullsync max_fssource_node <value>` +* Default: `1` +* Example: `riak-repl fullsync max_fssource_node 2` + +#### `fullsync max_fssource_cluster` + +This is the hard limit of fullsync workers that will be running on the +source side of a cluster across all nodes on that cluster for a fullsync +to a sink cluster. This means if one has configured fullsync for two +different clusters, both with a max_fssource_cluster of 5, 10 fullsync +workers can be in progress. Only affects nodes on the source cluster on +which this parameter is defined via the configuration file or the +command line. + +* Syntax: `riak-repl fullsync max_fssource_cluster <value>` +* Default: `5` +* Example: `riak-repl fullsync max_fssource_cluster 5` + + +#### `fullsync max_fssink_node` + +This limits the number of fullsync workers allowed to run on each +individual node in a sink cluster. This is a hard limit for each +fullsync source node interacting with a sink node. Thus, multiple +simultaneous source connections to a sink node will have to share the +sink node’s number of maximum connections. Only affects nodes on the +sink cluster on which this parameter is defined via the configuration +file or command line. + +* Syntax: `riak-repl fullsync max_fssink_node <value>` +* Default: `1` +* Example: `riak-repl fullsync max_fssink_node 5` + + +## Mixing Version 2 Replication with Version 3 Replication + +Riak Version 2 Replication and Version 3 Replication can be safely used +at the same time. If you choose to move to Version 3 Replication +completely, we recommend disabling Version 2 realtime +replication bucket hooks with the `riak-repl modes` command. + +#### `riak-repl modes` + +`modelist` is one or both of `mode_repl12` (Version 2) or `mode_repl13` +(Version 3) separated by spaces (without commas). + +* Syntax: `riak-repl modes <modelist>` +* Example: + + ```bash + riak-repl modes mode_repl12 mode_repl13 + ``` + + Possible output: + + ``` + Current replication modes: [mode_repl12,mode_repl13] + ``` + +To check the current replication modes: + +* Syntax: `riak-repl modes` +* Example: + + ```bash + riak-repl modes + ``` + + Possible output: + + ``` + Current replication modes: [mode_repl12,mode_repl13] + ``` + +## Configurations and Metadata in Replication + +Fullsync and Realtime replication replicates data from source clusters to sink clusters, +but some configurations and metadata (such as search indices and bucket properties) will +not be replicated. + +Non-replication of certain configurations and metadata supports +heterogenous cluster configurations in Replication, but there operational things you can +do when you want homogeneous cluster configurations. + +### Search Indices in Replication + +Any search index that is created on a source cluster will _not_ be +created on sink clusters as part of replication. + +If you want search indices on a source cluster to be present on the +sink clusters, you should update this data for each +cluster at the same time you would change the source cluster. + +### Buckets and Bucket Types in Replication + +Buckets and Bucket Type properties on the source cluster +will _not_ be replicated from source clusters to sink clusters. + +If you want the properties for Buckets or Bucket Types +present on the source cluster to be propagated to sink clusters +you should update this data for each cluster at the same +time you would change the source cluster. diff --git a/content/riak/kv/2.2.6/using/performance.md b/content/riak/kv/2.2.6/using/performance.md new file mode 100644 index 0000000000..1f3b9cd64b --- /dev/null +++ b/content/riak/kv/2.2.6/using/performance.md @@ -0,0 +1,264 @@ +--- +title: "Improving Performance" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Performance" + identifier: "managing_performance" + weight: 206 + parent: "managing" +toc: true +aliases: + - /riak-docs/riak/kv/2.2.6/ops/tuning/linux/ + - /riak-docs/riak/2.2.6/ops/tuning/linux/ +--- + +Many Unix-like operating systems and distributions are tuned for desktop +or light use out of the box and not for a production database. This +guide describes recommended system performance tunings for operators of +new and existing Riak clusters. The tunings present in this guide should +be considered as a starting point. It is important to make note of what +changes are made and when in order to measure the impact of those +changes. + +For performance and tuning recommendations specific to running Riak +clusters on the Amazon Web Services EC2 environment, see [AWS Performance Tuning]({{<baseurl>}}riak/kv/2.2.6/using/performance/amazon-web-services). + +{{% note title="Note on other operating systems" %}} +Unless otherwise specified, the tunings recommended below are for Linux +distributions. Users implementing Riak on BSD and Solaris distributions can +use these tuning recommendations to make analogous changes in those operating +systems. +{{% /note %}} + +## Storage and File System Tuning + +### Virtual Memory + +Due to the heavily I/O-focused profile of Riak, swap usage can result in +the entire server becoming unresponsive. We recommend setting +`vm.swappiness` to 0 in `/etc/sysctl.conf` to prevent swapping as much +as possible: + +```config +vm.swappiness = 0 +``` + +Ideally, you should disable swap to ensure that Riak's process pages are +not swapped. Disabling swap will allow Riak to crash in situations where +it runs out of memory. This will leave a crash dump file, named +`erl_crash.dump`, in the `/var/log/riak` directory which can be used to +determine the cause of the memory usage. + +### Transparent Huge Pages (THP) + +Owing to the way that THP handles memory usage, disproportionately large amounts of memory can become held up in any large database application. We recommend disabling THP at boot time. Unfortunately this operation is rather OS specific. As many of our customers are running Red Hat 6, we have included instructions on how to do so underneath. If you are using a different operating system, please refer to documentation for your OS. + +In Red Hat 6, you can disable THP by editing `grub.conf` and adding the following line: + +``` +transparent_hugepage=never +``` + +For the change to become effective, a server reboot is required. + +{{% note title="Note on Kernel Tuning Tools" %}} +Some Kernel tuning tools such as ktune specify that THP should be enabled. This can cause THP to seem to be enabled even though `transparent_hugepage=never` has already been added to `grub.conf` and the system rebooted. Should this occur, please refer to the documentation for the Kernel tuning tool you are using as to how to disable THP. +{{% /note %}} + +### Mounts + +Riak makes heavy use of disk I/O for its storage operations. It is +important that you mount volumes that Riak will be using for data +storage with the `noatime` flag, meaning that filesystem +[inodes](http://en.wikipedia.org/wiki/Inode) on the volume will not be +touched when read. This flag can be set temporarily using the following +command: + + +```bash +mount -o remount,noatime <riak_data_volume> +``` + +Replace `<riak_data_volume>` in the above example with your actual Riak +data volume. The `noatime` can be set in `/etc/fstab` to mount +permanently. + +### Schedulers + +I/O or disk scheduling is a blanket term used to describe the method by +which an operating system chooses how to order input and output +operations to and from storage. + +The default I/O scheduler (elevator) on Linux is completely fair queuing +or `cfq`, which is designed for desktop use. While a good +general-purpose scheduler, is not designed to provide the kind of +throughput expected in production database deployments. + +Scheduler recommendations: + +* The `noop` scheduler when deploying on iSCSI over HBAs, or any + hardware-based RAID. +* The `deadline` scheduler when using SSD-based storage. + +To check the scheduler in use for block device `sda`, for example, use +the following command: + +```bash +cat /sys/block/sda/queue/scheduler +``` + +To set the scheduler to `deadline`, use the following command: + +```bash +echo deadline > /sys/block/sda/queue/scheduler +``` + +The default I/O scheduler queue size is 128. The scheduler queue sorts +writes in an attempt to optimize for sequential I/O and reduce seek +time. Changing the depth of the scheduler queue to 1024 can increase the +proportion of sequential I/O that disks perform and improve overall +throughput. + +To check the scheduler depth for block device `sda`, use the following +command: + +```bash +cat /sys/block/sda/queue/nr_requests +``` + +To increase the scheduler depth to 1024, use the following command: + +```bash +echo 1024 > /sys/block/sda/queue/nr_requests +``` + +### Filesystem + +Advanced journaling filesystems like [ZFS](http://zfsonlinux.org/) and +[XFS](http://xfs.org/index.php/Main_Page) are recommended on some +operating systems for greater reliability and recoverability. + +At this time, Basho can recommend using ZFS on Solaris, SmartOS, and +OmniOS. ZFS may work well with Riak on direct Solaris clones like +IllumOS, but we cannot yet recommend this. [ZFS on +Linux](http://zfsonlinux.org) is still too early in its project lifetime +to be recommendable for production use due to concerns that have been +raised about excessive memory use. ZFS on FreeBSD is more mature than +ZFS on Linux, but Basho has not yet performed sufficient performance and +reliability testing to recommend using ZFS and Riak on FreeBSD. + +In the meantime, the [ext3](http://en.wikipedia.org/wiki/Ext3) and +[ext4](http://en.wikipedia.org/wiki/Ext4) filesystems are sufficient on +operating systems on which ZFS or XFS are not available or recommended. + +The ext4 file system defaults include two options that increase +integrity but slow performance. Because Riak's integrity is based on +multiple nodes holding the same data, these two options can be changed +to boost I/O performance. We recommend setting `barrier=0` and +`data=writeback` when using the ext4 filesystem. + +Similarly, the XFS file system defaults can be optimized to improve +performance. We recommend setting `nobarrier`, `logbufs=8`, +`logbsize=256k`, and `allocsize=2M` when using the XFS filesystem. + +As with the `noatime` setting, these settings should be added to +`/etc/fstab` so that they are persisted across server restarts. + +## Kernel and Network Tuning + +The following settings are minimally sufficient to improve many aspects +of Riak usage on Linux, and should be added or updated in +`/etc/sysctl.conf`: + +```config +net.ipv4.tcp_max_syn_backlog = 40000 +net.core.somaxconn = 40000 +net.core.wmem_default = 8388608 +net.core.rmem_default = 8388608 +net.ipv4.tcp_sack = 1 +net.ipv4.tcp_window_scaling = 1 +net.ipv4.tcp_fin_timeout = 15 +net.ipv4.tcp_keepalive_intvl = 30 +net.ipv4.tcp_tw_reuse = 1 +net.ipv4.tcp_moderate_rcvbuf = 1 +``` + +{{% note title="Note on system default" %}} +In general, these recommended values should be compared with the system +defaults and only changed if benchmarks or other performance metrics indicate +that networking is the bottleneck. +{{% /note %}} + +The following settings are optional, but may improve performance on a +10Gb network: + +```config +net.core.rmem_max = 134217728 +net.core.wmem_max = 134217728 +net.ipv4.tcp_mem = 134217728 134217728 134217728 +net.ipv4.tcp_rmem = 4096 277750 134217728 +net.ipv4.tcp_wmem = 4096 277750 134217728 +net.core.netdev_max_backlog = 300000 +``` + +Certain network interfaces ship with on-board features that have been +shown to hinder Riak network performance. These features can be disabled +via `ethtool`. + +For an Intel chipset NIC using the +[ixgbe](http://www.intel.com/support/network/adapter/pro100/sb/CS-032530.htm) +driver running as `eth0`, for example, run the following command: + +```bash +ethtool -K eth0 lro off +``` + +For a Broadcom chipset NIC using the `bnx` or `bnx2` driver, run: + +```bash +ethtool -K eth0 tso off +``` + +`ethtool` settings can be persisted across reboots by adding the above +command to the `/etc/rc.local` script. + +{{% note title="Pro tip" %}} +Tuning these values will be required if they are changed, as they affect all +network operations. +{{% /note %}} + +## Optional I/O Settings + +If your cluster is experiencing excessive I/O blocking, the following +settings may help prevent disks from being overwhelmed during periods of +high write activity at the expense of peak performance for spiky +workloads: + +```config +vm.dirty_background_ratio = 0 +vm.dirty_background_bytes = 209715200 +vm.dirty_ratio = 40 +vm.dirty_bytes = 0 +vm.dirty_writeback_centisecs = 100 +vm.dirty_expire_centisecs = 200 +``` + +These settings have been tested and benchmarked by Basho in nodes with +16 GB of RAM. + +## Open Files Limit + +Riak and supporting tools can consume a large number of open file +handles during normal operation. For stability, increasing the number of +open files limit is necessary. See [Open Files Limit]({{<baseurl>}}riak/kv/2.2.6/using/performance/open-files-limit/) for more +details. + +## Other Tuning Docs + +* [AWS Performance Tuning]({{<baseurl>}}riak/kv/2.2.6/using/performance/amazon-web-services) +* [Erlang VM Tuning]({{<baseurl>}}riak/kv/2.2.6/using/performance/erlang) +* [Latency Reduction]({{<baseurl>}}riak/kv/2.2.6/using/performance/latency-reduction) +* [Open Files Limit]({{<baseurl>}}riak/kv/2.2.6/using/performance/open-files-limit/) diff --git a/content/riak/kv/2.2.6/using/performance/amazon-web-services.md b/content/riak/kv/2.2.6/using/performance/amazon-web-services.md new file mode 100644 index 0000000000..dd322d7aa8 --- /dev/null +++ b/content/riak/kv/2.2.6/using/performance/amazon-web-services.md @@ -0,0 +1,243 @@ +--- +title: "Amazon Web Services Performance Tuning" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Amazon Web Services" + identifier: "performance_aws" + weight: 106 + parent: "managing_performance" +toc: true +aliases: + - /riak-docs/riak/2.2.6/ops/tuning/aws + - /riak-docs/riak/kv/2.2.6/ops/tuning/aws +--- + +This guide introduces best practices for tuning Riak cluster performance +in the Amazon Web Services (AWS) Elastic Compute Cloud (EC2) environment. + +> **Note:** +> +> The following guide is supplementary. Be sure to check out [Improving Performance](../) for general performance and tuning recommendations before continuing with this guide. + +## EC2 Instances + +EC2 instances are available as predefined types which encapsulate a +fixed amount of computing resources. For Riak, the most important of +these resources are Disk I/O, RAM, and Network I/O, followed by CPU +cores. With this in mind, Riak users have reported success with large, +extra large, and cluster compute instance types for use as cluster nodes +in the AWS EC2 environment. + +The most commonly used [instance types](http://aws.amazon.com/ec2/instance-types/) for Riak cluster nodes are `large` and `xlarge` `m` class (General Purpose), such as `m4.xlarge`. In cases where 10-gigabit Ethernet networking is desired, the Cluster Compute class of EC2 instances, such as `cc2.8xlarge` can be used. + +Amazon also offers a High I/O Quadruple Extra Large instance +(`hi1.4xlarge`) that is backed by solid state drives (SSD) and features +very high I/O performance. + +EBS-Optimized EC2 instances, which provide between 500 Megabits per +second and 1,000 Megabits per second of throughput with [Provisioned +IOPS](http://aws.amazon.com/about-aws/whats-new/2012/07/31/announcing-provisioned-iops-for-amazon-ebs/) +EBS volumes are also available, and recommended for use with Provisioned +IOPS EBS volumes. + +Riak's primary bottleneck will be disk and network I/O, meaning that in +most cases, standard EBS will incur too much latency and iowait. Riak's +I/O pattern tends to operate on small blobs from many places on the +disk, whereas EBS is best at bulk reads and writes. The negative effects +of this pattern can be mitigated by adding RAID over multiple volumes, +using Provisioned IOPS, and/or choosing the Bitcask backend if secondary +indexes are not needed for the application. + +In any case, proper benchmarking and tuning are needed to achieve the +desired performance. + +{{% note title="Tip" %}} +Most successful AWS cluster deployments use more EC2 instances than they would +the same number of physical nodes to compensate for the performance +variability caused by shared, virtualized resources. Plan to have more EC2 +instance based nodes than physical server nodes when estimating cluster size +with respect to node count. +{{% /note %}} + +## Operating System + +### Clocks + +NTP is configured by default on Amazon EC2 Linux instances. Please +refer to the [Set the Time for an +Instance](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/set-time.html) +section of the EC2 documentation for steps on verifying if NTP is +working properly. If NTP is not working properly, significant clock +drift can occur. + +### Mounts and Scheduler + +On EBS volumes, the **deadline** scheduler should be used. To check the +scheduler in use for block device xvdf, for example, use the following +command: + +```bash +cat /sys/block/xvdf/queue/scheduler +``` + +To set the scheduler to deadline, use the following command: + +```bash +echo deadline > /sys/block/xvdf/queue/scheduler +``` + +More information on the disk scheduler is available in [Improving Performance](../). + +### Virtual Memory Subsystem + +EBS volumes have considerably less bandwidth than hardware disks. To +avoid saturating EBS bandwidth and inducing IO latency spikes, it is +recommended to tune the Linux virtual memory subsystem to flush smaller +amounts of data more often. To do so, please see [Linux system performance tuning](../#optional-i-o-settings). + +### Forensics + +When a failure occurs, collect as much information as possible. Check +monitoring systems, back up log and configuration files if they are +available, including system logs like `dmesg` and `syslog`. Make sure +that the other nodes in the Riak cluster are still operating normally +and are not affected by a wider problem like an AWS service outage. Try +to determine the cause of the problem from the data you have collected. +If you are paying for [TI Tokyo support services](https://www.tiot.jp/en/solutions/riak/), either directly or re-sold under Erlang Solutions, and the failure comes from Riak or is not immediately obvious, you may open a ticket on the TI Tokyo Client Services help desk. + +Have your collected data ready when contacting TI Tokyo Client Services. A +Client Services Engineer (CSE) might request log files, configuration +files, or other information. + +## Data Loss + +Many failures either do not entail data loss or have minimal loss that +can be repaired automatically, without intervention. Outage of a single +node does not necessarily cause data loss, as other replicas of every +key are available elsewhere in the cluster. Once the node is detected as +down, other nodes in the cluster will take over its responsibilities +temporarily and transmit the updated data to it when it eventually +returns to service (also called hinted handoff). + +The more severe data loss scenarios usually relate to hardware failure +(in the case of AWS, service failure or instance termination). In the +cases where data is lost, several options are available for restoring +the data: + +1. Restore from backup. A daily backup of Riak nodes can be helpful. + The data in this backup may be stale depending on the time at which + the node failed, but can be used to partially restore data from + lost EBS volumes. If running in a RAID configuration, rebuilding the + array may also be possible. +2. Restore from Multi-Datacenter Replication. If replication is enabled + between two or more clusters, the missing data will gradually be + restored via realtime replication and fullsync replication. A + fullsync operation can also be triggered manually via the + `riak-repl` command. +3. Restore using intra-cluster repair. Riak versions 1.2 and greater + include a "repair" feature which will restore lost partitions with + data from other replicas. This currently has to be invoked manually + using the Riak console and should be performed with guidance from a + Basho CSE. + +Once data has been restored, normal operations should continue. If +multiple nodes completely lose their data, consultation and assistance +from Basho is strongly recommended. + +## Benchmarking + +Using a tool such as [Basho Bench](https://github.com/basho/basho_bench), you can generate load that +simulates application operations by constructing and communicating +approximately-compatible data payloads with the Riak cluster directly. + +Benchmarking is critical to determining the appropriate EC2 instance +types, and strongly recommended. More information is available on +benchmarking Riak clusters with [Basho Bench](../benchmarking). + +Besides running Basho Bench, we also advise that you load test Riak with +your own tests to ensure that load imparted by MapReduce queries, +full-text queries, and index queries are within the expected range. + +## Simulating Upgrades, Scaling, and Failure states + +In addition to simply measuring performance, it is also important to +measure how performance degrades when the cluster is not in +steady-state. While under a simulation of live load, the following +states might be simulated: + +1. Stop one or more nodes normally and restart them after a few moments + (simulates [rolling upgrade](../../../setup/upgrading/cluster)). +2. Join two or more nodes to the cluster. +3. Leave nodes from the cluster (after step #2). +4. Hard-kill the Riak `beam.smp` process (i.e., `kill -9`) and then + restart it. +5. Hard-reboot a node's instance using the AWS console and then + restart it. +6. Hard-stop and destroy a node's instance and build a new one from + backup. +7. Via networking, e.g. firewall, partition one or more nodes from + the rest of the cluster and then restore the original + configuration. + +## Out-of-Memory + +Sometimes, Riak will exit when it runs out of available RAM. While this +does not necessarily cause data loss, it may indicate that the cluster +needs to be scaled out. While the Riak node is out, other nodes may also +be at risk if free capacity is low on the rest of the cluster, so +monitor carefully. + +Replacing the EC2 instance type with one that has greater RAM capacity +may temporarily alleviate the problem, but out of memory (OOM) tends to +be an indication that the cluster is underprovisioned. + +Software bugs (memory leaks) could also be a cause of OOM, so we +recommend paid support Riak users to contact TI Tokyo Client Services +if this problem occurs. + +## Dealing with IP addresses + +EC2 instances that are not provisioned inside a VPC can change the +following attributes after a restart: + +* Private IP address +* Public IP address +* Private DNS +* Public DNS + +Because these parameters play a role in a Riak instance's node name, +ensure that you follow the steps outlined in the [Node Name Changed](../../repair-recovery/failed-node/#node-name-changed) section to replace +it. + +To avoid this inconvenience, you can deploy Riak inside a +[VPC](http://aws.amazon.com/vpc/). Instances inside the VPC do not +change their private IP address on restart. In addition you get the +following benefits: + +* Access control lists can be defined at multiple levels +* The instance is not automatically open to the internet +* Amazon VPC is [free](http://aws.amazon.com/vpc/pricing/) + +## Choice of Storage + +EC2 instances support ephemeral and EBS storage. Ephemeral is local to +the instance, generally performs better, but disappears when instances +go down. + +On the other hand, EBS is effectively network attached storage that +persists after instances go down. Along with EBS you can optionally +enable [Provisioned +IOPS](http://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/USER_PIOPS.html) +(PIOPS) provide more stable performance. + +For more information on EC2 storage options, please see their +[documentation](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Storage.html). + +## References + +* [Improving Performance](../) +* [Failure and Recovery](../../repair-recovery) +* [Basho Client Services Help Desk](https://help.basho.com) diff --git a/content/riak/kv/2.2.6/using/performance/benchmarking.md b/content/riak/kv/2.2.6/using/performance/benchmarking.md new file mode 100644 index 0000000000..3783d1d41c --- /dev/null +++ b/content/riak/kv/2.2.6/using/performance/benchmarking.md @@ -0,0 +1,598 @@ +--- +title: "Benchmarking" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Benchmarking" + identifier: "performance_benchmarking" + weight: 100 + parent: "managing_performance" +toc: true +aliases: + - /riak-docs/riak/2.2.6/ops/building/benchmarking + - /riak-docs/riak/kv/2.2.6/ops/building/benchmarking +--- + +Basho Bench is a benchmarking tool created to conduct accurate and +repeatable performance tests and stress tests, and to produce +performance graphs. + +Basho Bench exposes a pluggable driver interface and has been extended +to serve as a benchmarking tool against a variety of projects. New +drivers can be written in Erlang and are generally less than 200 lines +of code. + +## Installation + +You will need: + +1. One or more load-generating machines on which to install + ```basho_bench```. Especially when testing larger clusters, a + single machine cannot generate enough load to properly exercise + the cluster. Do not run the ```basho_bench``` instances on the + Riak nodes themselves, since the load generation will compete with + Riak for resources. +2. The [R statistics language](http://www.r-project.org/) must be + installed (somewhere available to you) if you wish to generate + graphs (see the [Generating Benchmark Graphs](#generating-benchmark-graphs) section, below). + +### Download ```basho_bench``` + +You can download the pre-built packages below, or build it from source. + +* **Ubuntu 14.04 LTS:** + [basho-bench_0.10.0.53-1_amd64.deb](http://ps-tools.s3.amazonaws.com/basho-bench_0.10.0.53.g0e15158-ubuntu14.04LTS-1_amd64.deb) +* **CentOS 7:** + [basho-bench-0.10.0.53-1.el7.centos.x86_64.rpm](http://ps-tools.s3.amazonaws.com/basho-bench-0.10.0.53.g0e15158-1.el7.centos.x86_64.rpm) + +### Building from Source + +#### Prerequisites + +* Erlang must be installed. See [Installing Erlang]({{<baseurl>}}riak/kv/2.2.6/setup/installing/source/erlang) for instructions + and versioning requirements. Note: Unless you're an experienced + Erlang developer, we recommend that you use Ubuntu 14.04 LTS (and + not CentOS), when building ```basho_bench``` from source. Later + versions of CentOS (6 and 7) have difficulty with installing and + enabling certain parts of the ```erlang-crypto``` package, which + is required by ```basho_bench```. +* Install ```git``` (to check out the ```basho_bench``` code) + +#### Compiling + +```bash +git clone git://github.com/basho/basho_bench.git +cd basho_bench +make +``` + +## Usage + +Run the `basho_bench` script, pass in the config file and the +directory to generate the results into: + +```bash +basho_bench --results-dir <results dir> <config file> +``` + +If you've installed ```basho_bench``` from a pre-built package, you +must specify full paths for the test results directory and config +file. (Also, don't use the common ```~/``` shell notation, specify the +user's home directory explicitly) + +```bash +basho_bench --results-dir /home/username/bench_results/ /etc/basho_bench/riakc_pb.config +``` + +The example above will generate results in +```/home/username/bench_results/current/```. + +If you built ```basho_bench``` from source, you can get away with +relative paths (and the results directory will be created in the +current directory): + +```bash +./basho_bench myconfig.config +``` + +This will generate results in `tests/current/`. You will need to +create a configuration file. The recommended approach is to start from +a file in the `examples` directory and modify settings using the +[Configuration](#configuration) section below for +reference. + +## Generating Benchmark Graphs + +The output of from running the `basho_bench` script can be used to +create graphs showing the following: + +* Throughput --- Operations per second over the duration of the test. +* Latency at 99th percentile, 99.9th percentile and max latency for + the selected operations. +* Median latency, mean latency, and 95th percentile latency for the + selected operations. + +### Prerequisites + +The R statistics language is needed to generate graphs. Note: If +necessary, R can be installed on a different machine than the one +running basho_bench, and the performance data can be copied (via +rsync, for example) from the load testing machine to the one that will +be generating and viewing the graphs (such as a desktop). + +#### Installing R on Ubuntu + +``` +sudo apt-get install r-base +``` + +#### Installing R on Other Platforms + +- [More information](http://www.r-project.org/) +- [Download R](http://cran.r-project.org/mirrors.html) + +Follow the instructions for your platform to install R. + +### Generating Graphs + +If you have installed ```basho_bench``` from a pre-built package, and +you also have R installed on the same machine, you can generate the +current result graph with the following: + +```bash +Rscript --vanilla /usr/lib/basho_bench/lib/basho_bench*/priv/summary.r -i /home/username/bench_results/current/ +``` + +This will create a results file in +```/home/username/bench_results/summary.png```. + +If you have built ```basho_bench``` from source, you can just use +```make```. To generate a benchmark graph against the current +results, run: + +```bash +make results +``` + +This will create a results file in `tests/current/summary.png`. + +You can also run this manually: + +```bash +priv/summary.r -i tests/current +``` + +### Troubleshooting Graph Generation + +For additional help, see the [Troubleshooting Graph Generation](https://github.com/basho/basho_bench#troubleshooting-graph-generation) +section of the ```basho_bench/README```. + +## How does it work? + +When Basho Bench starts (`basho_bench.erl`), it reads the +configuration (`basho_bench_config.erl`), creates a new results +directory, and then sets up the test (`basho_bench_app.erl` and +`basho_bench_sup.erl`). + +During test setup, Basho Bench creates the following: + +* One **stats process** (`basho_bench_stats.erl`). This process + receives notifications when an operation completes, plus the + elapsed time of the operation, and stores it in a histogram. At + regular intervals, the histograms are dumped to `summary.csv` as + well as operation-specific latency CSVs (e.g. `put_latencies.csv` + for the PUT operation). +* N **workers**, where N is specified by the [concurrent](#concurrent) configuration setting + (`basho_bench_worker.erl`). The worker process wraps a driver + module, specified by the [driver](#driver) + configuration setting. The driver is randomly invoked using the + distribution of operations as specified by the [operations](#operations) configuration setting. The rate at which the + driver invokes operations is governed by the [mode](#mode) setting. + +Once these processes have been created and initialized, Basho Bench +sends a run command to all worker processes, causing them to begin the +test. Each worker is initialized with a common seed value for random +number generation to ensure that the generated workload is reproducible +at a later date. + +During the test, the workers repeatedly call `driver:run/4`, passing in +the next operation to run, a keygen function, a valuegen function, and +the last state of the driver. The worker process times the operation, +and reports this to the stats process when the operation has completed. + +Finally, once the test has been run for the duration specified in the +config file, all workers and stats processes are terminated and the +benchmark ends. The measured latency and throughput of the test can be +found in `./tests/current/`. Previous results are in timestamped +directories of the form `./tests/YYYYMMDD-HHMMSS/`. + +## Configuration + +Basho Bench ships with a number of sample configuration files, available +in the `/examples` directory. + +### Global Config Settings + +#### mode + +The `mode` setting controls the rate at which workers invoke the +`{driver:run/4}` function with a new operation. There are two possible +values: + +* `{max}` --- generate as many ops per second as possible +* `{rate, N}` --- generate N ops per second, with exponentially distributed interarrival times + +Note that this setting is applied to each driver independently. For +example, if `{rate, 5}` is used with 3 concurrent workers, Basho Bench +will be generating 15 (i.e. 5 * 3) operations per second. + +```erlang +% Run at max, i.e.: as quickly as possible +{mode, max} + +% Run 15 operations per second per worker +{mode, {rate, 15}} +``` + +#### concurrent + +The number of concurrent worker processes. The default is 3 worker +processes. This determines the number of concurrent clients running +requests on API under test. + +```erlang +% Run 10 concurrent processes +{concurrent, 10} +``` + +#### duration + +The duration of the test, in minutes. The default is 5 minutes. + +```erlang +% Run the test for one hour +{duration, 60} +``` + +#### operations + +The possible operations that the driver will run, plus their "weight," +or likelihood of being run. The default is `[{get,4},{put,4},{delete, +1}]`, which means that out of every 9 operations, GET will be called +four times, PUT will be called four times, and DELETE will be called +once, on average. + +```erlang +{operations, [{get, 4}, {put, 1}]}. +``` + +Operations are defined on a **per-driver** basis. Not all drivers will +implement the GET/PUT operations discussed above. Consult the driver +source to determine the valid operations. If you're testing the HTTP +interface, for example, the corresponding operations are GET and +UPDATE, respectively. + +If a driver does not support a specified operation (`asdfput` in this +example), you may see errors like this: + +```log +DEBUG:Driver basho_bench_driver_null crashed: {function_clause, + [{{{basho_bench_driver_null,run, + [asdfput, + #Fun<basho_bench_keygen.4.4674>, + #Fun<basho_bench_valgen.0.1334>, + undefined]}}}, + {{{basho_bench_worker, + worker_next_op,1}}}, + {{{basho_bench_worker, + max_worker_run_loop,1}}}]} +``` + +#### driver + +The module name of the driver that Basho Bench will use to generate +load. A driver may simply invoke code in-process (such as when +measuring the performance of DETS) or may open network connections and +generate load on a remote system (such as when testing a Riak +server/cluster). + +Available drivers include: + +* `basho_bench_driver_http_raw` --- Uses Riak's HTTP interface to + get/update/insert data on a Riak server +* `basho_bench_driver_riakc_pb` --- Uses Riak's Protocol Buffers + interface to get/put/update/delete data on a Riak serve +* `basho_bench_driver_riakclient` --- Uses Riak's Distributed Erlang + interface to get/put/update/delete data on a Riak server +* `basho_bench_driver_bitcask` --- Directly invokes the Bitcask API +* `basho_bench_driver_dets` --- Directly invokes the DETS API + +On invocation of the `driver:run/4` method, the driver may return one of +the following results: + +* `{ok, NewState}` --- operation completed successfully +* `{error, Reason, NewState}` --- operation failed but the driver can + continue processing (i.e. recoverable error) +* `{stop, Reason}` --- operation failed; driver can't/won't continue + processing +* `{'EXIT', Reason}` --- operation failed; driver crashed + +#### code_paths + +Some drivers need additional Erlang code in order to run. Specify the +paths to this code using the `code_paths` configuration setting. + +#### key_generator + +The generator function to use for creating keys. Generators are defined +in `basho_bench_keygen.erl`. Available generators include: + +* `{sequential_int, MaxKey}` --- generates integers from 0..`MaxKey` + in order and then stops the system. Note that each instance of + this keygen is specific to a worker. +* `{partitioned_sequential_int, MaxKey}` --- the same as + `{sequential_int}`, but splits the keyspace evenly among the + worker processes. This is useful for pre-loading a large dataset. +* `{partitioned_sequential_int, StartKey, NumKeys}` --- the same as + `partitioned_sequential_int`, but starting at the defined + `StartKey` and going up to `StartKey + NumKeys`. +* `{uniform_int, MaxKey}` --- selects an integer from uniform + distribution of 0..`MaxKey`, i.e. all integers are equally probable. +* `{pareto_int, MaxKey}` --- selects an integer from a Pareto + distribution, such that 20% of the available keys get selected 80% + of the time. Note that the current implementation of this + generator _may_ yield values larger than `MaxKey` due to the + mathematical properties of the Pareto distribution. +* `{truncated_pareto_int, MaxKey}` --- the same as `{pareto_int}`, but + will _not> yield values above `MaxKey`. +* `{function, Module, Function, Args}` --- specifies an external + function that should return a key generator function. The worker + `Id` will be prepended to `Args` when the function is called. +* `{int_to_bin, Generator}` --- takes any of the above `_int` + generators and converts the number to a 32-bit binary. This is + needed for some drivers that require a binary key. +* `{int_to_str, Generator}` --- takes any of the above `_int` + generators and converts the number to a string. This is needed for + some drivers that require a string key. + +The default key generator is `{uniform_int, 100000}`. + +Examples: + +```erlang +% Use a randomly selected integer between 1 and 10,000 +{key_generator, {uniform_int, 10000}}. + +% Use a randomly selected integer between 1 and 10,000, as binary. +{key_generator, {int_to_bin, {uniform_int, 10000}}}. + +% Use a pareto distributed integer between 1 and 10,000; values < 2000 +% will be returned 80% of the time. +{key_generator, {pareto_int, 10000}}. +``` + +#### value_generator + +The generator function to use for creating values. Generators are +defined in `basho_bench_valgen.erl`. Available generators include: + +* `{fixed_bin, Size}` --- generates a random binary of `Size` + bytes. Every binary is the same size, but varies in content. +* `{exponential_bin, MinSize, Mean}` --- generates a random binary + which has an exponentially distributed size. Most values will be + approximately `MinSize` + `Mean` bytes in size, with a long tail + of larger values. +* `{uniform_bin, MinSize, MaxSize}` --- generates a random binary + which has an evenly distributed size between `MinSize` and + `MaxSize`. +* `{function, Module, Function, Args}` --- specifies an external + function that should return a value generator function. The worker + `Id` will be prepended to `Args` when the function is called. + +The default value generator is `{value_generator, {fixed_bin, 100}}`. + +Examples: + +```erlang +% Generate a fixed size random binary of 512 bytes +{value_generator, {fixed_bin, 512}}. + +% Generate a random binary whose size is exponentially distributed +% starting at 1000 bytes and a mean of 2000 bytes +{value_generator, {exponential_bin, 1000, 2000}}. +``` + +#### rng_seed + +The initial random seed to use. This is explicitly seeded, rather than +seeded from the current time, so that a test can be run in a +predictable, repeatable fashion. + +Default is `{rng_seed, {42, 23, 12}}`. + +```erlang +% Seed to {12, 34, 56} +{rng_seed, {12, 34, 56}. +``` + +#### log_level + +The `log_level` setting determines which messages Basho Bench will log +to the console and to disk. + +The default level is `debug`. + +| Valid levels +|:------------ +| `debug` +| `info` +| `warning` +| `error` + +#### report_interval + +How often, in seconds, the stats process should write histogram data +to disk. The default is 10 seconds. + +#### test_dir + +The directory in which result data is written. The default is `/tests`. + +### basho_bench_driver_riakclient Settings + +These configuration settings apply to the +`basho_bench_driver_riakclient` driver. + +#### riakclient_nodes + +List of Riak nodes to use for testing. + +```erlang +{riakclient_nodes, ['riak1@127.0.0.1', 'riak2@127.0.0.1']}. +``` + +#### riakclient_cookie + +The Erlang cookie to use to connect to Riak clients. The default is `riak`. + +```erlang +{riakclient_cookie, riak}. +``` + +#### riakclient_mynode + +The name of the local node. This is passed into +[net_kernel:start/1](http://erlang.org/doc/man/net_kernel.html). + +```erlang +{riakclient_mynode, ['basho_bench@127.0.0.1', longnames]}. +``` + +#### riakclient_replies + +This value is used for R-values during a get operation, and W-values +during a put operation. + +```erlang +% Expect 1 reply. +{riakclient_replies, 1}. +``` + +#### riakclient_bucket + +The Riak bucket to use for reading and writing values. The Default is +`<<"test">>`. + +```erlang +% Use the "bench" bucket. +{riakclient_bucket, <<"bench">>}. +``` + +### basho_bench_driver_riakc_pb Settings + +#### riakc_pb_ips + +A list of IP addresses to connect the workers to. A random IP will be +chosen for each worker. + +The default is `{riakc_pb_ips, [{127,0,0,1}]}` + +```erlang +% Connect to a cluster of 3 machines +{riakc_pb_ips, [{10,0,0,1},{10,0,0,2},{10,0,0,3}]} +``` + +#### riakc_pb_port + +The port on which to connect to the PBC interface. + +The default is `{riakc_pb_port, 8087}` + +#### riakc_pb_bucket + +The bucket to use for testing. + +The default is `{riakc_pb_bucket, <<"test">>}` + +### basho_bench_driver_http_raw Settings + +#### http_raw_ips + +A list of IP addresses to connect the workers to. Each worker makes +requests to each IP in a round-robin fashion. + +The default is `{http_raw_ips, ["127.0.0.1"]}` + +```erlang +% Connect to a cluster of machines in the 10.x network +{http_raw_ips, ["10.0.0.1", "10.0.0.2", "10.0.0.3"]}. +``` + +#### http_raw_port + +Select the default port to connect to for the HTTP server. + +The default is `{http_raw_port, 8098}`. + +```erlang +% Connect on port 8090 +{http_raw_port, 8090}. +``` + +#### http_raw_path + +The base path to use for accessing Riak, usually `"/riak/<bucket>"`. + +The default is `{http_raw_path, "/riak/test"}`. + +```erlang +% Place test data in another_bucket +{http_raw_path, "/riak/another_bucket"}. +``` + +#### http_raw_params + +Additional parameters to add to the end of the URL. This can be used +to set the `r`/`w`/`dw`/`rw` parameters as desired. + +The default is `{http_raw_params, ""}`. + +```erlang +% Set R=1, W=1 for testing a system with n_val set to 1 +{http_raw_params, "?r=1&w=1"}. +``` + +#### http_raw_disconnect_frequency + +How often, in seconds or number of operations, the HTTP clients +(workers) should forcibly disconnect from the server. + +The default is `{http_raw_disconnect_frequency, infinity}` (which +means that Basho Bench should never forcibly disconnect). + +```erlang +% Disconnect after 60 seconds +{http_raw_disconnect_frequency, 60}. + +% Disconnect after 200 operations +{http_raw_disconnect_frequency, {ops, 200}}. +``` + +## Custom Driver + +A custom driver must expose the following callbacks. + +```erlang +% Create the worker +% ID is an integer +new(ID) -> {ok, State} or {error, Reason}. + +% Run an operation +run(Op, KeyGen, ValueGen, State) -> {ok, NewState} or {error, Reason, NewState}. +``` + +See the [existing +drivers](https://github.com/basho/basho_bench/tree/master/src) for +more details. diff --git a/content/riak/kv/2.2.6/using/performance/erlang.md b/content/riak/kv/2.2.6/using/performance/erlang.md new file mode 100644 index 0000000000..2bec7213a3 --- /dev/null +++ b/content/riak/kv/2.2.6/using/performance/erlang.md @@ -0,0 +1,367 @@ +--- +title: "Erlang VM Tuning" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Erlang VM" + identifier: "performance_erlang" + weight: 105 + parent: "managing_performance" +toc: true +aliases: + - /riak-docs/riak/2.2.6/ops/tuning/erlang + - /riak-docs/riak/kv/2.2.6/ops/tuning/erlang +--- + +Riak was written almost exclusively in [Erlang](http://www.erlang.org) +and runs on an Erlang virtual machine (VM), which makes proper Erlang VM +tuning an important part of optimizing Riak performance. The Erlang VM +itself provides a wide variety of [configurable parameters](http://erlang.org/doc/man/erl.html) that you can use to tune its performance; Riak enables you to tune a subset of those parameters in each node's [configuration files](../../../configuring/reference/#erlang-vm). + +The table below lists some of the parameters that are available, showing +both their names as used in Erlang and their names as Riak parameters. + +Erlang parameter | Riak parameter +:----------------|:-------------- +[`+A`](http://erlang.org/doc/man/erl.html#async_thread_pool_size) | `erlang.async_threads` +[`+K`](http://erlang.org/doc/man/erl.html#emu_flags) | `erlang.K` +[`+P`](http://erlang.org/doc/man/erl.html#+P) | `erlang.process_limit` +[`+Q`](http://erlang.org/doc/man/erl.html#+Q) | `erlang.max_ports` +[`+S`](http://erlang.org/doc/man/erl.html#+S) | `erlang.schedulers.total`, `erlang.schedulers.online` +[`+W`](http://erlang.org/doc/man/erl.html#emu_flags) | `erlang.W` +[`+a`](http://erlang.org/doc/man/erl.html#async_thread_stack_size) | `erlang.async_threads.stack_size` +[`+e`](http://www.erlang.org/doc/man/ets.html#+e) | `erlang.max_ets_tables` +[`+scl`](http://www.erlang.org/doc/main/erl.html#+scl) | `erlang.schedulers.compaction_of_load` +[`+sfwi`](http://www.erlang.org/doc/man/erl.html#+sfwi) | `erlang.schedulers.force_wakeup_interval` +[`-smp`](http://erlang.org/doc/man/erl.html#smp) | `erlang.smp` +[`+sub`](http://www.erlang.org/doc/man/erl.html#+sub) | `erlang.schedulers.utilization_balancing` +[`+zdbbl`](http://erlang.org/doc/man/erl.html#+zdbbl) | `erlang.distribution_buffer_size` +[`-kernel net_ticktime`](http://www.erlang.org/doc/man/kernel_app.html#net_ticktime) | `erlang.distribution.net_ticktime` +[`-env FULLSWEEP_AFTER`](http://www.erlang.org/doc/man/erlang.html#system_flag-2) | `erlang.fullsweep_after` +[`-env ERL_CRASH_DUMP`](http://www.erlang.org/doc/apps/erts/crash_dump.html) | `erlang.crash_dump` +[`-env ERL_MAX_ETS_TABLES`](http://learnyousomeerlang.com/ets) | `erlang.max_ets_tables` +`-name` | `nodename` + +{{% note title="Note on upgrading to 2.0" %}} +In versions of Riak prior to 2.0, Erlang VM-related parameters were specified +in a `vm.args` configuration file; in versions 2.0 and later, all +Erlang-VM-specific parameters are set in the `riak.conf` file. If you're +upgrading to 2.0 from an earlier version, you can still use your old `vm.args` +if you wish. Please note, however, that if you set one or more parameters in +both `vm.args` and in `riak.conf`, the settings in `vm.args` will override +those in `riak.conf`. +{{% /note %}} + +## SMP + +Some operating systems provide Erlang VMs with Symmetric Multiprocessing +capabilities +([SMP](http://en.wikipedia.org/wiki/Symmetric_multiprocessing)) for +taking advantage of multi-processor hardware architectures. SMP support +can be turned on or off by setting the `erlang.smp` parameter to +`enable` or `disable`. It is enabled by default. The following would +disable SMP support: + +```riakconf +erlang.smp = disable +``` + +Because Riak is supported on some operating systems that do not provide +SMP support. Make sure that your OS supports SMP before enabling it for +use by Riak's Erlang VM. If it does not, you should set `erlang.smp` to +`disable` prior to starting up your cluster. + +Another safe option is to set `erlang.smp` to `auto`. This will instruct +the Erlang VM to start up with SMP support enabled if (a) SMP support is +available on the current OS and (b) more than one logical processor is +detected. If neither of these conditions is met, the Erlang VM will +start up with SMP disabled. + +## Schedulers + +> **Note on missing scheduler flags** +> +> We recommend that _all_ users set the `+sfwi` to `500` (milliseconds) +and the `+scl` flag to `false` if using the older, `vm.args`-based +configuration system. If you are using the new, `riak.conf`-based +configuration system, the corresponding parameters are +`erlang.schedulers.force_wakeup_interval` and +`erlang.schedulers.compaction_of_load`. +> +> Please note that you will need to uncomment the appropriate lines in +your `riak.conf` for this configuration to take effect. + +If [SMP support](#smp) has been enabled on your Erlang +VM, i.e. if `erlang.smp` is set to `enable` or `auto` on a machine +providing SMP support _and_ more than one logical processor, you can +configure the number of logical processors, or [scheduler +threads](http://www.erlang.org/doc/man/erl.html#+S), that are created +when starting Riak, as well as the number of threads that are set +online. + +The total number of threads can be set using the +`erlang.schedulers.total` parameter, whereas the number of threads set +online can be set using `erlang.schedulers.online`. These parameters map +directly onto `Schedulers` and `SchedulersOnline`, both of which are +used by [`erl`](http://www.erlang.org/doc/man/erl.html#+S). + +While the maximum for both parameters is 1024, there is no universal +default for either. Instead, the Erlang VM will attempt to determine the +number of configured processors, as well as the number of available +processors, on its own. If the Erlang VM _can_ make that determination, +`schedulers.total` will default to the total number of configured +processors while `schedulers.online` will default to the number of +processors available; if the Erlang VM can't make that determination, +both values will default to 1. + +If either parameter is set to a negative integer, that value will be +subtracted from the default number of processors that are configured or +available, depending on the parameter. For example, if there are 100 +configured processors and `schedulers.total` is set to `-50`, then the +calculated value for `schedulers.total` will be 50. Setting either +parameter to 0, on the other hand, will reset both values to their +defaults. + +If SMP support is not enabled, i.e. if `erlang.smp` is set to `disable` +(or set to `auto` on a machine without SMP support or with only one +logical processor), then the values of `schedulers.total` and +`schedulers.online` will be ignored. + +### Scheduler Wakeup Interval + +Scheduler wakeup is an optional process whereby Erlang VM schedulers are +periodically scanned to determine whether they have "fallen asleep," +i.e. whether they have an empty [run +queue](http://en.wikipedia.org/wiki/Run_queue). The interval at which +this process occurs can be set, in milliseconds, using the +`erlang.schedulers.force_wakeup_interval` parameter, which corresponds +to the Erlang VM's `+sfwi` flag. This parameter is set to `0` by +default, which disables scheduler wakeup. + +Erlang distributions like R15Bx have a tendency to put schedulers to +sleep too often. If you are using a more recent distribution, i.e. a if +you are running Riak 2.0 or later, you most likely won't need to enable +scheduler wakeup. + +### Scheduler Compaction and Balancing + +The Erlang scheduler offers two methods of distributing load across +schedulers: **compaction of load** and **utilization balancing** of +load. + +Compaction of load is used by default. When enabled, the Erlang VM will +attempt to fully load as many scheduler threads as possible, i.e. it +will attempt to ensure that scheduler threads do not run out of work. To +that end, the VM will take into account the frequency with which +schedulers run out of work when making decisions about which schedulers +should be assigned work. You can disable compaction of load by setting +the `erlang.schedulers.compaction_of_load` setting to `false` (in the +older configuration system, set `+scl` to `true`). + +The other option, utilization balancing, is disabled by default in favor +of load balancing. When utilization balancing is enabled instead, the +Erlang VM will strive to balance scheduler utilization as equally as +possible between schedulers, without taking into account the frequency +at which schedulers run out of work. You can enable utilization +balancing by setting the `erlang.schedulers.utilization_balancing` +setting to `true` (or the `+scl` parameter to `false` in the older +configuration system). + +At any given time, only compaction of load _or_ utilization balancing +can be used. If you set both parameters to `false`, Riak will default to +using compaction of load; if both are set to `true`, Riak will enable +whichever setting is listed first in `riak.conf` (or `vm.args` if you're +using the older configuration system). + +## Port Settings + +Riak uses [epmd](http://www.erlang.org/doc/man/epmd.html), the Erlang +Port Mapper Daemon, for most inter-node communication. In this system, +other nodes in the [cluster](../../../learn/concepts/clusters) use the Erlang identifiers specified by the `nodename` parameter (or `-name` in `vm.args`), for example `riak@10.9.8.7`. On each node, the daemon resolves these node +identifiers to a TCP port. You can specify a port or range of ports for +Riak nodes to listen on as well as the maximum number of concurrent +ports/sockets. + +### Port Range + +By default, epmd binds to TCP port 4369 and listens on the wildcard +interface. epmd uses an unpredictable port for inter-node communication +by default, binding to port 0, which means that it uses the first +available port. This can make it difficult to configure [firewalls](../../security). + +To make configuring firewalls easier, you can instruct the Erlang VM to +use either a limited range of TCP ports or a single TCP port. The +minimum and maximum can be set using the +`erlang.distribution.port_range.minimum` and +`erlang.distribution.port.maximum` parameters, respectively. The +following would set the range to ports between 3000 and 5000: + +```riakconf +erlang.distribution.port_range.minimum = 3000 +erlang.distribution.port_range.maximum = 5000 +``` + +```appconfig +%% The older, app.config-based system uses different parameter names +%% for specifying the minimum and maximum port + +{kernel, [ + % ... + {inet_dist_listen_min, 3000}, + {inet_dist_listen_max, 5000} + % ... + ]} +``` + +You can set the Erlang VM to use a single port by setting the minimum to +the desired port while setting no maximum. The following would set the +port to 5000: + +```riakconf +erlang.distribution.port_range.minimum = 5000 +``` + +```appconfig +{kernel, [ + % ... + {inet_dist_listen_min, 5000}, + % ... + ]} +``` + +If the minimum port is unset, the Erlang VM will listen on a random +high-numbered port. + +### Maximum Ports + +You can set the maximum number of concurrent ports/sockets used by the +Erlang VM using the `erlang.max_ports` setting. Possible values range +from 1024 to 134217727. The default is 65536. In `vm.args` you can use +either `+Q` or `-env ERL_MAX_PORTS`. + +## Asynchronous Thread Pool + +If thread support is available in your Erlang VM, you can set the number +of asynchronous threads in the Erlang VM's asynchronous thread pool +using `erlang.async_threads` (`+A` in `vm.args`). The valid range is 0 +to 1024. If thread support is available on your OS, the default is 64. +Below is an example setting the number of async threads to 600: + +```riakconf +erlang.async_threads = 600 +``` + +```vmargs ++A 600 +``` + +### Stack Size + +In addition to the number of asynchronous threads, you can determine the +memory allocated to each thread using the +`erlang.async_threads.stack_size` parameter, which corresponds to the +`+a` Erlang flag. You can determine that size in Riak using KB, MB, GB, +etc. The valid range is 16-8192 kilowords, which translates to 64-32768 +KB on 32-bit architectures. While there is no default, we suggest a +stack size of 16 kilowords, which translates to 64 KB. We suggest such a +small size because the number of asynchronous threads, as determined by +`erlang.async_threads` might be quite large in your Erlang VM. The 64 KB +default is enough for drivers delivered with Erlang/OTP but might not be +large enough to accommodate drivers that use the `driver_async()` +functionality, documented +[here](http://www.erlang.org/doc/man/erl_driver.html). We recommend +setting higher values with caution, always keeping the number of +available threads in mind. + +## Kernel Polling + +You can utilize kernel polling in your Erlang distribution if your OS +supports it. Kernel polling can improve performance if many file +descriptors are in use; the more file descriptors, the larger an effect +kernel polling may have on performance. Kernel polling is enabled by +default on Riak's Erlang VM, i.e. the default for `erlang.K` is `on`. +This corresponds to the +[`+K`](http://erlang.org/doc/man/erl.html#emu_flags) setting on the +Erlang VM. You can disable it by setting `erlang.K` to `off`. + +## Warning Messages + +Erlang's +[`error_logger`](http://www.erlang.org/doc/man/error_logger.html) is an +event manager that registers error, warning, and info events from the +Erlang runtime. By default, events from the `error_logger` are mapped as +warnings, but you can also set messages to be mapped as errors or info +reports using the `erlang.W` parameter (or `+W` in `vm.args`). The +possible values are `w` (warnings), `errors`, or `i` (info reports). + +## Process Limit + +The `erlang.process_limit` parameter can be used to set the maximum +number of simultaneously existing system processes (corresponding to +Erlang's `+P` parameter). The valid range is 1024 to 134217727. The +default is 256000. + +## Distribution Buffer + +You can set the size of the Erlang VM's distribution buffer busy limit +(denoted by `+zdbbl` on the VM and in `vm.args`) using +`erlang.distribution_buffer_size`. Modifying this setting can be useful +on nodes with many `busy_dist_port` events, i.e. instances when the +Erlang distribution is overloaded. The default is 32 MB (i.e. `32MB`), +but this may be insufficient for some workloads. The maximum value is +2097151 KB. + +A larger buffer limit will allow processes to buffer more outgoing +messages. When the limit is reached, sending processes will be suspended +until the the buffer size has shrunk below the limit specified by +`erlang.distribution_buffer_size`. Higher values will tend to produce +lower latency and higher throughput but at the expense of higher RAM +usage. You should evaluate your RAM resources prior to increasing this +setting. + +## Erlang Built-in Storage + +Erlang uses a built-in database called +[ets](http://www.erlang.org/doc/man/ets.html) \(Erlang Term Storage) +for some processes that require fast access from memory in constant +access time (rather than logarithmic access time). The maximum number +of tables can be set using the `erlang.max_ets_tables` setting. The +default is 256000, which is higher than the default limit of 1400 on the +Erlang VM. The corresponding setting in `vm.args` is `+e`. + +Higher values for `erlang.max_ets_tables` will tend to provide more +quick-access data storage but at the cost of higher RAM usage. Please +note that the default values for `erlang.max_ets_tables` and +`erlang.distribution_size` (explained in the section [above](#distribution-buffer)) are the same. + +## Crash Dumps + +By default, crash dumps from Riak's Erlang distribution are deposited in +`./log/erl_crash.dump`. You can change this location using +`erlang.crash_dump`. This is the equivalent of setting the +[`ERL_CRASH_DUMP`](http://www.erlang.org/doc/man/erl.html#environment_variables) +environment variable for the Erlang VM. + +## Net Kernel Tick Time + +The [net kernel](http://erlang.org/doc/man/net_kernel.html) is an Erlang +system process that provides various forms of network monitoring. In a +Riak cluster, one of the functions of the net kernel is to periodically +check node liveness. **Tick time** is the frequency with which those +checks happen. You can determine that frequency using the +`erlang.distribution.net_ticktime`. The tick will occur every N seconds, +where N is the value set. Thus, setting +`erlang.distribution.net_ticktime` to `60` will make the tick occur once +every minute. The corresponding flag in `vm.args` is `-kernel +net_ticktime`. + +## Shutdown Time + +You can determine how long the Erlang VM spends shutting down using the +`erlang.shutdown_time` parameter. The default is `10s` (10 seconds). +Once this duration elapses, all existing processes are killed. +Decreasing shutdown time can be useful in situations in which you are +frequently starting and stopping a cluster, e.g. in test clusters. In +`vm.args` you can set the `-shutdown_time` flag in milliseconds. diff --git a/content/riak/kv/2.2.6/using/performance/latency-reduction.md b/content/riak/kv/2.2.6/using/performance/latency-reduction.md new file mode 100644 index 0000000000..5791a3c49e --- /dev/null +++ b/content/riak/kv/2.2.6/using/performance/latency-reduction.md @@ -0,0 +1,263 @@ +--- +title: "Latency Reduction Checklist" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Latency Reduction" + identifier: "performance_latency_reduction" + weight: 104 + parent: "managing_performance" +toc: true +aliases: + - /riak-docs/riak/2.2.6/ops/tuning/latency-reduction + - /riak-docs/riak/kv/2.2.6/ops/tuning/latency-reduction +--- + +Although latency is unavoidable in distributed systems like Riak, there +are a number of actions that can be undertaken to reduce latency +to the lowest levels possible within a cluster. In this guide, we'll +list potential sources of high latency and what you can do about it. + +## Large Objects + +Riak always performs best with smaller objects. Large objects, which can +be mistakenly inserted into Riak by your application or caused by +siblings (see below), can often increase latency. + +We recommend keeping all objects stored in Riak smaller than 1-2 MB, +preferably below 100 KB. Large objects lead to increased I/O activity +and can put strain on memory resources. In some cases, just a few large +objects can impact latency in a cluster, even for requests that are +unrelated to those objects. + +If your use case requires large objects, we recommend checking out +[Riak CS]({{<baseurl>}}riak/cs/latest/), which is intended as a storage system for large objects. + +### Mitigation + +The best way to find out if large objects are impacting latency is to +monitor each node's object size stats. If you run [`riak-admin status`](../../admin/riak-admin/#status) or make an HTTP `GET` request +to Riak's `/stats` endpoint, you will see the results for the following +metrics related to object size, all of which are calculated only for +`GET` operations (i.e. reads): + +Metric | Explanation +:-----------------------------|:----------- +`fsm_node_get_objsize_mean` | The mean object size encountered by this node in the last minute +`fsm_node_get_objsize_median` | The median object size encountered by this node in the last minute +`fsm_node_get_objsize_95` | The 95th-percentile object size encountered by this node in the last minute +`fsm_node_get_objsize_99` | The 99th-percentile object size encountered by this node in the last minute +`fsm_node_get_objsize_100` | The 100th-percentile object size encountered by this node in the last minute + +The `mean` and `median` measurements may not be good indicators, +especially if you're storing billions of keys. Instead, you should be on +the lookout for trends in the `95`, `99`, and `100` measures: + +* Is there an upward trend? +* Do the metrics indicate that there are outliers? +* Do these trends coincide with increased latency? + +If you suspect that large object size is impacting latency, try making +the following changes to each node's [configuration](../../../configuring/reference): + +* If you are using the newer, `riak.conf`-based configuration system, +the commented-out value for `erlang.distribution_buffer_size` is `32MB`. +Uncomment this setting and re-start your node. +* If you are using the older, `app.config`/`vm.args`-based configuration +system, try increasing the `+zddbl` setting in `vm.args` to `32768` or +higher (measured in kilobytes). This increases the size of the +distributed Erlang buffer from its default of 1024 KB. Re-start your +node when configuration changes have been made. + +Large objects can also impact latency even if they're only present on +some nodes. If increased latency occurs only on N nodes, where N is your +[replication factor](../../../developing/app-guide/replication-properties/#n-value-and-replication), also known as `n_val`, this could indicate that a single large object and its replicas are slowing down _all_ requests on those nodes. + +If large objects are suspected, you should also audit the behavior of +siblings in your cluster, as explained in the [next section](#siblings). + +## Siblings + +In Riak, object conflicts are handled by keeping multiple versions of +the object in the cluster either until a client takes action to resolve +the conflict or until [active anti-entropy](../../../learn/glossary/#active-anti-entropy) resolves the conflict without client intervention. While sibling production is normal, [sibling explosion](../../../learn/concepts/causal-context/#sibling-explosion) is a problem that can come about if many siblings of an object are produced. The negative effects are the same as those associated with [large objects](#large-objects). + +### Mitigation + +The best way to monitor siblings is through the same [`riak-admin status`](../../admin/riak-admin/#status) interface used to monitor +object size (or via an HTTP `GET` request to `/stats`). In the output of +`riak-admin status` in each node, you'll see the following +sibling-related statistics: + +Metric | Explanation +:------------------------------|:----------- +`node_get_fsm_siblings_mean` | The mean number of siblings encountered during all GET operations by this node within the last minute +`node_get_fsm_siblings_median` | The median number of siblings encountered during all GET operations by this node within the last minute +`node_get_fsm_siblings_95` | The 95th percentile of the number of siblings encountered during all GET operations by this node within the last minute +`node_get_fsm_siblings_99` | The 99th percentile of the number of siblings encountered during all GET operations by this node within the last minute +`node_get_fsm_siblings_100` | The 100th percentile of the number of siblings encountered during all GET operations by this node within the last minute + +Is there an upward trend in these statistics over time? Are there any +large outliers? Do these trends correspond to your observed latency +spikes? + +If you believe that sibling creation problems could be responsible for +latency issues in your cluster, you can start by checking the following: + +* If `allow_mult` is set to `true` for some or all of your buckets, be + sure that your application is correctly resolving siblings. Be sure to + read our documentation on [conflict resolution](../../../developing/usage/conflict-resolution) for a fuller picture of how this can be done. **Note**: In Riak versions 2.0 and later, `allow_mult` is set to `true` by default for all bucket types that you create and activate. + If you wish to set `allow_mult` to `false` on a bucket type, you will have to do so explicitly. +* Application errors are a common source of problems with + siblings. Updating the same key over and over without passing a + [causal context](../../../learn/concepts/causal-context) to Riak can cause sibling explosion. If this seems to be the issue, modify your application's [conflict resolution](../../../developing/usage/conflict-resolution) + strategy. Another possibility worth exploring is using [dotted version vectors](../../../learn/concepts/causal-context/#dotted-version-vectors) \(DVVs) in place of traditional vector clocks. DVVs can be enabled [using bucket types](../../../developing/usage/bucket-types) by setting the `dvv_enabled` parameter to `true` for buckets that seem to be experiencing sibling explosion. + +## Compaction and Merging + +The [Bitcask](../../../setup/planning/backend/bitcask) and [LevelDB](../../../setup/planning/backend/leveldb) storage backends occasionally go through +heavily I/O-intensive compaction phases during which they remove deleted +data and reorganize data files on disk. During these phases, affected +nodes may be slower to respond to requests than other nodes. If your +cluster is using one or both of these backends, there are steps that can +be taken to monitor and address latency issues. + +### Mitigation + +To determine whether compaction and merging cycles align with increased +latency, keep an eye on on your `console.log` files (and LevelDB `LOG` +files if you're using LevelDB). Do Bitcask merging and/or LevelDB +compaction events overlap with increased latencies? + +If so, our first recommendation is to examine your [replication properties](../../../developing/app-guide/replication-properties/) to make sure that neither R nor W are set to N, i.e. that you're not requiring that reads or writes go to all nodes in the cluster. The problem with setting `R=N` or `W=N` is that any request will only respond as quickly as the slowest node amongst the N nodes involved in the request. + +Beyond checking for `R=N` or `W=N` for requests, the recommended +mitigation strategy depends on the backend: + +#### Bitcask + +With Bitcask, it's recommended that you: + +* Limit merging to off-peak hours to decrease the effect of merging +cycles on node traffic +* Stagger merge windows between nodes so that no more than one node is +undergoing a merge phase at any given time + +Instructions on how to accomplish both can be found in our guide to +[tuning Bitcask](../../../setup/planning/backend/bitcask/#tuning-bitcask). + +It's also important that you adjust your maximum file size and merge +threshold settings appropriately. This setting is labeled +`bitcask.max_file_size` in the newer, `riak.conf`-based [configuration files](../../../configuring/reference) and `max_file_size` in the older, `app.config`-based system. + +Setting the maximum file size lower will cause Bitcask to merge more +often (with less I/O churn), while setting it higher will induce less +frequent merges with more I/O churn. To find settings that are ideal for +your use case, we recommend checking out our guide to [configuring Bitcask](../../../setup/planning/backend/bitcask/#configuring-bitcask). + +#### LevelDB + +The more files you keep in memory, the faster LevelDB will perform in +general. To make sure that you are using your system resources +appropriately with LevelDB, check out our guide to [LevelDB parameter planning](../../../setup/planning/backend/leveldb/#parameter-planning). + +## OS Tuning + +While a number of latency-related problems can manifest themselves in +development and testing environments, some performance limits only +become clear in production environments. + +### Mitigation + +If you suspect that OS-level issues might be impacting latency, it might +be worthwhile to revisit your OS-specific configurations. The following +guides may be of help: + +* [Open files limit](../open-files-limit) +* General [System performance tuning](../) +* [AWS performance tuning](../amazon-web-services) if you're running Riak on [Amazon Web Services](http://aws.amazon.com/) + +## I/O and Network Bottlenecks + +Riak is a heavily I/O- and network resource-intensive system. +Bottlenecks on either front can lead to undue latency in your cluster. +We recommend an active monitoring strategy to detect problems +immediately when they arise. + +### Mitigation + +To diagnose potential I/O bottlenecks, there are a number of Linux tools +at your disposal, including +[iowait](http://www.linuxquestions.org/questions/linux-newbie-8/what-is-iowait-415961/) +and [netstat](http://en.wikipedia.org/wiki/Netstat). + +To diagnose potential overloads, Riak versions 1.3.2 and later come +equipped with an overload protection feature designed to prevent +cascading failures in overly busy nodes. This feature limits the number +of GET and PUT finite state machines (FSMs) that can exist +simultaneously on a single Riak node. Increased latency can result if a +node is frequently running up against these maximums. + +* Monitor `node_get_fsm_active` and `node_get_fsm_active_60s` to get an + idea of how many operations your nodes are coordinating. If you see + non-zero values in `node_get_fsm_rejected` or + `node_get_fsm_rejected_60s`, that means that some of your requests are + being discarded due to overload protection. +* The FSM limits can be increased, but disabling overload protection + entirely is not recommended. More details on these settings are + available in the [release + notes](https://github.com/basho/riak/blob/1.3/RELEASE-NOTES.md) for + Riak version 1.3. + +## Object Settings + +In versions 2.0 and later, Riak enables you to configure a variety of +settings regarding Riak objects, including allowable object sizes, how +many [siblings](../../../learn/concepts/causal-context/#siblings) to allow, and so on. If you suspect that undue latency in your cluster stems from object size or related factors, you may consider adjusting these settings. + +A concise listing of object-related settings can be found in the [Riak configuration](../../../configuring/reference/#object-settings) documentation. The sections below explain these settings in detail. + +> **Note on configuration files in 2.0** +> +> The object settings listed below are only available using the new system +for [configuration files](../../../configuring/reference/) in Riak 2.0. If you are using the older, `app.config`-based system, you will not have access to +these settings. + +### Object Size + +As stated above, we recommend _always_ keeping objects below 1-2 MB +and preferably below 100 KB if possible. If you want to ensure that +objects above a certain size do not get stored in Riak, you can do so by +setting the `object.size.maximum` parameter lower than the default of +`50MB`, which is far above the ideal object size. If you set this +parameter to, say, `1MB` and attempt to store a 2 MB object, the write +will fail and an error message will be returned to the client. + +You can also set an object size threshold past which a write will +succeed but will register a warning in the logs, you can adjust the +`object.size.warning_threshold` parameter. The default is `5MB`. + +### Sibling Explosion Management + +In order to prevent or cut down on [sibling explosion](../../../learn/concepts/causal-context/#sibling explosion), you can either prevent Riak from storing +additional siblings when a specified sibling count is reached or set a +warning threshold past which Riak logs an error (or both). This can be +done using the `object.siblings.maximum` and +`object.siblings.warning_threshold` settings. The default maximum is 100 +and the default warning threshold is 25. + +### Object Storage Format + +There are currently two possible binary representations for objects +stored in Riak: + +* Erlang's native `term_to_binary` format, which tends to have a higher + space overhead +* A newer, Riak-specific format developed for more compact storage of + smaller values + +You can set the object storage format using the `object.format` +parameter: `0` selects Erlang's `term_to_binary` format while `1` (the +default) selects the Riak-specific format. diff --git a/content/riak/kv/2.2.6/using/performance/multi-datacenter-tuning.md b/content/riak/kv/2.2.6/using/performance/multi-datacenter-tuning.md new file mode 100644 index 0000000000..f7ec83749f --- /dev/null +++ b/content/riak/kv/2.2.6/using/performance/multi-datacenter-tuning.md @@ -0,0 +1,42 @@ +--- +title_supertext: "Multi Data Center Replication:" +title: "System Tuning" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Multi-Datacenter Replication" + identifier: "performance_multi_datacenter_tuning" + weight: 110 + parent: "managing_performance" +toc: true +commercial_offering: true +--- + +[perf index]: {{<baseurl>}}riak/kv/2.2.6/using/performance + +Depending on the size of your objects and your replication latency +needs, you may need to configure your kernel settings to optimize +throughput. + +## Linux + +Refer to the [System Performance Tuning][perf index] document. + +## Solaris + +On Solaris, the following settings are suggested: + +```bash +/usr/sbin/ndd -set /dev/tcp tcp_ip_abort_interval 60000 +/usr/sbin/ndd -set /dev/tcp tcp_keepalive_interval 900000 +/usr/sbin/ndd -set /dev/tcp tcp_rexmit_interval_initial 3000 +/usr/sbin/ndd -set /dev/tcp tcp_rexmit_interval_max 10000 +/usr/sbin/ndd -set /dev/tcp tcp_rexmit_interval_min 3000 +/usr/sbin/ndd -set /dev/tcp tcp_time_wait_interval 60000 +/usr/sbin/ndd -set /dev/tcp tcp_max_buf 4000000 +/usr/sbin/ndd -set /dev/tcp tcp_cwnd_max 4000000 +/usr/sbin/ndd -set /dev/tcp tcp_xmit_hiwat 4000000 +/usr/sbin/ndd -set /dev/tcp tcp_recv_hiwat 4000000 +``` diff --git a/content/riak/kv/2.2.6/using/performance/open-files-limit.md b/content/riak/kv/2.2.6/using/performance/open-files-limit.md new file mode 100644 index 0000000000..718d21fa0b --- /dev/null +++ b/content/riak/kv/2.2.6/using/performance/open-files-limit.md @@ -0,0 +1,347 @@ +--- +title: "Open Files Limit" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Open Files Limit" + identifier: "performance_open_files_limit" + weight: 101 + parent: "managing_performance" +toc: true +aliases: + - /riak-docs/riak/2.2.6/ops/tuning/open-files-limit/ + - /riak-docs/riak/kv/2.2.6/ops/tuning/open-files-limit/ +--- + +[plan backend]: {{<baseurl>}}riak/kv/2.2.6/setup/planning/backend/ +[blog oracle]: http://blogs.oracle.com/elving/entry/too_many_open_files + +Riak KV can accumulate a large number of open file handles during operation. The creation of numerous data files is normal, and the [backend][plan backend] performs periodic merges of data file collections to avoid accumulating file handles. + +To accomodate this you should increase the open files limit on your system. We recommend setting a soft limit of 65536 and a hard limit of 200000. + +{{% note %}} +Superuser or root access may be required to perform these steps. +{{% /note %}} + +## Changing Limit For Current Session + +Most operating systems can change the open-files limit for the current shell session using the `ulimit -n` command: + +```bash +ulimit -n 200000 +``` + +## Debian & Ubuntu + +Start by checking the current open file limit values with: + +```bash +ulimit -Hn # Hard limit +ulimit -Sn # Soft limit +``` + +If you installed Riak KV from a binary package, you will need to the add the following settings to the /etc/security/limits.conf file for the `riak` user: + +```/etc/security/limits.conf +riak soft nofile 65536 +riak hard nofile 200000 +``` + +If you use initialization scripts to start Riak KV, you can create a /etc/default/riak file and add the following to specify a limit: + +```/etc/default/riak +ulimit -n 200000 +``` + +This file is automatically sourced from the initialization script, and the Riak KV process will inherit this setting. Since initialization scripts are always run as the root user, there’s no need to set limits in /etc/security/limits.conf. + +## Enable PAM-Based Limits for Debian & Ubuntu + +You can enable PAM-based user limits so that non-root users, such as the `riak` user, may specify a higher value for maximum open files. + +For example, follow these steps to enable PAM-based limits for all users to allow a maximum of 200000 open files. + +1\. Edit /etc/pam.d/common-session and add the following line: + +```/etc/pam.d/common-session +session required pam_limits.so +``` + +2\. Save and close the file. If /etc/pam.d/common-session-noninteractive exists, append the same line as above. + +3\. Edit /etc/security/limits.conf and append the following lines to the file: + +```/etc/security/limits.conf +* soft nofile 65536 +* hard nofile 200000 +``` + +4\. Save and close the file. + +5\. (**Optional**) If you will be accessing the Riak KV nodes via secure shell (SSH), you should also edit /etc/ssh/sshd_config and uncomment the following line: + +```/etc/ssh/sshd_config +#UseLogin no +``` + +And set its value to `yes` as shown here: + +```/etc/ssh/sshd_config +UseLogin yes +``` + +6\. Restart the machine so the limits take effect and verify that the new limits are set with the following command: + +```bash +ulimit -a +``` + +{{% note %}} +In the above examples, the open files limit is raised for all users of the system. The limit can be specified for the `riak` user only by substituting the +two asterisks (`*`) in the examples with `riak`. +{{% /note %}} + + +## CentOS & Red Hat + +Start by checking the current open file limit values with: + +```bash +ulimit -Hn # Hard limit +ulimit -Sn # Soft limit +``` + +If you installed Riak KV from a binary package, you will need to the add the following settings to the /etc/security/limits.conf file for the `riak` user: + +```/etc/security/limits.conf +riak soft nofile 65536 +riak hard nofile 200000 +``` + +If you use initialization scripts to start Riak KV, you can create a /etc/default/riak file and add the following to specify a limit: + +```/etc/default/riak +ulimit -n 200000 +``` + +This file is automatically sourced from the initialization script, and the Riak KV process will inherit this setting. Since initialization scripts are always run as the root user, there’s no need to set limits in /etc/security/limits.conf. + +## Enable PAM-Based Limits for CentOS and Red Hat + +You can enable PAM-based user limits so that non-root users, such as the `riak` user, may specify a higher value for maximum open files. + +For example, follow these steps to enable PAM-based limits for all users to allow a maximum of 200000 open files. + +1\. Edit /etc/pam.d/login and add the following line: + +```/etc/pam.d/login +session required pam_limits.so +``` + +2\. Save and close /etc/pam.d/login + +3\. Edit /etc/security/limits.conf and append the following lines to the file: + +```/etc/security/limits.conf +* soft nofile 65536 +* hard nofile 200000 +``` + +4\. Save and close the /etc/security/limits.conf file. + +5\. Restart the machine so that the limits to take effect and verify that +the new limits are set with the following command: + +```bash +ulimit -a +``` + +{{% note %}} +In the above examples, the open files limit is raised for all users of the system. The limit can be specified for the `riak` user only by substituting the +two asterisks (`*`) in the examples with `riak`. +{{% /note %}} + + +## Solaris + +To increase the open file limit on Solaris, add the following line to the /etc/system file: + +```/etc/system +set rlim_fd_max=200000 +``` + +[Reference][blog oracle] + +## macOS Sierra and High Sierra + +Start by checking the current open file limit values with: + +```bash +launchctl limit maxfiles +``` + +The response should look something like this: + +```bash +maxfiles 65536 65536 +``` +The first column is the soft limit and the last column is the hard limit. + +To change the open files limits on macOS Sierra or High Sierra, perform the following steps: + +1\. Add the following line to your .bash\_profile or analogous file: + +```bash +ulimit -n 65536 200000 +``` + +2\. Save and close the file. Next create the file /Library/LaunchDaemons/limit.maxfiles.plist (owned by `root` in the group `wheel` with the mode `0644`). In it place the following XML: + +``` +<?xml version="1.0" encoding="UTF-8"?> +<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" + "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> + +<plist version="1.0"> + <dict> + <key>Label</key> + <string>limit.maxfiles</string> + <key>ProgramArguments</key> + <array> + <string>launchctl</string> + <string>limit</string> + <string>maxfiles</string> + <string>65536</string> + <string>200000</string> + </array> + <key>RunAtLoad</key> + <true/> + <key>ServiceIPC</key> + <false/> + </dict> +</plist> + +``` + +3\. Save and close the file. + +4\. Restart your computer and enter `ulimit -n` into your terminal. If your system is configured correctly, you should see that `maxfiles` has been set to 200000. + +## Mac OS X El Capitan + +Start by checking the current open file limit values with: + +```bash +launchctl limit maxfiles +``` + +The response should look something like this: + +```bash +maxfiles 65536 65536 +``` + +The first column is the soft limit and the last column is the hard limit. + +To change the open files limits on Mac OS X El Capitan, perform the following steps: + +1\. Add the following line to your .bash_profile or analogous file: + +```bash +ulimit -n 65536 200000 +``` + +2\. Save and close the file. Next open /etc/sysctl.conf (or create it if it doesn't already exist) and add the following settings: + +```/etc/sysctl.conf +kern.maxfiles=200000 +kern.maxfilesperproc=200000 +``` + +4\. Restart your computer and enter `ulimit -n` into your terminal. If your system is configured correctly, you should see that `maxfiles` has been set to 200000. + + +## Mac OS X Yosemite + +Start by checking the current open file limit values with: + +```bash +launchctl limit maxfiles +``` + +The response should look something like this: + +```bash +maxfiles 65536 65536 +``` + +The first column is the soft limit and the last column is the hard limit. + +To change the open files limits on Mac OS X Yosemite, perform these steps: + +1\. Add the following line to your .bash_profile or analogous file: + +```bash +ulimit -n 65536 200000 +``` + +2\. Save and close the file. Next edit the /etc/launchd.conf file and add: + +```/etc/launchd.conf +limit maxfiles 200000 +``` + +3\. Save and close the file. + +4\. After restarting, verify the new limits by running: + +```bash +launchctl limit maxfiles +``` + +The response output should look something like this: + +```bash +maxfiles 65536 200000 +``` + +## Mac OS X Older Versions + +Start by checking the current open file limit values with: + +```bash +launchctl limit maxfiles +``` + +The response should look something like this: + +```bash +maxfiles 10240 10240 +``` + +The first column is the soft limit and the last column is the hard limit. + +To adjust the maximum open file limits in OS X 10.7 (Lion) up to but not including OS X Yosemite, perform the following steps: + +1\. Edit (or create) /etc/launchd.conf and increase the limits by adding: + +```bash +limit maxfiles 65536 200000 +``` + +2\. Save the file and restart the system for the new limits to take effect. + +3\. After restarting, verify the new limits by running: + +```bash +launchctl limit maxfiles +``` + +The response output should look something like this: + +```bash +maxfiles 65536 200000 +``` diff --git a/content/riak/kv/2.2.6/using/performance/v2-scheduling-fullsync.md b/content/riak/kv/2.2.6/using/performance/v2-scheduling-fullsync.md new file mode 100644 index 0000000000..50eaf22ae9 --- /dev/null +++ b/content/riak/kv/2.2.6/using/performance/v2-scheduling-fullsync.md @@ -0,0 +1,45 @@ +--- +title: "V2 Scheduling Fullsync" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "V2 Scheduling Fullsync" + identifier: "performance_v2_scheduling_fullsync" + weight: 103 + parent: "managing_performance" +toc: true +commercial_offering: true +--- + +{{% note title="Deprecation Warning" %}} +v2 Multi-Datacenter Replication is deprecated and will be removed in a future version. Please use [v3]({{<baseurl>}}riak/kv/2.2.6/using/cluster-operations/v3-multi-datacenter/#fullsync-replication-commands/) instead. +{{% /note %}} + + +With the `pause` and `resume` commands it is possible to limit the +fullsync operation to off-peak times. First, disable `fullsync_interval` +and set `fullsync_on_connect` to `false`. Then, using cron or something +similar, execute the commands below at the start of the sync window. +In these examples, the commands are combined in a `.sh` or analogous +file: + +```bash +#!/bin/sh + +## Resume from where we left off +riak-repl resume-fullsync + +## Start fullsync if nothing is running +riak-repl start-fullsync +``` + +At the end of the sync window: + +```bash +#!/bin/sh + +## Stop fullsync until start of next sync window +riak-repl pause-fullsync +``` diff --git a/content/riak/kv/2.2.6/using/reference.md b/content/riak/kv/2.2.6/using/reference.md new file mode 100644 index 0000000000..b592a18277 --- /dev/null +++ b/content/riak/kv/2.2.6/using/reference.md @@ -0,0 +1,130 @@ +--- +title: "Riak KV Usage Reference" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Reference" + identifier: "managing_ref" + weight: 208 + parent: "managing" +toc: true +--- + +[ref log]: ./logging +[ref handoff]: ./handoff +[ref bucket types]: ./bucket-types +[ref obj del]: ./object-deletion/ +[ref runtime]: ./runtime-interaction/ +[ref monitoring]: ./statistics-monitoring +[ref snmp]: ./snmp +[ref jmx]: ./jmx +[ref search]: ./search +[ref 2i]: ./secondary-indexes +[ref custom code]: ./custom-code +[ref strong consistency]: ./strong-consistency +[ref mdc]: ./multi-datacenter +[ref v3 mdc]: ./v3-multi-datacenter +[ref v2 mdc]: ./v2-multi-datacenter +[ref arch]: ./architecture + +## In This Section + +#### [Logging Reference][ref log] + +Overview of logging in Riak KV. + +[Learn More >>][ref log] + + +#### [Handoff Reference][ref handoff] + +Details Riak KV's handoff system. + +[Learn More >>][ref handoff] + + +#### [Bucket Types Reference][ref bucket types] + +Explanation of bucket types in Riak KV. + +[Learn More >>][ref bucket types] + + +#### [Object Deletion Reference][ref obj del] + +Information on object deletion scenarios and tombstones. + +[Learn More >>][ref obj del] + + +#### [Runtime Interaction Reference][ref runtime] + +Describes the how Riak interacts with distribution ports and operating system +processes/garbage collection. + +[Learn More >>][ref runtime] + + +#### [Statistics & Monitoring Reference][ref monitoring] + +Presents commonly monitored & gathered statistics, as well as solutions for monitoring and gathering statistics. + +[Learn More >>][ref monitoring] + + +#### [Simple Network Management Protocol][ref snmp] + +Cover's Riak Enterprise's deprecated SNMP server used allow an external system to query nodes for statistics. + +[Learn More >>][ref snmp] + + +#### [JMX Monitoring][ref jmx] + +Details Riak KV's deprecated JMX monitoring system. + +[Learn More >>][ref jmx] + + +#### [Search Reference][ref search] + +Overview of search in Riak KV. + +[Learn More >>][ref search] + + +#### [Secondary Indexes Reference][ref 2i] + +Implementation details for Riak KV's secondary indexes feature + +[Learn More >>][ref 2i] + + +#### [Installing Custom Code][ref custom code] + +Steps for installing custom code modules for pre/post-commit hooks and MapReduce operations. + +[Learn More >>][ref custom code] + + +#### [Strong Consistency Reference][ref strong consistency] + +Overview of strong consistency in Riak KV. + +[Learn More >>][ref strong consistency] + + +#### [Multi-Datacenter Reference][ref mdc] + +Overview of Riak's Multi-Datacenter system. + +[Learn More >>][ref mdc] + + +#### [V3 Multi-Datacenter Replication Reference][ref v3 mdc] + +Details Riak's V3 Multi-Datacenter system. + +[Learn More >>][ref v3 mdc] diff --git a/content/riak/kv/2.2.6/using/reference/architecture.md b/content/riak/kv/2.2.6/using/reference/architecture.md new file mode 100644 index 0000000000..ecfae7c2dd --- /dev/null +++ b/content/riak/kv/2.2.6/using/reference/architecture.md @@ -0,0 +1,16 @@ +--- +draft: true +title: "Architecture Reference" +description: "" +project: "riak_kv" +project_version: "2.2.6" +#menu: +# riak_kv-2.2.6: +# name: "Architecture" +# identifier: "managing_ref_architecture" +# weight: 116 +# parent: "managing_ref" +toc: true +--- + +<!-- TODO: Content --> diff --git a/content/riak/kv/2.2.6/using/reference/bucket-types.md b/content/riak/kv/2.2.6/using/reference/bucket-types.md new file mode 100644 index 0000000000..35ea693cc6 --- /dev/null +++ b/content/riak/kv/2.2.6/using/reference/bucket-types.md @@ -0,0 +1,818 @@ +--- +title: "Bucket Types" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Bucket Types" + identifier: "managing_ref_bucket_types" + weight: 102 + parent: "managing_ref" +toc: true +--- + +Bucket types allow groups of buckets to share configuration details and +for Riak users to manage bucket properties more efficiently than in the +older configuration system based on [bucket properties]({{<baseurl>}}riak/kv/2.2.6/developing/usage/bucket-types/#bucket-properties-and-operations). + +{{% note title="Important note on cluster downgrades" %}} +If you upgrade a Riak to version 2.0 or later, you can still downgrade the +cluster to a pre-2.0 version _as long as you have not created and activated a +bucket type in the cluster_. Once any bucket type has been created and +activated, you can no longer downgrade the cluster to a pre-2.0 version. +{{% /note %}} + +## How Bucket Types Work + +The older configuration system, based on bucket properties, involves +setting bucket properties for specific buckets either through +[HTTP]({{<baseurl>}}riak/kv/2.2.6/developing/api/http/set-bucket-props) or [Protocol Buffers]({{<baseurl>}}riak/kv/2.2.6/developing/api/protocol-buffers/set-bucket-props). With this approach, you can take any given bucket and +modify a wide range of properties, from `n_val` to `allow_mult` and far +beyond. + +Using bucket *types* also involves dealing with bucket properties, but +with a few crucial differences: + +* Bucket types enable you to create bucket configurations and assign + those configurations to as many buckets as you wish, whereas the + previous system required configuration to be set on a per-bucket basis +* Nearly all bucket properties can be updated using bucket types, except the + `datatype` and `consistent` properties, related to + [Riak data types]({{<baseurl>}}riak/kv/2.2.6/developing/data-types), and [strong consistency]({{<baseurl>}}riak/kv/2.2.6/developing/app-guide/strong-consistency) respectively +* Bucket types are more performant than bucket properties because + divergence from Riak's defaults doesn't have to be gossiped around the + cluster for every bucket, which means less computational overhead + +It is important to note that buckets are not assigned types in the same +way that they are configured when using [bucket properties]({{<baseurl>}}riak/kv/2.2.6/developing/usage/bucket-types/#bucket-properties-and-operations). You cannot simply take a +bucket `my_bucket` and assign it a type the way that you would, say, +set `allow_mult` to `false` or `n_val` to `5`, because there is no +`type` parameter contained within the bucket's properties (i.e. +`props`). + +Instead, bucket types are applied to buckets _on the basis of how those +buckets are queried_. Queries involving bucket types take the following +form: + +``` +GET/PUT/DELETE /types/<type>/buckets/<bucket>/keys/<key> +``` + +In the older system, only bucket and key are specified in queries: + +``` +GET/PUT/DELETE /buckets/<bucket>/keys/<key> +``` + +## When to Use Bucket Types + +In many respects, bucket types are a major improvement over the older +system of bucket configuration, including the following: + +* Bucket types are more flexible because they enable you to define a + bucket configuration and then change it if you need to. +* Bucket types are more reliable because the buckets that bear a given + type only have their properties changed when the type is changed. + Previously, it was possible to change the properties of a bucket only + through client requests. +* Whereas bucket properties can only be altered by clients interacting + with Riak, bucket types are more of an operational concept. The + `riak-admin bucket-type` interface (discussed in depth below) enables + you to manage bucket configurations on the operations side, without + recourse to Riak clients. + +For these reasons, we recommend _always_ using bucket types in versions +of Riak 2.0 and later. + +## Managing Bucket Types Through the Command Line + +Bucket types are created, updated, activated, and more through the +`riak-admin bucket-type` interface. + +Below is a full list of available sub-commands: + +Command | Action | Form | +:-------|:-------|:-----| +`create` | Create or modify a bucket type before activation | `create <type> <json>` | +`activate` | Activate a bucket type | `activate <type>` | +`list` | List all currently available bucket types and their activation status | `list` | +`status` | Display the status and properties of a specific bucket type | `status <type>` | +`update` | Update a bucket type after activation | `update <type> <json>` | + +### Creating a Bucket Type + +Creating new bucket types involves using the `create <type> <json>` +command, where `<type>` is the name of the type and `<json>` is a JSON +object of the following form: + +```json +{ + "props": { + "prop1": "val1", + "prop2": "val2", + ... + } +} +``` + + +> **Getting started with Riak clients** +> +> If you are connecting to Riak using one of Basho's official [client libraries]({{<baseurl>}}riak/kv/2.2.6/developing/client-libraries), you can find more information about getting started with your client in our [Developing with Riak KV: Getting Started]({{<baseurl>}}riak/kv/2.2.6/developing/getting-started) section. + +If creation is successful, you should see the following output: + +``` +type_using_defaults created +``` + +{{% note %}} +The `create` command can be run multiple times prior to a bucket type being +activated. Riak will persist only those properties contained in the final call +of the command. +{{% /note %}} + +Creating bucket types that assign properties _always_ involves passing +stringified JSON to the `create` command. One way to do that is to pass +a JSON string directly. The following creates a bucket type +`n_equals_1`, which sets `n_val` to 1: + +```bash +riak-admin bucket-type create n_equals_1 '{"props":{"n_val":1}}' +``` + +If you wish, you can also pass in a JSON string through a file, such as +a `.json` file: + +```bash +riak-admin bucket-type create from_json_file '`cat props.json`' +``` + +Like all bucket types, this type needs to be activated to be usable +within the cluster. + +### Activating a Bucket Type + +Activating a bucket type involves the `activate` command from the same +`bucket-type` interface used before: + +```bash +riak-admin bucket-type activate my_bucket_type +``` + +When activation has succeeded, you should see the following output: + +``` +my_bucket_type has been activated +``` + +A bucket type can be activated only when the type has been propagated to +all running nodes. You can check on the type's readiness by running +`riak-admin bucket-type status <type_name>`. The first line of output +will indicate whether or not the type is ready. + +In a stable cluster, bucket types should propagate very quickly. If, +however, a cluster is experiencing network partitions or other issues, +you will need to resolve those issues before bucket types can be +activated. + +### Listing Bucket Types + +You can list currently available bucket types using the `list` command: + +```bash +riak-admin bucket-type list +``` + +This will return a simple list of types along with their current status +(either `active` or `not active`). Here is an example console output: + +```bash +riak-admin bucket-type list +``` + +An example response: + +``` +type1 (active) +type2 (not active) +type3 (active) +``` + +### Checking a Type's Status + +You can check on the status---i.e. the configuration details---of a +bucket type using the `status <type>` command: + +```bash +riak-admin bucket-type status my_bucket_type +``` + +The console will output two things if the type exists: + +1. Whether or not the type is active +2. The bucket properties associated with the type + +If you check the status of a currently active type called +`my_bucket_type` that simply bears a default bucket configuration, the +output will be as follows: + +```bash +my_bucket_type is active + +active: true +allow_mult: true + +... other properties ... + +w: quorum +young_vclock:20 +``` + +### Updating a Bucket Type + +The `bucket-type update` command functions much like the `bucket-type +create` command. It simply involves specifying the name of the bucket +type that you wish to modify and a JSON object containing the properties +of the type: + +```bash +riak-admin bucket-type update type_to_update '{"props":{ ... }}' +``` + +{{% note title="Immutable Configurations" %}} +Any bucket properties associated with a type can be modified after a bucket is +created, with three important exceptions: + +* `consistent` +* `datatype` +* `write_once` + +If a bucket type entails strong consistency (requiring that `consistent` be +set to `true`), is set up as a `map`, `set`, or `counter`, or is defined as a +write-once bucket (requiring `write_once` be set to `true`), then this will +be true of the bucket types. + +If you need to change one of these properties, we recommend that you simply +create and activate a new bucket type. +{{% /note %}} + +## Buckets as Namespaces + +In versions of Riak prior to 2.0, all queries are made to a bucket/key +pair, as in the following example read request: + +```java +Location myKey = new Location(new Namespace("my_bucket"), "my_key"); +FetchValue fetch = new FetchValue.Builder(myKey).build(); +client.execute(fetch); +``` + +```ruby +bucket = client.bucket('my_bucket') +bucket.get('my_key') +``` + +```php +$location = new Location('my_key', new Bucket('my_bucket')); +(new \Basho\Riak\Command\Builder\FetchObject($riak)) + ->atLocation($location) + ->build() + ->execute(); +``` + +```python +bucket = client.bucket('my_bucket') +bucket.get('my_key') +``` + +```csharp +var id = new RiakObjectId("my_bucket", "my_key"); +client.Get(id); +``` + +```javascript +client.fetchValue({ bucket: 'my_bucket', key: 'my_key' }, function (err, rslt) { +}); +``` + +```erlang +{ok, Object} = riakc_pb_socket:get(Pid, + <<"my_bucket">>, + <<"my_key">>). +``` + +```curl +curl http://localhost:8098/buckets/my_bucket/keys/my_key +``` + +With the addition of bucket types in Riak 2.0, bucket types can be used +as _an additional namespace_ on top of buckets and keys. The same bucket +name can be associated with completely different data if it used in +accordance with a different type. Thus, the following two requests will +be made to _completely different objects_, even though the bucket and key +names are the same: + +```java +Location key1 = + new Location(new Namespace("type1", "my_bucket"), "my_key"); +Location key2 = + new Location(new Namespace("type2", "my_bucket"), "my_key"); +FetchValue fetch1 = new FetchValue.Builder(key1).build(); +FetchValue fetch2 = new FetchValue.Builder(key2).build(); +client.execute(fetch1); +client.execute(fetch2); +``` + +```ruby +bucket1 = client.bucket_type('type1').bucket('my_bucket') +bucket2 = client.bucket_type('type2').bucket('my_bucket') +bucket1.get('my_key') +bucket2.get('my_key') +``` + +```php +$location1 = new \Basho\Riak\Location('my_key', new Bucket('my_bucket', 'type1')); +$location2 = new Location('my_key', new Bucket('my_bucket', 'type2')); +$builder = new \Basho\Riak\Command\Builder\FetchObject($riak); +$builder->atLocation($location1) + ->build() + ->execute(); +$builder->atLocation($location2) + ->build() + ->execute(); +``` + +```python +bucket1 = client.bucket_type('type1').bucket('my_bucket') +bucket2 = client.bucket_type('type2').bucket('my_bucket') +bucket1.get('my_key') +bucket2.get('my_key') +``` + +```csharp +var id1 = new RiakObjectId("type1", "my_bucket", "my_key"); +var id2 = new RiakObjectId("type2", "my_bucket", "my_key"); +var rslt1 = client.Get(id1); +var rslt2 = client.Get(id2); +``` + +```javascript +client.fetchValue({ + bucketType: 'type1', bucket: 'my_bucket', key: 'my_key' +}, function (err, rslt) { +}); + +client.fetchValue({ + bucketType: 'type2', bucket: 'my_bucket', key: 'my_key' +}, function (err, rslt) { +}); +``` + +```erlang +{ok, Obj1} = riakc_pb_socket:get(Pid, + {<<"type1">>, <<"my_bucket">>}, + <<"my_key">>), +{ok, Obj2} = riakc_pb_socket:get(Pid, + {<<"type2">>, <<"my_bucket">>}, + <<"my_key">>). +``` + +```curl +curl http://localhost:8098/types/type1/buckets/my_bucket/keys/my_key +curl http://localhost:8098/types/type2/buckets/my_bucket/keys/my_key +``` + +{{% note title="Note on object location" %}} +In Riak 2.x, _all requests_ must be made to a location specified by a bucket +type, bucket, and key rather than to a bucket/key pair, as in previous +versions. +{{% /note %}} + +If requests are made to a bucket/key pair without a specified bucket +type, `default` will be used in place of a bucket type. The following +queries are thus identical: + +```java +Location withDefaultBucketType = + new Location(new Namespace("default", "my_bucket"), "my_key"); +Location noBucketType = + new Location(new Namespace("my_bucket"), "my_key"); +FetchValue fetch1 = new FetchValue.Builder(withDefaultBucketType).build(); +FetchValue fetch2 = new FetchValue.Builder(noBucketType).build(); +client.execute(fetch1); +client.execute(fetch2); +``` + +```ruby +bucket1 = client.bucket_type('default').bucket('my_bucket') +bucket2 = client.bucket('my_bucket') +bucket1.get('my_key') +bucket2.get('my_key') +``` + +```php +$location1 = new \Basho\Riak\Location('my_key', new Bucket('my_bucket', 'default')); +$location2 = new \Basho\Riak\Location('my_key', new Bucket('my_bucket')); +$builder = new \Basho\Riak\Command\Builder\FetchObject($riak); +$builder->atLocation($location1) + ->build() + ->execute(); +$builder->atLocation($location2) + ->build() + ->execute(); +``` + +```python +bucket1 = client.bucket_type('default').bucket('my_bucket') +bucket2 = client.bucket('my_bucket') +bucket1.get('my_key') +bucket2.get('my_key') +``` + +```csharp +var id1 = new RiakObjectId("default", "my_bucket", "my_key"); +var obj1 = new RiakObject(id1, "value", RiakConstants.ContentTypes.TextPlain); +client.Put(obj1); + +var id2 = new RiakObjectId("my_bucket", "my_key"); +var getRslt = client.Get(id2); + +RiakObject obj2 = getRslt.Value; +// Note: obj1.Value and obj2.Value are equal +``` + +```javascript +var obj1 = new Riak.Commands.KV.RiakObject(); +obj1.setContentType('text/plain'); +obj1.setBucketType('default'); +obj1.setBucket('my_bucket'); +obj1.setKey('my_key'); +obj1.setValue('value'); +client.storeValue({ value: obj1 }, function (err, rslt) { + if (err) { + throw new Error(err); + } + + client.fetchValue({ + bucketType: 'default', bucket: 'my_bucket', key: 'my_key' + }, function (err, rslt) { + if (err) { + throw new Error(err); + } + var obj2 = rslt.values.shift(); + assert(obj1.value == obj2.value); + }); +}); +``` + +```erlang +{ok, Obj1} = riakc_pb_socket:get(Pid, + {<<"default">>, <<"my_bucket">>}, + <<"my_key">>), +{ok, Obj2} = riakc_pb_socket:get(Pid, + <<"my_bucket">>, + <<"my_key">>). +``` + +```curl +curl http://localhost:8098/buckets/my_bucket/keys/my_key +curl http://localhost:8098/types/default/my_bucket/keys/my_key +``` + +## Default Bucket Properties + +Below is a listing of the default bucket properties (i.e. `props`) +associated with the `default` bucket type: + +```json +{ + "props": { + "allow_mult": false, + "basic_quorum": false, + "big_vclock": 50, + "chash_keyfun": { + "fun": "chash_std_keyfun", + "mod": "riak_core_util" + }, + "dvv_enabled": false, + "dw": "quorum", + "last_write_wins": false, + "linkfun": { + "fun": "mapreduce_linkfun", + "mod": "riak_kv_wm_link_walker" + }, + "n_val": 3, + "notfound_ok": true, + "old_vclock": 86400, + "postcommit": [], + "pr": 0, + "precommit": [], + "pw": 0, + "r": "quorum", + "rw": "quorum", + "small_vclock": 50, + "w": "quorum", + "young_vclock": 20 + } +} +``` + +## Bucket Types and the `allow_mult` Setting + +Prior to Riak 2.0, Riak created [siblings]({{<baseurl>}}riak/kv/2.2.6/learn/concepts/causal-context/#siblings) in the case of conflicting updates only when explicitly instructed to do so, i.e. when `allow_mult` is to `true`. The default `allow_mult` setting was `false`. + +In version 2.0, this is changing in a subtle way. Now, there are two +different default settings for `allow_mult` in play: + +* For the `default` bucket type, `allow_mult` is set to `false` by + default, as in previous versions of Riak +* For all newly-created bucket types, the default is now `true`. It is + possible to set `allow_mult` to `false` if you wish to avoid resolving + sibling conflicts, but this needs to be done explicitly. + +The consequence is that applications that have previously ignored +conflict resolutions in certain buckets (or all buckets) can continue to +do so. New applications, however, are encouraged to retain and [resolve siblings]({{<baseurl>}}riak/kv/2.2.6/developing/usage/conflict-resolution) with the appropriate application-side business logic. + +To give an example, let's have a look at the properties associated with +the `default` bucket type: + +```bash +riak-admin bucket-type status default | grep allow_mult +``` + +The output: + +``` +allow_mult: false +``` + +Now, let's create a new bucket type called `n_val_of_2`, which sets the +`n_val` to 2 but doesn't explicitly set `allow_mult`: + +```bash +riak-admin bucket-type create n_val_of_2 '{"props":{"n_val":2}}' +``` + +When specifying this bucket type's properties as above, the `allow_mult` +parameter was not changed. However, if we view the bucket type's +properties, we can see in the console output that `allow_mult` is set to +`true`: + +```bash +riak-admin bucket-type status n_val_of_2 | grep allow_mult +``` + +The output: + +``` +allow_mult: true +``` + +This is important to bear in mind when using versions of Riak 2.0 and +later any time that you create, activate, and use your own bucket types. +It is still possible to set `allow_mult` to `false` in any given bucket +type, but it must be done explicitly. If we wanted to set +`allow_mult` to `false` in our `n_val_of_2` bucket type from above, we +would need to create or modify the already existing type as follows: + +```bash +riak-admin bucket-type update n_val_of_2 '{"props":{"allow_mult":false}}' +``` + +## Bucket Type Example + +Let's say that you'd like to create a bucket type called +`user_account_bucket` with a [pre-commit hook]({{<baseurl>}}riak/kv/2.2.6/developing/usage/commit-hooks/#pre-commit-hooks) called `syntax_check` and two [post-commit +hooks]({{<baseurl>}}riak/kv/2.2.6/developing/usage/commit-hooks/#Post-Commit-Hooks) called `welcome_email` and `update_registry`. This would involve four steps: + +1. Creating a JavaScript object containing the appropriate `props` + settings: + + ```json + { + "props": { + "precommit": ["syntax_check"], + "postcommit": ["welcome_email", "update_registry"] + } + } + ``` + +2. Passing that JSON to the `bucket-type create` command: + + ```bash + riak-admin bucket-type create user_account_bucket '{"props":{"precommit": ["syntax_check"], ... }}' + ``` + + If creation is successful, the console will return + `user_account_bucket created`. + +3. Verifying that the type is ready to be activated: + + Once the type is created, you can check whether your new type is + ready to be activated by running: + + ```bash + riak-admin bucket-type status user_account_bucket + ``` + + If the first line reads `user_account_bucket has been created and + may be activated`, then you can proceed to the next step. If it + reads `user_account_bucket has been created and is not ready to + activate`, then wait a moment and try again. If it still does not + work, then there may be network partition or other issues that need + to be addressed in your cluster. + +4. Activating the new bucket type: + + ```bash + riak-admin bucket-type activate user_account_bucket + ``` + + If activation is successful, the console will return + `user_account_bucket has been activated`. The bucket type is now + ready to be used. + +## Client Usage Example + +If you have created the bucket type `no_siblings` (with the property +`allow_mult` set to `false`) and would like that type to be applied to +the bucket `sensitive_user_data`, you would need to run operations on +that bucket in accordance with the format above. Here is an example +write: + +```java +Location key = new Location("sensitive_user_data") + .setBucketType("no_siblings") + .setKey("user19735"); +RiakObject obj = new RiakObject() + .setContentType("application/json") + .setValue(BinaryValue.create("{ ... user data ... }")); +StoreValue store = new StoreValue.Builder(obj).build(); +client.execute(store); +``` + +```ruby +bucket = client.bucket_type('no_siblings').bucket('sensitive_user_data') +obj = Riak::RObject.new(bucket, 'user19735') +obj.content_type = 'application/json' +obj.raw_data = '{ ... user data ... }' +obj.store +``` + +```php +(new \Basho\Riak\Command\Builder\StoreObject($riak)) + ->buildJsonObject("{ ... user data ... }") + ->buildLocation('user19735', 'sensitive_user_data', 'no_siblings') + ->build() + ->execute(); +``` + +```python +bucket = client.bucket_type('no_siblings').bucket('sensitive_user_data') +obj = RiakObject(client, bucket, 'user19735') +obj.content_type = 'application/json' +obj.data = '{ ... user data ... }' +obj.store() +``` + +```csharp +var id = new RiakObjectId("no_siblings", "sensitive_user_data", "user19735"); +var obj = new RiakObject(id, "{\"name\":\"Bob\"}"); +var rslt = client.Put(obj); +``` + +```javascript +var obj = { name: 'Bob' }; +client.storeValue({ + bucketType: 'no_siblings', bucket: 'sensitive_user_data', + key: 'user19735', value: obj +}, function (err, rslt) { + if (err) { + throw new Error(err); + } +}); +``` + +```erlang +Object = riakc_obj:new({<<"no_siblings">>, <<"sensitive_user_data">>}, + <<"user19735">>, + <<"{ ... user data ... }">>, + <<"application/json">>), +riakc_pb_socket:put(Pid, Object). +``` + +```curl +curl -XPUT \ + -H "Content-Type: application/json" \ + -d "{ ... user data ... }" \ + http://localhost:8098/types/no_siblings/buckets/sensitive_user_data/keys/user19735 +``` + +In this example, the bucket `sensitive_user_data` bears the +configuration established by the `no_siblings` bucket type, and it bears +that configuration _on the basis of the query's structure_. This is +because buckets act as a [separate namespace](#buckets-as-namespaces) in Riak, in addition to [buckets]({{<baseurl>}}riak/kv/2.2.6/learn/concepts/buckets) and [keys]({{<baseurl>}}riak/kv/2.2.6/learn/concepts/keys-and-objects). + +Let's say that we're using Riak to store internet memes. We've been +using a bucket called `current_memes` using the bucket type +`no_siblings` (from above). At a certain point, we decide that our +application needs to use a new bucket called `old_memes` to store memes +that have gone woefully out of fashion, but that bucket also needs to +bear the type `no_siblings`. + +The following request seeks to add the meme "all your base are belong to +us" to the `old_memes` bucket. If the bucket type `no_siblings` has been +created and activated, the request will ensure that the `old_memes` +bucket inherits all of the properties from the type `no_siblings`: + +```java +Location allYourBaseKey = + new Location(new Namespace("no_siblings", "old_memes"), "all_your_base"); +RiakObject obj = new RiakObject() + .setContentType("text/plain") + .setValue(BinaryValue.create("all your base are belong to us")); +StoreValue store = new StoreValue.Builder(obj).build(); +client.execute(store); +``` + +```ruby +bucket = client.bucket_type('no_siblings').bucket('old_memes') +obj = Riak::RObject.new(bucket, 'all_your_base') +obj.content_type = 'text/plain' +obj.raw_data = 'all your base are belong to us' +obj.store +``` + +```php +(new \Basho\Riak\Command\Builder\StoreObject($riak)) + ->buildObject("all your base are belong to us", ['Content-Type' => 'text/plain']) + ->buildLocation('user19735', 'sensitive_user_data', 'no_siblings') + ->build() + ->execute(); +``` + +```python +bucket = client.bucket_type('no_siblings').bucket('old_memes') +obj = RiakObject(client, bucket, 'all_your_base') +obj.content_type = 'text/plain' +obj.data = 'all your base are belong to us' +obj.store() +``` + +```csharp +var id = new RiakObjectId("no_siblings", "old_memes", "all_your_base"); +var obj = new RiakObject(id, "all your base are belong to us", + RiakConstants.ContentTypes.TextPlain); +var rslt = client.Put(obj); +``` + +```javascript +var obj = new Riak.Commands.KV.RiakObject(); +obj.setContentType('text/plain'); +obj.setBucketType('no_siblings'); +obj.setBucket('old_memes'); +obj.setKey('all_your_base'); +obj.setValue('all your base are belong to us'); +client.storeValue({ value: obj }, function (err, rslt) { + if (err) { + throw new Error(err); + } +}); +``` + +```erlang +Object = riakc_obj:new({<<"no_siblings">>, <<"old_memes">>}, + <<"all_your_base">>, + <<"all your base are belong to us">>, + <<"text/plain">>), +riakc_pb_socket:put(Pid, Object). +``` + +```curl +curl -XPUT \ + -H "Content-Type: text/plain" \ + -d "all your base are belong to us" \ + http://localhost:8098/types/no_siblings/buckets/old_memes/keys/all_your_base +``` + +This query would both create the bucket `old_memes` and ensure that the +configuration contained in the `no_siblings` bucket type is applied to +the bucket all at once. + +If we wished, we could also store both old and new memes in +buckets with different types. We could use the `no_siblings` bucket from +above if we didn't want to deal with siblings, vclocks, and the like, +and we could use a `siblings_allowed` bucket type (with all of the +default properties except `allow_mult` set to `true`). This would give +use four bucket type/bucket pairs: + +* `no_siblings` / `old_memes` +* `no_siblings` / `new_memes` +* `siblings_allowed` / `old_memes` +* `siblings_allowed` / `new_memes` + +All four of these pairs are isolated keyspaces. The key `favorite_meme` +could hold different values in all four bucket type/bucket spaces. diff --git a/content/riak/kv/2.2.6/using/reference/custom-code.md b/content/riak/kv/2.2.6/using/reference/custom-code.md new file mode 100644 index 0000000000..657311365f --- /dev/null +++ b/content/riak/kv/2.2.6/using/reference/custom-code.md @@ -0,0 +1,131 @@ +--- +title: "Installing Custom Code" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Installing Custom Code" + identifier: "managing_ref_custom_code" + weight: 111 + parent: "managing_ref" +toc: true +aliases: + - /riak-docs/riak/2.2.6/ops/advanced/install-custom-code/ + - /riak-docs/riak/kv/2.2.6/ops/advanced/install-custom-code/ +--- + +Riak supports the use of Erlang named functions in compiled modules for +[pre/post-commit hooks]({{<baseurl>}}riak/kv/2.2.6/developing/usage/commit-hooks), and MapReduce operations. This +doc contains installation steps with simple examples for each use case. + +Your developers can compile [custom erlang code]({{<baseurl>}}riak/kv/2.2.6/developing/usage/commit-hooks), which +they can send to you as a *beam* file. You should note that in Erlang, a file +name must have the same name the module. So if you are given a file named +`validate_json.beam`, do not rename it. + +> *Note: The [Configure](#configure) step (`add_paths`) also applies to installing JavaScript files.* + +### Compiling + +If you have been given Erlang code and are expected to compile it for +your developers, keep the following notes in mind. + +{{% note title="Note on the Erlang Compiler" %}} +You must use the Erlang compiler (`erlc`) associated with the Riak +installation or the version of Erlang used when compiling Riak from source. +For packaged Riak installations, you can consult Table 1 below for the default +location of Riak's `erlc` for each supported platform. If you compiled from +source, use the `erlc` from the Erlang version you used to compile Riak. +{{% /note %}} + +<table style="width: 100%; border-spacing: 0px;"> +<tbody> +<tr align="left" valign="top"> +<td style="padding: 15px; margin: 15px; border-width: 1px 0 1px 0; border-style: solid;"><strong>CentOS & RHEL Linux</strong></td> +<td style="padding: 15px; margin: 15px; border-width: 1px 0 1px 0; border-style: solid;"> +<p><tt>/usr/lib64/riak/erts-5.9.1/bin/erlc</tt></p> +</td> +</tr> +<tr align="left" valign="top"> +<td style="padding: 15px; margin: 15px; border-width: 1px 0 1px 0; border-style: solid;"><strong>Debian & Ubuntu Linux</strong></td> +<td style="padding: 15px; margin: 15px; border-width: 1px 0 1px 0; border-style: solid;"> +<p><tt>/usr/lib/riak/erts-5.9.1/bin/erlc</tt></p> +</td> +</tr> +<tr align="left" valign="top"> +<td style="padding: 15px; margin: 15px; border-width: 1px 0 1px 0; border-style: solid;"><strong>FreeBSD</strong></td> +<td style="padding: 15px; margin: 15px; border-width: 1px 0 1px 0; border-style: solid;"> +<p><tt>/usr/local/lib/riak/erts-5.9.1/bin/erlc</tt></p> +</td> +</tr> +<tr align="left" valign="top"> +<td style="padding: 15px; margin: 15px; border-width: 1px 0 1px 0; border-style: solid;"><strong>SmartOS</strong></td> +<td style="padding: 15px; margin: 15px; border-width: 1px 0 1px 0; border-style: solid;"> +<p><tt>/opt/local/lib/riak/erts-5.9.1/bin/erlc</tt></p> +</td> +</tr> +<tr align="left" valign="top"> +<td style="padding: 15px; margin: 15px; border-width: 1px 0 1px 0; border-style: solid;"><strong>Solaris 10</strong></td> +<td style="padding: 15px; margin: 15px; border-width: 1px 0 1px 0; border-style: solid;"> +<p><tt>/opt/riak/lib/erts-5.9.1/bin/erlc</tt></p> +</td> +</tr> +</tbody> +</table> + +Table 1: Erlang compiler executable location for packaged Riak installations + on supported platforms + +Compiling the module is a straightforward process. + +```text +erlc validate_json.erl +``` + +Next, you'll need to define a path from which compiled modules can be stored +and loaded. For our example, we'll use a temporary directory `/tmp/beams`, +but you should choose a directory for production functions based on your +own requirements such that they will be available where and when needed. + +{{% note %}} +Ensure that the directory chosen above can be read by the `riak` user. +{{% /note %}} + +Successful compilation will result in a new `.beam` file, +`validate_json.beam`. + +### Configure + +Take the `validate_json.beam` and copy this file to the `/tmp/beams` directory. + +```text +cp validate_json.beam /tmp/beams/ +``` + +After copying the compiled module into `/tmp/beams/`, you must update +`app.config` and configure Riak to allow loading of compiled modules from +the directory where they're stored (again in our example case, `/tmp/beams`). + +Edit `app.config` and insert an `add_paths` setting into the `riak_kv` +section as shown: + +```erlang +{riak_kv, [ + %% ... + {add_paths, ["/tmp/beams/"]}, + %% ... +``` + +After updating `app.config`, Riak must be restarted. In production cases, you +should ensure that if you are adding configuration changes to multiple nodes, +that you do so in a rolling fashion, taking time to ensure that the Riak key +value store has fully initialized and become available for use. + +This is done with the `riak-admin wait-for-service` command as detailed +in the [Commands documentation]({{<baseurl>}}riak/kv/2.2.6/using/admin/riak-admin/#wait-for-service). + +{{% note %}} +It is important that you ensure riak_kv is active before restarting the next +node. +{{% /note %}} diff --git a/content/riak/kv/2.2.6/using/reference/failure-recovery.md b/content/riak/kv/2.2.6/using/reference/failure-recovery.md new file mode 100644 index 0000000000..293aa47499 --- /dev/null +++ b/content/riak/kv/2.2.6/using/reference/failure-recovery.md @@ -0,0 +1,80 @@ +--- +draft: true +title: "Failure & Recovery Reference" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Failure & Recovery" + identifier: "managing_ref_failure_recovery" + weight: 105 + parent: "managing_ref" +toc: true +--- + +## Hanc capellae + +Lorem markdownum Byblida. Modo **etiam** litora mittat vellera infelix caeli. +Studiosius forte, potuit pectore. Puer undas dignior iam turpe sorores abesse. +Deae Saturnia levius viribus membra. + +## Iussorum ad fronti rutilasque tenuit cursu quae + +Nostros vovistis artes. **Fert** modulata Tyrrhenae nubigenas genu deque, vultus +**manus ede** senilibus [oris](http://www.youtube.com/watch?v=MghiBW3r65M) +transcurrere quem rarissima. Viderunt nutu quod, tumidaque, mihi mihi sacer pia. +Summis rediit pavidus tersere et at prosiluit natus Phaethon noxa. Singultibus +oblita **foedabis** orsa. + +- Fecere aliis postquam inviti caliginis ab inque +- Voverat dividuae et tardus huc magna non +- Sex barba ipsaque Caucason corpora sono ecce +- Non esse +- Sibi atris regna licuit Antium carituraque nubes + +## Omni levare gelidumque minanti + +Omnis adeunt ossibus gravis, Venus pinuque capit, et sereno viros ignara *plena +incaluere* percussit mellaque, vertere arte. Ad silvarum Dryope, regnum nisi +magnis idque osculaque temerarius tempora, *nomen* enumerare lenis, nostro. Ac +mutabit [arma](http://www.thesecretofinvisibility.com/) operiri saxum ratione, +crudelior feram, est usu tamen quod, hasta. Equos **sonant et deum**. Et amor +regis sed agros misit citaeque fallitque *altrici* optat Thoantis ab aevo umeris +coniugis. + +## Troiana quoque + +Equo uni Stygias trahunt, interea, in tela labores lumina, nam *Aganippe +sanctique meum*; est. [Gente inimica +premeret](http://en.wikipedia.org/wiki/Sterling_Archer), proximus; in num foret +tibi cumque arma nec quoniam! Contribuere mollis, tu dum parem viscera, tamen +ante. Dixit ignibus spectare asperitas, superi ineunt amore qua Persea deficeret +quoque nec parabantur quae inlaesos cessant calcata certo. Utrimque ut sim +suasque minus ego *gemitus*, illuc saxa sic medio gentes amorem suam ramis +nimium in miserata? + +1. `In naribus aequos aberant` +2. Naturae murmura te rimas suarum vulnus quod +3. Socios leto loquor timide +4. Ergo sub +5. Patrias mihi consumite breve + +## Ruit huic movit luminibus excubias arma + +> Loco humo tecum gurgite timui. Peragant tu regia ut umbras premit condit. Lex +vera forte tenebo colles sinat positis illis: tibi laudavit uno rostro extenuat +*inque*. Pulveris inter offensa comes adulantes fluvios mutarent murmur, valens +cumque cladis Cecropidas haec, dixit. Lucus cognomine **Achilles**: pastor nec. + +1. Hic causam et dilecte nudae nec corpus +2. Cor Si nive +3. Petis equos perosa tu perterrita exitus non +4. Per et et ire geminos parte +5. Aqua coniunx cecidisse sonum + +``` +Nominis haec lacrimis orba gloria obstipuere tu Ceyx tepebat fetus me equorum +potero! Iampridem illi; deducit [reor orbem](http://heeeeeeeey.com/), comes, et +nec rubebant pietas, ipsa. +``` diff --git a/content/riak/kv/2.2.6/using/reference/handoff.md b/content/riak/kv/2.2.6/using/reference/handoff.md new file mode 100644 index 0000000000..2f0441c97d --- /dev/null +++ b/content/riak/kv/2.2.6/using/reference/handoff.md @@ -0,0 +1,197 @@ +--- +title: "Handoff Reference" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Handoff" + identifier: "managing_ref_handoff" + weight: 101 + parent: "managing_ref" +toc: true +aliases: + - /riak-docs/riak/2.2.6/ops/running/handoff/ + - /riak-docs/riak/kv/2.2.6/ops/running/handoff/ +--- + +[cluster ops handoff]: {{<baseurl>}}riak/kv/2.2.6/using/cluster-operations/handoff + +Riak is a distributed system built with two essential goals in mind: + +* **fault tolerance**, whereby a Riak cluster can withstand node + failure, network partitions, and other events in a way that does not + disrupt normal functioning, and +* **scalability**, whereby operators can gracefully add and remove nodes + to/from a Riak cluster + +Both of these goals demand that Riak is able to either temporarily or +permanently re-assign responsibility for portions of the keyspace. That +re-assigning is referred to as **intra-cluster handoff** (or simply +**handoff** in our documentation). + +## Types of Handoff + +Intra-cluster handoff typically takes one of two forms: **hinted +handoff** and **ownership transfer**. + +Hinted handoff occurs when a [vnode]({{<baseurl>}}riak/kv/2.2.6/learn/glossary/#vnode) temporarily takes over responsibility for some data and then returns that data to its original "owner." Imagine a 3-node cluster with nodes A, B, and C. If node C goes offline, e.g. during a network partition, nodes A and B will pick +up the slack, so to speak, assuming responsibility for node C's +operations. When node C comes back online, responsibility will be handed +back to the original vnodes. + +Ownership transfer is different because it is meant to be permanent. +It occurs when a [vnode]({{<baseurl>}}riak/kv/2.2.6/learn/glossary/#vnode) no longer belongs to the node on which it's running. This typically happens when the very +makeup of a cluster changes, e.g. when nodes are added or removed from +the cluster. In this case, responsibility for portions of the keyspace +needs to be fundamentally re-assigned. + +Both types of handoff are handled automatically by Riak. Operators do +have the option, however, of enabling and disabling handoff on +particular nodes or all nodes and of configuring key aspects of Riak's +handoff behavior. More information can be found below. + +## Configuring Handoff + +A full listing of configurable parameters can be found in our +[configuration files]({{<baseurl>}}riak/kv/2.2.6/configuring/reference/#intra-cluster-handoff) +document. The sections below provide a more narrative description of +handoff configuration. + +### SSL + +If you want to encrypt handoff behavior within a Riak cluster, you need +to provide each node with appropriate paths for an SSL certfile (and +potentially a keyfile). The configuration below would designate a +certfile at `/ssl_dir/cert.pem` and a keyfile at `/ssl_dir/key.pem`: + +```riakconf +handoff.ssl.certfile = /ssl_dir/cert.pem +handoff.ssl.keyfile = /ssl_dir/key.pem +``` + +```appconfig +{riak_core, [ + %% Other configs + {handoff_ssl_options, [ + {certfile, "/ssl_dir/cert.pem"}, + {keyfile, "/ssl_dir/key.pem"} + ]}, + %% Other configs +]} +``` + +### Port + +You can set the port used by Riak for handoff-related interactions using +the `handoff.port` parameter. The default is 8099. This would change the +port to 9000: + +```riakconf +handoff.port = 9000 +``` + +```appconfig +{riak_core, [ + %% Other configs + {handoff_port, 9000}, + %% Other configs +]} +``` + +### Background Manager + +Riak has an optional background manager that limits handoff activity in +the name of saving resources. The manager can help prevent system +response degradation during times of heavy load, when multiple +background tasks may contend for the same system resources. The +background manager is disabled by default. The following will enable it: + +```riakconf +handoff.use_background_manager = on +``` + +```appconfig +{riak_kv, [ + %% Other configs + {handoff_use_background_manager, on}, + %% Other configs +]} +``` + +### Maximum Rejects + +If you're using Riak features such as [Riak Search]({{<baseurl>}}riak/kv/2.2.6/developing/usage/search/), +those subsystems can block handoff of primary key/value data, i.e. data +that you interact with via normal reads and writes. + +The `handoff.max_rejects` setting enables you to set the maximum +duration that a [vnode]({{<baseurl>}}riak/kv/2.2.6/learn/glossary/#vnode) can be blocked by multiplying the +`handoff.max_rejects` setting by the value of +[`vnode_management_timer`]({{<baseurl>}}riak/kv/2.2.6/configuring/reference/#vnode_management_timer). +Thus, if you set `handoff.max_rejects` to 10 and +`vnode_management_timer` to 5 seconds (i.e. `5s`), non-K/V subsystems +can block K/V handoff for a maximum of 50 seconds. The default for +`handoff.max_rejects` is 6, while the default for +`vnode_management_timer` is `10s`. This would set `max_rejects` to 10: + +```riakconf +handoff.max_rejects = 10 +``` + +```appconfig +{riak_kv, [ + %% Other configs + {handoff_rejected_max, 10}, + %% Other configs +]} +``` + +### Transfer Limit + +You can adjust the number of node-to-node transfers (which includes +handoff) using the `transfer_limit` parameter. The default is 2. Setting +this higher will increase node-to-node communication but at the expense +of higher resource intensity. This would set `transfer_limit` to 5: + +```riakconf +transfer_limit = 5 +``` + +```appconfig +{riak_core, [ + %% Other configs + {handoff_concurrency, 5}, + %% Other configs +]} +``` + +## Enabling and Disabling Handoff + +Handoff can be enabled and disabled in two ways: via configuration or +on the command line. + +### Enabling and Disabling via Configuration + +You can enable and disable both outbound and inbound handoff on a node +using the `handoff.outbound` and `handoff.inbound` settings, +respectively. Both are enabled by default. The following would disable +both: + +```riakconf +handoff.outbound = off +handoff.inbound = off +``` + +```appconfig +{riak_core, [ + %% Other configs + {disable_outbound_handoff, true}, + {disable_inbound_handoff, true}, + %% Other configs +]} +``` + +### Enabling and Disabling Through the Command Line + +Check out the [Cluster Operations: Handoff][cluster ops handoff] for steps on enabling and disabling handoff via the command line. diff --git a/content/riak/kv/2.2.6/using/reference/jmx.md b/content/riak/kv/2.2.6/using/reference/jmx.md new file mode 100644 index 0000000000..a6f2f14935 --- /dev/null +++ b/content/riak/kv/2.2.6/using/reference/jmx.md @@ -0,0 +1,186 @@ +--- +title: "JMX Monitoring" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "JMX Monitoring" + identifier: "managing_ref_jmx" + weight: 108 + parent: "managing_ref" +toc: true +commercial_offering: true +aliases: + - /riak-docs/riak/2.2.6/ops/running/monitoring/jmx + - /riak-docs/riak/kv/2.2.6/ops/running/monitoring/jmx +--- + +Riak exposes monitoring data via JMX. To enable JMX monitoring, edit the [`app.config`]({{<baseurl>}}riak/kv/2.2.6/configuring/reference/#app-config) associated with your Riak installation and set the `enabled` property of the `riak_jmx` section to `true` as shown below. The TCP port on which the JMX provider listens is also configurable in this section (the default JMX port is `41110`). + +```erlang + {riak_jmx, [ + {enabled, true}, + {port, 41110} + ]} +``` + +To view JMX data---assuming that you have the Sun JDK installed---launch JConsole as follows: + +```bash +$ jconsole <hostname_to_monitor>:<jmx_port> +``` + +Once connected, click on the **MBeans** tab, expand the **com.basho.riak** tree view, and select **Attributes**. The attributes listed in the table below will be displayed. + +Riak JMX has been tested with the Sun JRE 1.6.0_12 and 1.6.0_20. Some older/non-Sun JREs do not work (e.g. the default java-gcj JRE installed on Debian lenny). If you have problems with JMX or see the message below, please try upgrading to the Sun JRE: + +```log + =INFO REPORT==== 9-Jun-2010::08:14:57 === + JMX server monitor <pid> exited with code <non-zero>. +``` + +## Exported JMX Attributes +<br> +<table> + <tr> + <th WIDTH="30%">Attribute</th> + <th WIDTH="15%">Type</th> + <th WIDTH="55%">Description</th> + </tr> + <tr> + <td><tt>CPUNProcs</tt></td> + <td>int</td> + <td>Number of running processes</td> + </tr> + <tr> + <td><tt>CpuAvg1</tt></td> + <td>int</td> + <td>1 minute load average</td> + </tr> + <tr> + <td><tt>CpuAvg5</tt></td> + <td>int</td> + <td>5 minute load average</td> + </tr> + <tr> + <td><tt>CpuAvg15</tt></td> + <td>int</td> + <td>15 minute load average</td> + </tr> + <tr> + <td><tt>NodeGetFsmTime95</tt></td> + <td>float</td> + <td>95th percentile GET time (microseconds)</td> + </tr> + <tr> + <td><tt>NodeGetFsmTime99</tt></td> + <td>float</td> + <td>99th percentile GET time (microseconds)</td> + </tr> + <tr> + <td><tt>NodeGetFsmTimeMax</tt></td> + <td>float</td> + <td>Maximum GET time (microseconds)</td> + </tr> + <tr> + <td><tt>NodeGetFsmTimeMean</tt></td> + <td>float</td> + <td>Mean GET time (microseconds)</td> + </tr> + <tr> + <td><tt>NodeGetFsmTimeMedian</tt></td> + <td>float</td> + <td>Median GET time (microseconds)</td> + </tr> + <tr> + <td><tt>NodeGets</tt></td> + <td>int</td> + <td>Number of GETs in past minute</td> + </tr> + <tr> + <td><tt>NodeGetsTotal</tt></td> + <td>int</td> + <td>Number of GETs since node start</td> + </tr> + <tr> + <td><tt>NodeName</tt></td> + <td>string</td> + <td>Node name</td> + </tr> + <tr> + <td><tt>NodePutFsmTime95</tt></td> + <td>float</td> + <td>95th percentile PUT time (microseconds)</td> + </tr> + <tr> + <td><tt>NodePutFsmTime99</tt></td> + <td>float</td> + <td>99th percentile PUT time (microseconds)</td> + </tr> + <tr> + <td><tt>NodePutFsmTimeMax</tt></td> + <td>float</td> + <td>Maximum PUT time (microseconds)</td> + </tr> + <tr> + <td><tt>NodePutFsmTimeMean</tt></td> + <td>float</td> + <td>Mean PUT time (microseconds)</td> + </tr> + <tr> + <td><tt>NodePutFsmTimeMedian</tt></td> + <td>float</td> + <td>Median PUT time (microseconds)</td> + </tr> + <tr> + <td><tt>NodePuts</tt></td> + <td>int</td> + <td>Number of PUTs in past minute</td> + </tr> + <tr> + <td><tt>NodePutsTotal</tt></td> + <td>int</td> + <td>Number of PUTs since node start</td> + </tr> + <tr> + <td><tt>PBCActive</tt></td> + <td>int</td> + <td>Number of active Protocol Buffers connections</td> + </tr> + <tr> + <td><tt>PBCConnects</tt></td> + <td>int</td> + <td>Number of Protocol Buffers connections in past minute</td> + </tr> + <tr> + <td><tt>PBCConnectsTotal</tt></td> + <td>int</td> + <td>Number of Protocol Buffers connections since node start</td> + </tr> + <tr> + <td><tt>RingCreationSize</tt></td> + <td>int</td> + <td>Number of partitions in Riak ring</td> + </tr> + <tr> + <td><tt>VnodeGets</tt></td> + <td>int</td> + <td>Number of vnode-level GETs in past minute</td> + </tr> + <tr> + <td><tt>VnodeGetsTotal</tt></td> + <td>int</td> + <td>Number of vnode-level GETs since node start</td> + </tr> + <tr> + <td><tt>VnodePuts</tt></td> + <td>int</td> + <td>Number of vnode-level PUTs in past minute</td> + </tr> + <tr> + <td><tt>VnodePutsTotal</tt></td> + <td>int</td> + <td>Number of vnode-level PUTs since node start</td> + </tr> +</table> diff --git a/content/riak/kv/2.2.6/using/reference/logging.md b/content/riak/kv/2.2.6/using/reference/logging.md new file mode 100644 index 0000000000..04cd92d259 --- /dev/null +++ b/content/riak/kv/2.2.6/using/reference/logging.md @@ -0,0 +1,297 @@ +--- +title: "Logging Reference" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Logging" + identifier: "managing_ref_logging" + weight: 100 + parent: "managing_ref" +toc: true +aliases: + - /riak-docs/riak/2.2.6/ops/running/logging + - /riak-docs/riak/kv/2.2.6/ops/running/logging +--- + +[cluster ops log]: {{<baseurl>}}riak/kv/2.2.6/using/cluster-operations/logging + +Logging in Riak KV is handled by a Basho-produced logging framework for +[Erlang](http://www.erlang.org) called +[lager](https://github.com/basho/lager). + +lager provides a number of configuration options that you can use to fine-tune your Riak cluster's logging output. A compact listing of parameters can be found in our [configuration files]({{<baseurl>}}riak/kv/2.2.6/configuring/reference/#logging) documentation. A more thorough explanation of these options can be found in this document. + +## Log Directory + +Riak's log files are stored in a `/log` directory on each node. The +location of that directory differs from platform to platform. The table +below shows you where log files are stored on all supported operating +systems. + +OS | Directory +:--|:--------- +Ubuntu, Debian, CentOS, RHEL | `/var/log/riak` +Solaris, OpenSolaris | `/opt/riak/log` +Source install and Mac OS X | `./log` (where the `.` represents the root installation directory) + +## Log Files + +Below is a list of files that can be found in each node's `/log` +directory: + +File | Significance +:----|:------------ +`console.log` | Console log output +`crash.log` | Crash logs +`erlang.log` | Logs emitted by the [Erlang VM](../../performance/erlang) on which Riak runs. +`error.log` | [Common errors](../../repair-recovery/errors) emitted by Riak. +`run_erl.log` | The log file for an Erlang process called `run_erl`. This file can typically be ignored. + +## Log Syntax + +Riak logs tend to be structured like this: + +```log +<date> <time> [<level>] <PID> <prefix>: <message> +``` + +The `date` segment is structured `YYYY-MM-DD`, `time` is structured +`hh:mm:ss.sss`, `level` depends on which log levels are available in the +file you are looking at (consult the sections below), the `PID` is the +Erlang process identifier for the process in which the event occurred, +and the message `prefix` will often identify the Riak subsystem +involved, e.g. `riak_ensemble_peer` or `alarm_handler` (amongst many +other possibilities). + +{{% note title="Warning: Log messages may contain newline characters" %}} +As of Riak KV 2.2.6 a few of the log messages may contain newline +characters, preventing reliable identification of the end of each log +when attempting log files ingestion by external tools. + +A known workaround is ingesting not the logs enabled by the +`log.console` configurable parameter but rather the logs as enabled by +the `log.syslog` configurable parameter and processed by syslog, +e.g. exploiting the +[`no-multi-line`](https://www.balabit.com/documents/syslog-ng-ose-3.5-guides/en/syslog-ng-ose-guide-admin/html-single/index.html) +option (e.g. see [this StackExchange topic +answer](https://unix.stackexchange.com/questions/317422/is-there-a-way-to-rewrite-parts-of-a-message-globally-instead-of-inserting-rewri/317474#317474)) +- or equivalent - of syslog implementations. +{{% /note %}} + +The exception to this syntax is in crash logs (stored in `crash.log` +files). For crash logs, the syntax tends to be along the following +lines: + +```log +<date> <time> =<report title>==== +<message> +``` + +Here is an example crash report: + +```log +2014-10-17 15:56:38 =ERROR REPORT==== +Error in process <0.4330.323> on node 'dev1@127.0.0.1' with exit value: ... +``` + +## Log Files + +In each node's `/log` directory, you will see at least one of each of +the following: + +File | Contents +:----|:-------- +`console.log` | General messages from all Riak subsystems +`crash.log` | Catastrophic events, such as node failures, running out of disk space, etc. +`erlang.log` | Events from the Erlang VM on which Riak runs +`run_erl.log` | The command-line arguments used when starting Riak + +### Log File Rotation + +Riak maintains multiple separate files for `console.log`, `crash.log`, +`erlang.log`, and `error.log`, which are rotated as each file reaches +its maximum capacity of 100 KB. In each node's `/log` directory, you may +see, for example, files name `console.log`, `console.log.0`, +`console.log.1`, and so on. Riak's log rotation is somewhat non +traditional, as it does not always log to `*.1` (e.g. `erlang.log.1`) +but rather to the oldest log file. + +After, say, `erlang.log.1` is filled up, the logging system will begin +writing to `erlang.log.2`, then `erlang.log.3`, and so on. When +`erlang.log.5` is filled up, it will loop back to `erlang.log.1`. + +## SASL + +[SASL](http://www.erlang.org/doc/man/sasl_app.html) (System Architecture +Support Libraries) is Erlang's built-in error logger. You can enable it +and disable it using the `sasl` parameter (which can be set to `on` or +`off`). It is disabled by default. The following would enable it: + +```riakconf +sasl = on +``` + +## Error Messages + +By default, Riak stores error messages in `./log/error.log` by default. +You can change this using the `log.error.file` parameter. Here is an +example, which uses the default: + +```riakconf +log.error.file = ./log/error.log +``` + +By default, error messages are redirected into lager, i.e. the +`log.error.redirect` parameter is set to `on`. The following would +disable the redirect: + +```riakconf +log.error.redirect = off +``` + +You can also throttle the number of error messages that are handled per +second. The default is 100. + +```riakconf +log.error.messages_per_second = 100 +``` + +## Crash Logs + +Riak crash logs are stored in `./log/crash.log` by default. You can +change this using the `log.crash.file` parameter. This example uses the +default: + +```riakconf +log.crash.file = ./log/crash.log +``` + +While crash logs are kept by default, i.e. the `log.crash` parameter is +set to `on`, you can disable crash logs like this: + +```riakconf +log.crash = off +``` + +### Crash Log Rotation + +Like other Riak logs, crash logs are rotated. You can set the crash logs +to be rotated either when a certain size threshold is reached and/or at +designated times. + +You can set the rotation time using the `log.crash.rotation` parameter. +The default is `$D0`, which rotates the logs every day at midnight. You +can also set the rotation to occur weekly, on specific days of the +month, etc. Complete documentation of the syntax can be found +[here](https://github.com/basho/lager/blob/master/README.md#internal-log-rotation). +Below are some examples: + +* `$D0` --- Every night at midnight +* `$D23` --- Every day at 23:00 (11 pm) +* `$W0D20` --- Every week on Sunday at 20:00 (8 pm) +* `$M1D0` --- On the first day of every month at midnight +* `$M5D6` --- On the fifth day of the month at 6:00 (6 am) + +To set the maximum size of the crash log before it is rotated, use the +`log.crash.size` parameter. You can specify the size in KB, MB, etc. The +default is `10MB`. + + +### Other Crash Log Settings + +The maximum size of individual crash log messages can be set using the +`log.crash.maximum_message_size`, using any size denomination you wish, +e.g. `KB` or `MB` The default is 64 KB. The following would set that +maximum message size to 1 MB: + +```riakconf +log.crash.maximum_message_size = 1MB +``` + +## Syslog + +Riak log output does not go to syslog by default, i.e. the `log.syslog` +setting is set to `off` by default. To enable syslog output: + +```riakconf +log.syslog = on +``` + +If syslog output is enabled, you can choose a prefix to be appended to +each syslog message. The prefix is `riak` by default. + +```riakconf +log.syslog.ident = riak +``` + +### Syslog Level and Facility Level + +If syslog is enabled, i.e. if `log.syslog` is set to `on`, you can +select the log level of syslog output from amongst the available levels, +which are listed in the table below. The default is `info`. + +* `alert` +* `critical` +* `debug` +* `emergency` +* `error` +* `info` +* `none` +* `notice` +* `warning` + +In addition to a log level, you must also select a [facility +level](https://en.wikipedia.org/wiki/Syslog#Facility) for syslog +messages amongst the available levels, which are listed in the table +below. The default is `daemon`. + +* `auth` +* `authpriv` +* `clock` +* `cron` +* `daemon` +* `ftp` +* `kern` +* `lpr` +* `mail` +* `news` +* `syslog` +* `user` +* `uucp` + +In addition to these options, you may also choose one of `local0` +through `local7`. + +## Console Logs + +Riak console logs can be emitted to one of three places: to a log file +(you can choose the name and location of that file), to standard output, +or to neither. This is determined by the value that you give to the +`log.console` parameter, which gives you one of four options: + +* `file` --- Console logs will be emitted to a file. This is Riak's + default behavior. The location of that file is determined by the + `log.console.file` parameter. The default location is + `./log/console.log` on an installation from [source]({{<baseurl>}}riak/kv/2.2.6/setup/installing/source), but will differ on platform-specific installation, + e.g. `/var/log/riak` on Ubuntu, Debian, CentOS, and RHEL or + `/opt/riak/log` on Solaris-based platforms. +* `console` --- Console logs will be emitted to standard output, which + can be viewed by running the [`riak attach-direct`]({{<baseurl>}}riak/kv/2.2.6/using/admin/riak-cli/#attach-direct) command +* `both` --- Console logs will be emitted both to a file and to standard + output +* `off` --- Console log messages will be disabled + +In addition to the the placement of console logs, you can also choose +the severity of those messages using the `log.console.level` parameter. +The following four options are available: + +* `info` (the default) +* `debug` +* `warning` +* `error` + +## Enabling and Disabling Debug Logging + +Checkout [Cluster Operations: Enabling and Disabling Debug Logging][cluster ops log] diff --git a/content/riak/kv/2.2.6/using/reference/multi-datacenter.md b/content/riak/kv/2.2.6/using/reference/multi-datacenter.md new file mode 100644 index 0000000000..adc1dd1828 --- /dev/null +++ b/content/riak/kv/2.2.6/using/reference/multi-datacenter.md @@ -0,0 +1,48 @@ +--- +title: "Multi-Datacenter Reference" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Multi-Datacenter" + identifier: "managing_ref_mdc" + weight: 113 + parent: "managing_ref" +toc: true +commercial_offering: true +--- + +[ref mdc stats]: ./statistics +[ref mdc per bucket]: ./per-bucket-replication +[ref mdc monitor]: ./monitoring +[ref mdc comparison]: ./comparison + +## In This Section + +#### [Multi-Datacenter Replication Reference: Statistics][ref mdc stats] + +Describes the output of `riak-repl status` interface. + +[Learn More >>][ref mdc stats] + + +#### [Multi-Datacenter Replication Reference: Per Bucket][ref mdc per bucket] + +Details enabling & disabling of per bucket replication. + +[Learn More >>][ref mdc per bucket] + + +#### [Multi-Datacenter Replication Reference: Monitoring][ref mdc monitor] + +Overview of monitoring in a Multi-Datacenter environment. + +[Learn More >>][ref mdc monitor] + + +#### [Multi-Datacenter Replication Reference: Comparison][ref mdc comparison] + +Compares Version 2 and Version 3 of Riak's Multi-Datacenter Replication capabilities. + +[Learn More >>][ref mdc comparison] diff --git a/content/riak/kv/2.2.6/using/reference/multi-datacenter/comparison.md b/content/riak/kv/2.2.6/using/reference/multi-datacenter/comparison.md new file mode 100644 index 0000000000..679595d99e --- /dev/null +++ b/content/riak/kv/2.2.6/using/reference/multi-datacenter/comparison.md @@ -0,0 +1,96 @@ +--- +title: "Multi-Datacenter Replication Reference: Comparsion" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Comparison" + identifier: "managing_ref_mdc_comparison" + weight: 103 + parent: "managing_ref_mdc" +toc: true +aliases: + - /riak-docs/riak/2.2.6/ops/mdc/comparison + - /riak-docs/riak/kv/2.2.6/ops/mdc/comparison +--- + +This document is a systematic comparison of [Version 2]({{<baseurl>}}riak/kv/2.2.6/using/reference/v2-multi-datacenter) and [Version 3]({{<baseurl>}}riak/kv/2.2.6/using/reference/v3-multi-datacenter) of Riak's Multi-Datacenter +Replication capabilities. + +{{% note title="Important note on mixing versions" %}} +If you are installing Riak anew, you should use version 3 +replication. Under no circumstances should you mix version 2 and version 3 +replication. This comparison is meant only to list improvements introduced in +version 3. +{{% /note %}} + +## Version 2 + +* Version 2 replication relies upon the twin concepts of **listeners** + and **sites**. Listeners are the sources of replication data, while + sites are the destination of replication data. Sites and listeners are + manually configured on each node in a cluster. This can be a burden to + the administrator as clusters become larger. +* A single connection tied to the **cluster leader** manages all + replication communications. This can cause performance problems on the + leader and is a bottleneck for realtime and fullsync replication data. +* Connections are established from site to listener. This can be + confusing for firewall administrators. +* The realtime replication queue will be lost if the replication + connection breaks, even if it's re-established. Reconciling data in + this situation would require manual intervention using either of the + following: + * a fullsync + * another Riak write to the key/value on the listener, thus + re-queueing the object +* Riak CS MDC `proxy_get` connections can only request data from a + single leader node + +### When to use version 2 replication + +* If you are running clusters below version 1.3.0 of Riak Enterprise, + version 2 replication is the only method of replication available. +* In the Riak 1.3 series, version 3 replication was provided as a + technology preview and did not have feature parity with version 2. + This was provided in the Riak 1.4 series. + +## Version 3 + +* Version 3 replication uses the twin concepts of **sources** and + **sinks**. A source is considered the primary provider of replication + data, whereas a sink is the destination of replication data. +* Establishing replication connections between clusters has been + greatly simplified. A single `riak-repl connect` command needs to be + issued from a source cluster to a sink cluster. IP and port + information of all nodes that can participate in replication on both + source and sink clusters are exchanged by the **replication cluster + manager**. The replication cluster manager also tracks nodes joining + and leaving the cluster dynamically. +* If the source has M nodes, and the sink has N nodes, there will be M + realtime connections. Connections aren't tied to a leader node as they + are with version 2 replication. +* Communications for realtime, fullsync, and `proxy_get` operations are + multiplexed over the same connection for each node participating in + replication. This reduces the amount of firewall configuration on both + sources and sinks. +* A fullsync coordinator runs on a leader of the source cluster. The + coordinator assigns work across nodes in the sources cluster in an + optimized fashion. +* Realtime replication establishes a bounded queue on each source node + that is shared between *all* sinks. This queue requires consumers to + acknowledge objects when they have been replicated. Dropped TCP + connections won't drop objects from the queue. +* If a node in the source cluster is shut down via the command line, a + realtime replication queue is migrated to other running nodes in the + source cluster. +* Network statistics are kept per socket. +* Fullsyncs between clusters can be tuned to control the maximum number + of workers that will run on a source node, a sink node, and across the + entire source cluster. This allows for limiting impact on the cluster + and dialing in fullsync performance. +* Version 3 is able to take advantage of [Active Anti-Entropy]({{<baseurl>}}riak/kv/2.2.6/learn/concepts/active-anti-entropy/) \(AAE) + technology, which can greatly improve fullsync performance. +* Riak CS MDC `proxy_get` connections will be distributed across the + source cluster (as CS blocks are requested from the sink cluster in + this scenario). diff --git a/content/riak/kv/2.2.6/using/reference/multi-datacenter/monitoring.md b/content/riak/kv/2.2.6/using/reference/multi-datacenter/monitoring.md new file mode 100644 index 0000000000..7cdb005bac --- /dev/null +++ b/content/riak/kv/2.2.6/using/reference/multi-datacenter/monitoring.md @@ -0,0 +1,170 @@ +--- +title: "Multi-Datacenter Replication Reference: Monitoring" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Monitoring" + identifier: "managing_ref_mdc_monitor" + weight: 102 + parent: "managing_ref_mdc" +toc: true +aliases: + - /riak-docs/riak/2.2.6/ops/mdc/monitoring + - /riak-docs/riak/kv/2.2.6/ops/mdc/monitoring +--- + +Monitoring Riak's realtime replication allows you to identify trends and +to receive alerts during times when replication is halted or delayed. +Issues or delays in replication can be caused by: + +* Sudden increases or spikes in write traffic +* Network connectivity issues or outages +* Errors experienced in Riak + +Identification and trending of issues or delays in realtime replication +is important for identifying a root cause, while alerting is important +for addressing any SLA-impacting issues or delays. We recommend +combining the two approaches below when monitoring Riak's realtime +replication: + +* Monitor Riak's replication status output, from either `riak-repl + status` or the HTTP `/riak-repl/stats` endpoint +* Use canary (test) objects to test replication and establish trip times + from source to sink clusters + +{{% note title="Note on querying and time windows" %}} +Riak's statistics are calculated over a sliding 60-second window. Each time +you query the stats interface, each sliding statistic shown is a sum or +histogram value calculated from the previous 60 seconds of data. Because of +this, the stats interface should not be queried more than once per minute. +{{% /note %}} + +## Statistics + +The following questions can be answered through the monitoring and +graphing of realtime replication statistics: + +* Is the realtime replication queue backed up? +* Have any errors occurred on either the source or sink cluster? +* Have any objects been dropped from the realtime queue? + +--- + +#### Is the realtime replication queue backed up? + +Identifying times when the realtime replication queue experiences +increases in the number of `pending` objects can help identify problems +with realtime replication or identify times when replication becomes +overloaded due to increases in traffic. The `pending` statistic, found +under the `realtime_queue_stats` section of the replication status +output, should be monitored and graphed. Graphing this statistic allows +you to identify trends in the number of `pending` objects. Any repeating +or predictable trend in this statistic can be used to help identify a +need for tuning and capacity changes, while unexpected variation in this +statistic may indicate either sudden changes in load or errors at the +network, system, or Riak level. + +#### Have any errors occurred on either the source or sink cluster? + +Errors experienced on either the source or sink cluster can result in +failure to replicate object(s) via realtime replication. The top-level +`rt_dirty` statistic in `riak-repl status` indicates whether such an +error has occurred and how many times. This statistic only tracks +errors and does not definitively indicate that an object was not +successfully replicated. For this reason, a fullsync should be performed +any time `rt_dirty` is non-zero. `rt_dirty` is then reset to zero once a +fullsync successfully completes. + +The size of `rt_dirty` can quantify the number of errors that have +occurred and should be graphed. Since any non-zero value indicates an +error, an alert should be set so that a fullsync can be performed (if +not regularly scheduled). Like realtime queue back ups, trends in +`rt_dirty` can reveal problems with the network, system, or Riak. + +#### Have any objects been dropped from the realtime queue? + +The realtime replication queue will drop objects when the queue is full, +with the dropped object(s) being the last (oldest) in the queue. Each +time an object is dropped, the `drops` statistic, which can be found +under the `realtime_queue_stats` section of the replication status +output, is incremented. An object dropped from the queue has not been +replicated successfully, and a fullsync should be performed when a drop +occurs. A dropped object can indicate a halt or delay in replication or +indicate that the realtime queue is overloaded. In cases of high load, +increases to the maximum size of the queue (displayed in the +`realtime_queue_stats` section of the replication status output as +`max_bytes`) can be made to accommodate a usage pattern of expected high +load. + +--- + +Although the above statistics have been highlighted to answer specific +questions, other statistics can also be helpful in diagnosing issues +with realtime replication. We recommend graphing any statistic that is +reported as a number. While their values and trends may not answer +common questions or those we've highlighted here, they may nonetheless +be important when investigating issues in the future. Other questions +that cannot be answered through statistics alone may be addressed +through the use of canary objects. + +### Canary Objects + +Canary object testing is a technique that uses a test object stored in +your environment with your production data but not used or modified by +your application. This allows the test object to have predictable states +and to be used to answer questions about the functionality and duration +of realtime replication. + +The general process for using canary objects to test realtime replication is: + +* Perform a GET for your canary object on both your source and sink + clusters, noting their states. The state of the object in each cluster + can be referred to as state `S0`, or the object's initial state. +* PUT an update for your canary object to the source cluster, updating + the state of the object to the next state, `S1`. +* Perform a GET for your canary on the sink cluster, comparing the state + of the object on the source cluster to the state of the object on the + sink cluster. + +By expanding upon the general process above, the following questions can +be answered: + +* Is a backed-up realtime replication queue still replicating objects + within a defined SLA? +* How long is it taking for objects to be replicated from the source + cluster to the sink cluster? + +#### Is a backed-up realtime replication queue still replicating objects within a defined SLA? + +Building on the final step of the general process, we can determine if +our objects are being replicated from the source cluster to the sink +cluster within a certain SLA time period by adding the following steps: + +- If the state of the object on the source cluster is not equal to the + state of the object on the sink cluster, repeat step 3 until an SLA + time threshold is exceeded. +- If the SLA time threshold is exceeded, alert that replication is not + meeting the necessary SLA. + +#### How long is it taking for objects to be replicated from the source cluster to the sink cluster? + +Getting a rough estimate of how long it takes an object PUT to a source +cluster to be replicated to a sink cluster get be done by either: + +* Comparing the time the object was PUT to the source with the time the + states of the object in the source and sink were equivalent +* Comparing the timestamps of the object on the source and sink when the + states are equivalent + +These are rough estimates, as neither method is 100% accurate. The first +method relies on a timestamp for a GET and subsequent successful +comparison, which means that the object was replicated prior to that +timestamp; the second method relies on the system clocks of two +different machines, which may not be in sync. + +It's important to note that each node in a cluster has its own realtime +replication queue. The general process needs to be applied to every +node in the source cluster, with a variety of canary objects and states, +to get a complete picture of realtime replication between two clusters. diff --git a/content/riak/kv/2.2.6/using/reference/multi-datacenter/per-bucket-replication.md b/content/riak/kv/2.2.6/using/reference/multi-datacenter/per-bucket-replication.md new file mode 100644 index 0000000000..2ab72328c5 --- /dev/null +++ b/content/riak/kv/2.2.6/using/reference/multi-datacenter/per-bucket-replication.md @@ -0,0 +1,62 @@ +--- +title: "Multi-Datacenter Replication Reference: Per Bucket" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Per Bucket" + identifier: "managing_ref_mdc_per_bucket" + weight: 101 + parent: "managing_ref_mdc" +toc: true +aliases: + - /riak-docs/riak/2.2.6/ops/mdc/per-bucket + - /riak-docs/riak/kv/2.2.6/ops/mdc/per-bucket +--- + +To enable or disable replication per bucket, you can use the `repl` +bucket property. + +Some changes have occurred between 1.1 and 1.2. + +These `repl` values are available in Riak Enterprise version 1.1 and +above: + + * `true` --- Enable replication (realtime + fullsync) + * `false` --- Disable replication (realtime + fullsync) + +These option values are only available in Riak Enterprise version 1.2 +and above: + + * `realtime` --- Replication only occurs in realtime for this bucket + * `fullsync` --- Replication only occurs during a fullsync operation + * `both` --- Replication occurs in realtime and during fullsync + +### Example of Disabling + +```curl +curl -v -XPUT http://127.0.0.1:8098/buckets/my_bucket \ + -H "Content-Type: application/json" \ + -d '{"props":{"repl":false}}' +``` + +### Example of Enabling + +```curl +curl -v -XPUT http://127.0.0.1:8098/buckets/my_bucket \ + -H "Content-Type: application/json" \ + -d '{"props":{"repl":true}}' +``` + +## How Bucket Properties Work in Riak KV + +When using Multi-Datacenter Replication, each bucket's write properties +are derived from the bucket's properties in the destination cluster. If +the bucket doesn't exist, the default properties of the destination +cluster are used. + +It's important to note that this goes for properties such as `backend`. +If the bucket doesn't exist in the destination cluster, Riak will create +it with the default backend and _not_ with the backend used in the +source cluster. diff --git a/content/riak/kv/2.2.6/using/reference/multi-datacenter/statistics.md b/content/riak/kv/2.2.6/using/reference/multi-datacenter/statistics.md new file mode 100644 index 0000000000..620125043e --- /dev/null +++ b/content/riak/kv/2.2.6/using/reference/multi-datacenter/statistics.md @@ -0,0 +1,240 @@ +--- +title: "Multi-Datacenter Replication Reference: Statistics" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Statistics" + identifier: "managing_ref_mdc_stats" + weight: 100 + parent: "managing_ref_mdc" +toc: true +aliases: + - /riak-docs/riak/2.2.6/ops/mdc/statistics + - /riak-docs/riak/kv/2.2.6/ops/mdc/statistics +--- + +The following definitions describe the output of `riak-repl status`. +Both Version 2 and Version 3 Replication statistics can be obtained +using the `riak-repl status` command. + +There are two things that you should note: + +1. Many of these statistics will appear only on the current + leader node +2. The counts for all statistics will be reset to 0 upon restarting Riak + Riak unless otherwise noted + +Field | Description +:-----|:---------- +`cluster_leader` | Which node is the current leader of the cluster +`connected_clusters` | A list of all sink clusters to which this source is connected + +## Performance + +The `riak-repl status` command should not be executed more than once a +minute, as statistics are recalculated every time the command is +executed, and some statistics require network communication between +nodes. This performance note also applies to the HTTP `/riak-repl/stats` +endpoint. + +## Realtime Replication Statistics + +Statistics for both the source or sink sides of realtime replication. +These values can be found under either `sources.source_stats` or +`sinks.sink_stats`. + +Field | Description +------|------------ +`realtime_enabled` | A list of all realtime sinks that are enabled +`realtime_started` | A list of all realtime sinks that are started +`rt_dirty` | The number of errors detected that can prevent objects from being replicated via realtime. These include errors on the source or sink connection, or realtime queue overload resulting in objects being dropped from the queue. *This value will persist across restarts until a fullsync is complete.* +`rt_sink_errors` | A sink error has been detected on the source node. This value will be reset to 0 after a node restarts. +`rt_sink_connected_to.source_drops` | The number of dropped put transfers from the perspective of the sink cluster +`rt_source_errors` | A source error has been detected on the source node. This value will be reset to 0 after a node restarts. + +Field | Description +------|------------ +`rt_source_connected_to` | The name of the sink cluster to which the source cluster is connected +`rt_sink_connected_to` | The name of the source cluster to which the sink cluster is connected +`connected` | If `true`, then the source is connected to a sink (or vice versa) +`objects` | The number of realtime replication objects that have been successfully transmitted to the sink cluster +`sent_seq` | The last realtime queue sequence number that has been transmitted +`acked_seq` | The last realtime queue sequence number that has been acknowledged +`expect_seq` | The next realtime queue sequence number that is expected +`hb_rtt` | Realtime replication heartbeat round-trip time in milliseconds, recorded on the replication source +`hb_last` | `{MegaSeconds, Seconds, MicroSeconds}` since a heartbeat message was received on the realtime sink + + +These values are under `realtime_queue_stats`. + +Field | Description +------|------------ +`bytes` | The size in bytes of all objects currently in the realtime queue +`consumers` | A list of source consumers of the realtime queue +`consumers.<clustername>.drops` | The number of dropped realtime sync put transfers per sink cluster, from the perspective of the source cluster ("dropped" in this context meaning either that the outgoing data queue was full or that there was a connection error) +`drops` | The number of objects dropped from the realtime queue as the result of the queue being full or other errors +`errs` | The number of errors while pushing/popping from the realtime queue +`overload_drops` | The number of put transfers that have been dropped due to an overload of the message queue of the Erlang process responsible for processing outgoing transfers +`pending` | The number of objects waiting to be sent to the sink cluster +`sinkclustername` | A consumer of the realtime queue +`unacked` | The number of objects waiting to be acknowledged by a queue consumer + + +## Fullsync Replication Statistics + +Field | Description +------|------------ +`fullsync_enabled` | A list of all sinks that are enabled +`fullsync_running` | A list of all sinks that are running +`server_fullsyncs` | The number of fullsync operations that have occurred since the server was started +`fullsyncs_completed` | The number of fullsyncs that have been completed to the specified sink cluster. +`fullsync_start_time` | The time the current fullsink to the specified cluster began. +`last_fullsync_duration `| The duration (in seconds) of the last completed fullsync. + +If this cluster is acting as a **source**, the `fullsync_coordinator` field returns a list of `{<sink_clustername>:<fullsync_stats>}`. If this cluster is acting as a **sink**, the `fullsync_coordinator_srv` field returns a list of `{<LocalIP:Port>:<fullsync_coordinator_srv_stats>}`. + +Those fields are described in the following tables. + +Field | Description +------|------------ +`cluster` | The name of the sink cluster +`queued` | The number of partitions that are waiting for an available process +`in_progress` | The number of partitions that are being synced +`starting` | The number of partitions connecting to remote cluster +`successful_exits` | The number of partitions successfully synced. When completed, this will be the same number as total number of partitions in the ring. +`error_exits` | If a sync failed or was aborted, the partition will be queued again and try again later +`running_stats` | `[{<PID>, <stats>},…]` Any running sync processes are listed here, and described in the table below +`socket` | See [Socket Statistics](#socket-statistics) +`fullsync_suggested` | Realtime replication errors occurred on these nodes, a fullsync is suggested +`fullsync_suggested_during_fs` | Realtime replication errors occurred on these nodes while a fullsync is already in progress. A fullsync is suggested after the current fullsync completes. These value will be moved to the `fullsync_suggested` value when the current fullsync completes. +`socket` | `{peername: <RemoteIP:Port>`, `sockname: <LocalIP:Port>}` + +The `running_stats` field contains the following fields. + +Field | Description +------|------------ +`node` | The local cluster source node currently participating in fullsync replication +`site` | The name of the sink cluster. *Warning: This will be renamed in future versions of Riak*. +`strategy` | The strategy that fulfills fullsync replication. In previous versions of replication, different values could be configured. This value could be changed depending on your replication needs. +`fullsync_worker` | The Erlang process id of the fullsync worker. +`socket` | See [Socket Statistics](#socket-statistics) +`state` | The current state of fullsync replication. This can be used by Basho support to identify replication issues.<ul><li>**`wait_for_partition`**</li><li>**`build_keylist`**</li><li>**`wait_keylist`**</li><li>**`diff_bloom`**</li><li>**`diff_keylist`**</li></ul> +`fullsync` | The partition that is currently being synchronized with the sink cluster +`partition_start` | Elapsed time in seconds since the *fullsync* partition started replication to a sink +`stage_start` | Elapsed time in seconds since the `state` started running on the source +`get_pool_size` | The number of workers that are used to read data from Riak during a fullsync + +## Socket Statistics + +Many sections of the status output include a `socket` section. A reading is taken once every 10 seconds, and the last 7 readings are stored. + +Field | Description +------|------------ +`peername` | `<ip:port>` The address and port for the other end of a connection +`recv_avg` | The average size of packets in bytes received to the socket +`recv_cnt` | The number of packets received by the socket +`recv_dvi` | The average packet size deviation in bytes received by the socket +`recv_kbps` | Socket kilobits/second received +`recv_max` | Size of the largest packet in bytes received to the socket +`send_cnt` | Number of packets sent from the socket +`send_kbps` | Socket kilobits/second sent +`send_pend` | The number of bytes in the Erlang VM to be sent over the socket +`sockname` | `<host:port>` The address and port for "this end" of the connection + +## Version 2 Replication Statistics + +The following definitions describe the output of `riak-repl status`. +Please note that many of these statistics will only appear on the +current leader node. + +**Note**: All counts will be reset to 0 upon restarting Riak. + +Field | Description +------|------------ +`listener_[nodeid]` | Defines a replication listener (primary) that is running on node `[nodeid]` +`[sitename]_ips` | Defines a replication skin +`client_bytes_recv` | The total number of bytes the client has received since the server has been started +`client_bytes_sent` | The total number of bytes sent to all connected secondaries +`client_connect_errors` | The number of TCP/IP connection errors +`client_connects` | A count of the number of sink connections made to this node. +`client_redirect` | If a client connects to a non-leader node, it will be redirected to a leader node +`client_rx_kbps` | A snapshot of the sink received kilobits/second taken once a minute. The past 8 snapshots are stored in this list. Newest snapshots appear on the left side of the list. +`client_tx_kbps` | A snapshot of the sink sent kilobits/second taken once a minute. The past 8 snapshots are stored in this list. Newest snapshots appear on the left side of the list. +`elections_elected` | If the replication leader node becomes unresponsive or unavailable, a new leader node in the cluster will be elected +`elections_leader_changed` | The number of times a Riak node has surrendered leadership +`objects_dropped_no_clients` | If the realtime replication work queue is full and there are no clients to receive objects, then objects will be dropped from the queue. These objects will be synchronized during a fullsync operation. +`objects_dropped_no_leader` | If a sink cannot connect to a leader, objects will be dropped during realtime replication +`objects_forwarded` | The number of Riak objects forwarded to the leader the participate in replication. *Please note that this value will only be accurate on a non-leader node*. +`objects_sent` | The number of objects sent via realtime replication +`server_bytes_recv` | The total number of bytes the primary has received +`server_bytes_sent` | The total number of bytes the primary has sent +`server_connect_errors` | The number of primary to sink connection errors +`server_connects` | The number of times the primary connects to the client sink +`server_rx_kbps` | A snapshot of the primary received kilobits/second taken once a minute. The past 8 snapshots are stored in this list. Newest snapshots appear on the left side of the list +`server_tx_kbps` | A snapshot of the primary sent kilobits/second taken once a minute. The past 8 snapshots are stored in this list. Newest snapshots appear on the left side of the list. +`leader` | Which node is the current leader of the cluster for Version 2 Replication +`local_leader_message_queue_len` | The length of the object queue on the leader +`local_leader_heap_size` | The amount of memory the leader is using +`client_stats` | See [Client Statistics](#client-statistics) +`server_stats` | See [Server Statistics](#server-statistics) + +## Client Statistics + +Field | Description +------|------------ +`node` | A unique ID for the Riak node that the sink in running on +`site` | The connected site (sink) name. **Warning**: This will be renamed in a future version of Riak. +`strategy` | A replication strategy defines an implementation of the Riak Replication protocol. Valid values: `keylist`, `syncv1`. +`fullsync_worker` | The Erlang process ID of the fullsync worker +`waiting_to_retry` | The primaries currently waiting to retry replication after a failure +`connected` | A list of connected clients<ul><li>**`connected`** The IP address and port of a connected sink</li><li>**`cluster_name`** The name of the connected sink</li><li>**`connecting`** The PID, IP address, and port of a client currently establishing a connection</li></ul> +`state` | State shows what the current replication strategy is currently processing. The following definitions appear in the status output if keylist strategy is being used. They can be used by Basho support to identify replication issues.<ul><li>**`request_partition`**</li><li>**`wait_for_fullsync`**</li><li>**`send_keylist`**</li><li>**`wait_ack`**</li></ul> + + +## Server Statistics + +Field | Description +------|------------ +`node` | A unique ID for the Riak node that the source is running on +`site` | The connected site (sink) name configured with. *Warning: This will be renamed in a future version of Riak*. +`strategy` | A replication strategy defines an implementation of the Riak Replication protocol. Valid values: `keylist`, `syncv1`. +`fullsync_worker` | The Erlang process ID of the fullsync worker +`bounded_queue` | See [Bounded Queue](#bounded-queue) +`state` | State shows what the current replication strategy is currently processing. The following definitions appear in the status output if keylist strategy is being used. They can be used by Basho support to identify replication issues.<ul><li>**`wait_for_partition`**</li><li>**`build_keylist`**</li><li>**`wait_keylist`**</li><li>**`diff_bloom`**</li><li>**`diff_keylist`**</li></ul> +`message_queue_len` | The number of Erlang messages that are waiting to be processed by the server + + +## Bounded Queue + +The bounded queue is responsible for holding objects that are waiting to +participate in realtime replication. Please see the [Riak V2 MDC Replication Configuration][config v2 mdc] or [Riak V3 MDC Replication Configuration][config v3 mdc] guides for +more information. + +Field | Description +------|------------ +`queue_pid` | The Erlang process ID of the bounded queue +`dropped_count` | The number of objects that failed to be enqueued in the bounded queue due to the queue being full. *These objects will be replicated during the next fullsync operation*. +`queue_length` | The number of Riak objects currently in the bounded queue +`queue_byte_size` | The size of all objects currently in the queue +`queue_max_size `| The number of bytes the queue can hold before objects are dropped. *These objects will be replicated during the next fullsync operation*. +`queue_percentage` | The percentage of the queue that is full +`queue_pending` | The current count of "in-flight" objects we've sent that the client has not acknowledged +`queue_max_pending` | The maximum number of objects that can be "in flight" before we refuse to send any more. + + +## Accessing Replication Web-Based Statistics + +These stats can be accessed via the command line with the following +command: + +```curl +curl -q http://127.0.0.1:8098/riak-repl/stats +``` + +A simple way to view formatted statistics is to use a command such as: + +```curl +curl -q http://127.0.0.1:8098/riak-repl/stats | jsonpp +``` diff --git a/content/riak/kv/2.2.6/using/reference/object-deletion.md b/content/riak/kv/2.2.6/using/reference/object-deletion.md new file mode 100644 index 0000000000..060160cfbb --- /dev/null +++ b/content/riak/kv/2.2.6/using/reference/object-deletion.md @@ -0,0 +1,117 @@ +--- +title: "Object Deletion Reference" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Object Deletion" + identifier: "managing_ref_object_deletion" + weight: 103 + parent: "managing_ref" +toc: true +aliases: + - /riak-docs/riak/2.2.6/ops/advanced/deletion +--- + +[concept eventual consistency]: ../../../learn/concepts/eventual-consistency +[concept clusters]: ../../../learn/concepts/clusters +[glossary vnode]: ../../../learn/glossary/#vnode +[usage delete objects]: ../../../developing/usage/deleting-objects +[developing keylist]: ../../../developing/api/http/list-keys +[developing mapreduce]: ../../../developing/usage/mapreduce +[cluster mdc]: ../../cluster-operations/v3-multi-datacenter +[config advanced]: ../../../configuring/reference/#advanced-configuration +[glossary sloppy quorum]: ../../../learn/glossary/#sloppy-quorum +[bitcask merging]: ../../../setup/planning/backend/bitcask/#disk-usage-and-merging-settings +[leveldb compaction]: ../../../setup/planning/backend/leveldb/#compaction + +In single-server, non-clustered data storage systems, object deletion +is a trivial process. In an [eventually consistent][concept eventual consistency], [clustered][concept clusters] system like Riak, however, +object deletion is far less trivial because objects live on multiple +[nodes](../../../learn/glossary/#nodes), which means that a deletion process must be chosen to determine when an object can be removed from the storage backend. + +## Object Deletion Example + +The problem of object deletion in distributed systems can be illustrated more concretely using the following example: + +* An object is stored on nodes A, B, and C +* Node C suddenly goes offline due to a network failure +* A client sends a delete request to node A, which forwards that + request to node B, but it cannot reach node C +* On nodes A and B, the object is deleted +* Node C comes back online +* A client attempts to read the object, and the request hits node C +* Node C asks nodes A and B for the object, but they return `not_found`. Node C, on the other hand, still has the object. + +The question here is: how should node C respond to the client? Given only the above information, it isn't possible to determine which of two possible scenarios actually occurred: + +1. the object was deleted on A & B but not on C +2. the object was created on C but not on A & B + +To get around this problem, Riak uses *Tombstones*. + +## Tombstones + +Riak addresses the problem of deletion in distributed systems by replacing the deleted object with a special object called a **tombstone** rather than just removing it. + +This allows Riak to understand the difference between an object that has been deleted, and one that was never written in the first place. A tombstone specifically has `X-Riak-Deleted` = `true` in the metadata and a value of `<<>>` (the Erlang empty binary) in its contents, whereas an unwritten object has *no entry at all*. + +The problem with tombstones is that they take up space, albeit not very much. For this reason, Riak can be configured to automatically remove tombstones after a set period of time. This process is called **reaping**. + +After being reaped, a tombstone is completely removed, and the object entry ceases to exist entirely (as if it had never been written to). + +## Configuring Object Deletion + +The `delete_mode` setting in a cluster's [configuration files][config advanced] will determine how long a tombstone will remain before being reaped. + +There are three possible settings: + +* `keep` --- Disables tombstone removal +* `immediate` --- The tombstone is removed as soon as the request is + received +* Custom time interval --- How long to wait until the tombstone is + removed, expressed in milliseconds. The default is `3000`, i.e. to + wait 3 seconds + +In general, we recommend setting the `delete_mode` parameter to `keep` +if you plan to delete and recreate objects under the same key. This protects against failure scenario cases in which a deleted object may be resurrected. + +Setting `delete_mode` to `immediate` can be useful in situations in +which an aggressive space reclamation process is necessary, such as +when running [MapReduce jobs][developing mapreduce], but we do not recommend +this in general. + +Setting `delete_mode` to a longer time duration than the default can be +useful in certain cases involving [Multi-Datacenter Replication][cluster mdc], e.g. when +network connectivity is an issue. + +## Deletion from Backends + +When attempting to reclaim disk space, deleting data may seem like the obvious first step. However, in Riak this is not necessarily the best thing to do if the disk is nearly full. This is because Riak's disk-based backends don't remove data immediately. This is true both for the initial deletion when a Riak tombstone is created, and later when that tombstone is reaped. + +In the case of Bitcask, a new entry is written in the log with either the Riak tombstone or, after reaping, a Bitcask tombstone. The in-memory key-pointer is then updated to point to this new value. + +In LevelDB, a newly written value obscures the earlier value. Again, this is either the Riak tombstone or, after reaping, a LevelDB tombstone. + +Some time later, the backends will perform their regular garbage collection procedures. For Bitcask this is [merging][bitcask merging], for LevelDB it is [compaction][leveldb compaction]. At this time, stale entries containing the original objects will be purged from disk, along with any Bitcask or LevelDB tombstones. Riak tombstones will *not* be purged, because the backends treat them like regular objects. + +Thus, reclaiming disk space is not immediate with respect to delete operations, nor even with respect to reaping, and prior to garbage collection delete operations will actually cause disk space usage to rise slightly. + +## Tombstones & Reporting + +When designing applications and operating a live Riak cluster, it is important to know how to interpret Riak's responses to requests. With respect to deletion and tombstones, please note the following: + +* A delete request is considered a special case of an update. It will fail if the `W` and `PW` values are not satisfied. However, due to [Sloppy Quorum][glossary sloppy quorum], deletes will usually succeed. This does not mean that tombstones have been written over *all* copies of the object, but rather that tombstones have been written on at least `W` nodes, of which at least `PW` are primaries. +* Successful delete requests do not guarantee successful reaping. If a node fails before its reap timer expires, the reap timer will not automatically recommence upon restart. Rather, the tombstone will remain upon the node until a further request finds it. At this time, a new reap timer will be initiated. +* A GET request that sees a quorum of Riak tombstones will return a `not_found` response to the client, even though internally Riak knows there used to be an object there. +* A GET request will never see backend tombstones, because the backends report these as `not_found`. To RiakKV, such answers will appear as if the object has never been written. A `not_found` will be sent up to the client in this case, too. +* A [Keylist][developing keylist] or [MapReduce][developing mapreduce] operation *will* return Riak tombstones, but *not* backend tombstones. This is because these operations fold over the backends directly, and make no attempt to filter Riak tombstones out of the fold by default. + +## Client Library Examples + +Check out [Deleting Objects][usage delete objects] in the Developing section for examples of deleting objects client-side. + +## Resources + +* [Discussion on the Riak mailing list](http://lists.basho.com/pipermail/riak-users_lists.basho.com/2011-October/006048.html) diff --git a/content/riak/kv/2.2.6/using/reference/runtime-interaction.md b/content/riak/kv/2.2.6/using/reference/runtime-interaction.md new file mode 100644 index 0000000000..6027907811 --- /dev/null +++ b/content/riak/kv/2.2.6/using/reference/runtime-interaction.md @@ -0,0 +1,66 @@ +--- +title: "Runtime Interaction Reference" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Runtime Interaction" + identifier: "managing_ref_runtime_interaction" + weight: 104 + parent: "managing_ref" +toc: true +aliases: + - /riak-docs/riak/2.2.6/ops/advanced/runtime + - /riak-docs/riak/kv/2.2.6/ops/advanced/runtime +--- + +[config reference]: {{<baseurl>}}riak/kv/2.2.6/configuring/reference +[concept clusters]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/clusters + +Riak's [configuration files][config reference] provide a variety of parameters that +enable you to fine-tune how Riak interacts with two important elements +of the underlying operating system: distribution ports and OS +processes/garbage collection. + +## Ports + +Distribution ports connect Riak nodes within a [cluster][concept clusters]. The +following port-related parameters are available: + +* `runtime_health.triggers.distribution_port` --- Whether distribution + ports with full input buffers will be counted as busy. + * Default: `on` +* `runtime_health.triggers.port` --- Whether ports with full input + buffers will be counted as busy. Ports can represent open files or network sockets. + * Default: `on` +* `runtime_health.thresholds.busy_ports` --- The threshold at which a + warning will be triggered about the number of ports that are overly + busy. Ports with full input buffers count toward this threshold. + * Default: `2` + +## Processes + +Riak will log warnings related to busy operating system processes and +garbage collection. You can specify the conditions in which warnings are +triggered using the following parameters: + +* `runtime_health.thresholds.busy_processes` --- The threshold at which + a warning will be triggered about the number of processes that are + overly busy. Processes with large heaps or that take a long time to + garbage collect will count toward this threshold. + * Default: `30` +* `runtime_health.triggers.process.heap_size` --- A process will be + marked as busy when its size exceeds this size (in bytes). + * Default: `160444000` +* `runtime_health.triggers.process.garbage_collection` --- A process + will be marked as busy when it exceeds this amount of time doing + garbage collection. Enabling this setting can cause performance + problems on multi-core systems. + * Default: `off` + * Example when enabled: `50ms` +* `runtime_health.triggers.process.long_schedule` --- A process will + become busy when it exceeds this length of time during a single + process scheduling and execution cycle. + * Default: `off` + * Example when enabled: `20ms` diff --git a/content/riak/kv/2.2.6/using/reference/search.md b/content/riak/kv/2.2.6/using/reference/search.md new file mode 100644 index 0000000000..84ba0fee70 --- /dev/null +++ b/content/riak/kv/2.2.6/using/reference/search.md @@ -0,0 +1,454 @@ +--- +title: "Search Reference" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Search" + identifier: "managing_ref_search" + weight: 109 + parent: "managing_ref" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/advanced/search + - /riak-docs/riak/kv/2.2.6/dev/advanced/search +--- + +[concept clusters]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/clusters +[configuring search]: {{<baseurl>}}riak/kv/2.2.6/configuring/search + +> **Note on search 2.0 vs. legacy search** +> +> This document refers to Riak search 2.0 with +[Solr](http://lucene.apache.org/solr/) integration (codenamed +Yokozuna). For information about the deprecated Riak search, visit [the old Using Riak search docs](http://docs.basho.com/riak/1.4.10/dev/using/search/). + +The project that implements Riak search is codenamed Yokozuna. This is a +more detailed overview of the concepts and reasons behind the design of +Yokozuna, for those interested. If you're simply looking to use Riak +search, you should check out the [Using Search]({{<baseurl>}}riak/kv/2.2.6/developing/usage/search) document. + +![Yokozuna]({{<baseurl>}}images/yokozuna.png) + +## Riak Search is Erlang + +In Erlang OTP, an "application" is a group of modules and Erlang +processes which together perform a specific task. The word application +is confusing because most people think of an application as an entire +program such as Emacs or Photoshop. But Riak Search is just a sub-system +in Riak itself. Erlang applications are often stand-alone, but Riak +Search is more like an appendage of Riak. It requires other subsystems +like Riak Core and KV, but also extends their functionality by providing +search capabilities for KV data. + +The purpose of Riak Search is to bring more sophisticated and robust +query and search support to Riak. Many people consider Lucene and +programs built on top of it, such as Solr, as the standard for +open-source search. There are many successful applications built on +Lucene/Solr, and it sets the standard for the feature set that +developers and users expect. Meanwhile, Riak has a great story as a +highly-available, distributed key/value store. Riak Search takes +advantage of the fact that Riak already knows how to do the distributed +bits, combining its feature set with that of Solr, taking advantage of +the strengths of each. + +Riak Search is a mediator between Riak and Solr. There is nothing +stopping a user from deploying these two programs separately, but this +would leave the user responsible for the glue between them. That glue +can be tricky to write. It requires dealing with monitoring, querying, +indexing, and dissemination of information. + +Unlike Solr by itself, Riak Search knows how to do all of the following: + +* Listen for changes in key/value (KV) data and to make the appropriate + changes to indexes that live in Solr. It also knows how to take a user + query on any node and convert it to a Solr distributed search, which + will correctly cover the entire index without overlap in replicas. +* Take index creation commands and disseminate that information across + the cluster. +* Communicate and monitor the Solr OS process. + +## Solr/JVM OS Process + +Every node in a Riak [cluster][concept clusters] has a corresponding operating +system (OS) process running a JVM which hosts Solr on the Jetty +application server. This OS process is a child of the Erlang OS process +running Riak. + +Riak Search has a `gen_server` process which monitors the JVM OS +process. The code for this server is in `yz_solr_proc`. When the JVM +process crashes, this server crashes, causing its supervisor to restart +it. + +If there is more than 1 restart in 45 seconds, the entire Riak node will +be shut down. If Riak Search is enabled and Solr cannot function for +some reason, the Riak node needs to go down so that the user will notice +and take corrective action. + +Conversely, the JVM process monitors the Riak process. If for any reason +Riak goes down hard (e.g. a segfault) the JVM process will also exit. +This double monitoring along with the crash semantics means that neither +process may exist without the other. They are either both up or both +down. + +All other communication between Riak Search and Solr is performed via +HTTP, including querying, indexing, and administration commands. The +ibrowse Erlang HTTP client is used to manage these communications as +both it and the Jetty container hosting Solr pool HTTP connections, +allowing for reuse. Moreover, since there is no `gen_server` involved in +this communication, there's no serialization point to bottleneck. + +## Indexes + +An index, stored as a set of files on disk, is a logical namespace that +contains index entries for objects. Each such index maintains its own +set of files on disk---a critical difference from Riak KV, in which a +bucket is a purely logical entity and not physically disjoint at all. A +Solr index requires significantly less disk space than the corresponding +legacy Riak Search index, depending on the Solr schema used. + +Indexes may be associated with zero or more buckets. At creation time, +however, each index has no associated buckets---unlike the legacy Riak +Search, indexes in the new Riak Search do not implicitly create bucket +associations, meaning that this must be done as a separate configuration +step. + +To associate a bucket with an index, the bucket property `search_index` must +be set to the name of the index you wish to associate. Conversely, in +order to disassociate a bucket you use the sentinel value +`_dont_index_`. + +Many buckets can be associated with the same index. This is useful for +logically partitioning data into different KV buckets which are of the +same type of data, for example if a user wanted to store event objects +but logically partition them in KV by using a date as the bucket name. + +A bucket _cannot_ be associated with many indexes---the `search_index` +property must be a single name, not a list. + +See the [main Search documentation]({{<baseurl>}}riak/kv/2.2.6/developing/usage/search/#simple-setup) for details on creating an index. + +## Extractors + +There is a tension between Riak KV and Solr when it comes to data. Riak +KV treats object values as mostly opaque, and while KV does maintain an +associated content type, it is simply treated as metadata to be returned +to the user to provide context for interpreting the returned object. +Otherwise, the user wouldn't know what type of data it is! + +Solr, on the other hand, wants semi-structured data, more specifically a +flat collection of field-value pairs. "Flat" here means that a field's +value cannot be a nested structure of field-value pairs; the values are +treated as-is (non-composite is another way to say it). + +Because of this mismatch between KV and Solr, Riak Search must act as a +mediator between the two, meaning it must have a way to inspect a KV +object and create a structure which Solr can ingest for indexing. In +Solr this structure is called a **document**. This task of creating a +Solr document from a Riak object is the job of the **extractor**. To +perform this task two things must be considered. + +**Note**: This isn't quite right, the fields created by the extractor +are only a subset of the fields created. Special fields needed for +Yokozuna to properly query data and tagging fields are also created. +This call happens inside `yz_doc:make_doc`. + +1. Does an extractor exist to map the content-type of the object to a + Solr document? +2. If so, how is the object's value mapped from one to the other? + For example, the value may be `application/json` which contains + nested objects. This must somehow be transformed into a flat + structure. + +The first question is answered by the _extractor mapping_. By default +Yokozuna ships with extractors for several common data types. Below is a +table of this default mapping: + +Content Type | Erlang Module +:------------|:------------- +`application/json` | `yz_json_extractor` +`application/xml` | `yz_xml_extractor` +`text/plain` | `yz_text_extractor` +`text/xml` | `yz_xml_extractor` +N/A | `yz_noop_extractor` + +The answer to the second question is a function of the implementation +of the extractor module. Every extractor must conform to the +following Erlang specification: + +```erlang +-spec(ObjectValue::binary(), Options::proplist()) -> fields() | {error, term()}. +-type field_name() :: atom() | binary(). +-type field_value() :: binary(). +-type fields() :: [{field_name(), field_value()}] +``` + +The value of the object is passed along with options specific to each +extractor. Assuming the extractor correctly parses the value it will +return a list of fields, which are name-value pairs. + +The text extractor is the simplest one. By default it will use the +object's value verbatim and associate if with the field name `text`. +For example, an object with the value "How much wood could a woodchuck +chuck if a woodchuck could chuck wood?" would result in the following +fields list. + +```erlang +[{text, <<"How much wood could a woodchuck chuck if a woodchuck could chuck wood?">>}] +``` + +An object with the content type `application/json` is a little trickier. +JSON can be nested arbitrarily. That is, the key of a top-level object +can have an object as a value, and this object can have another object +nested inside, an so on. Yokozuna's JSON extractor must have some method +of converting this arbitrary nesting into a flat list. It does this by +concatenating nested object fields with a separator. The default +separator is `.`. An example should make this more clear. + +Below is JSON that represents a person, what city they are from and what +cities they have traveled to. + +```json +{ + "name": "ryan", + "info": { + "city": "Baltimore", + "visited": ["Boston", "New York", "San Francisco"] + } +} +``` + +Below is the field list that would be created by the JSON extract. + +```erlang +[{<<"info.visited">>,<<"San Francisco">>}, + {<<"info.visited">>,<<"New York">>}, + {<<"info.visited">>,<<"Boston">>}, + {<<"info.city">>,<<"Baltimore">>}, + {<<"name">>,<<"ryan">>}] +``` + +Some key points to notice. + +* Nested objects have their field names concatenated to form a field + name. The default field separator is `.`. This can be modified. +* Any array causes field names to repeat. This will require that your + schema defines this field as multi-valued. + +The XML extractor works in very similar fashion to the JSON extractor +except it also has element attributes to worry about. To see the +document created for an object, without actually writing the object, you +can use the extract HTTP endpoint. This will do a dry-run extraction and +return the document structure as `application/json`. + +```curl +curl -XPUT http://localhost:8098/search/extract \ + -H 'Content-Type: application/json' \ + --data-binary @object.json +``` + +## Schemas + +Every index must have a schema, which is a collection of field names and +types. For each document stored, every field must have a matching name +in the schema, used to determine the field's type, which in turn +determines how a field's value will be indexed. + +Currently, Yokozuna makes no attempts to hide any details of the Solr +schema: a user creates a schema for Yokozuna just as she would for Solr. +Here is the general structure of a schema. + + +```xml +<?xml version="1.0" encoding="UTF-8" ?> +<schema name="my-schema" version="1.5"> + <fields> + <!-- field definitions go here --> + </fields> + + <!-- DO NOT CHANGE THIS --> + <uniqueKey>_yz_id</uniqueKey> + + <types> + <!-- field type definitions go here --> + </types> +</schema> +``` + +The `<fields>` element is where the field name, type, and overriding +options are declared. Here is an example of a field for indexing dates. + +```xml +<field name="created" type="date" indexed="true" stored="true"/> +``` + +The corresponding date type is declared under `<types>` like so. + +```xml +<fieldType name="date" class="solr.TrieDateField" precisionStep="0" positionIncrementGap="0"/> +``` + +You can also find more information on to how customize your own [search schema]({{<baseurl>}}riak/kv/2.2.6/developing/usage/search-schemas). + +Yokozuna comes bundled with a [default schema](https://github.com/basho/yokozuna/blob/develop/priv/default_schema.xml) +called `_yz_default`. This is an extremely general schema which makes +heavy use of dynamic fields---it is intended for development and +testing. In production, a schema should be tailored to the data being +indexed. + +## Active Anti-Entropy (AAE) + +[Active Anti-Entropy]({{<baseurl>}}riak/kv/2.2.6/learn/concepts/active-anti-entropy/) \(AAE) is the process of discovering and +correcting entropy (divergence) between the data stored in Riak's +key-value backend and the indexes stored in Solr. The impetus for AAE is +that failures come in all shapes and sizes---disk failure, dropped +messages, network partitions, timeouts, overflowing queues, segment +faults, power outages, etc. Failures range from obvious to invisible. +Failure prevention is fraught with failure, as well. How do you prevent +your prevention system from failing? You don't. Code for detection, not +prevention. That is the purpose of AAE. + +Constantly reading and re-indexing every object in Riak could be quite +expensive. To minimize the overall cost of detection AAE make use of +hashtrees. Every partition has a pair of hashtrees; one for KV and +another for Yokozuna. As data is written the hashtrees are updated in +real-time. + +Each tree stores the hash of the object. Periodically a partition is +selected and the pair of hashtrees is _exchanged_. First the root hashes +are compared. If equal then there is no more work to do. You could have +millions of keys in one partition and verifying they **all** agree takes +the same time as comparing two hashes. If they don't match then the +root's children are checked and this process continues until the +individual discrepancies are found. If either side is missing a key or +the hashes for a key do not match then _repair_ is invoked on that key. +Repair converges the KV data and its indexes, removing the entropy. + +Since failure is inevitable, and absolute prevention impossible, the +hashtrees themselves may contain some entropy. For example, what if the +root hashes agree but a divergence exists in the actual data? Simple, +you assume you can never fully trust the hashtrees so periodically you +_expire_ them. When expired, a tree is completely destroyed and the +re-built from scratch. This requires folding all data for a partition, +which can be expensive and take some time. For this reason, by default, +expiration occurs after one week. + +For an in-depth look at Riak's AAE process, watch Joseph Blomstedt's +[screencast](http://coffee.jtuple.com/video/AAE.html). + + +## Analysis & Analyzers + +Analysis is the process of breaking apart (analyzing) text into a +stream of tokens. Solr allows many different methods of analysis, +an important fact because different field values may represent +different types of data. For data like unique identifiers, dates, and +categories you want to index the value verbatim---it shouldn't be +analyzed at all. For text like product summaries, or a blog post, +you want to split the value into individual words so that they may be +queried individually. You may also want to remove common words, +lowercase words, or perform stemming. This is the process of +_analysis_. + +Solr provides many different field types which analyze data in different +ways, and custom analyzer chains may be built by stringing together XML +in the schema file, allowing custom analysis for each field. For more +information on analysis, see [Search Schema]({{<baseurl>}}riak/kv/2.2.6/developing/usage/search-schemas). + +## Tagging + +Tagging is the process of adding field-value pairs to be indexed via +Riak object metadata. It is useful in two scenarios. + +1. The object being stored is opaque but your application has metadata + about it that should be indexed, for example storing an image with + location or category metadata. + +2. The object being stored is not opaque, but additional indexes must + be added _without_ modifying the object's value. + +See +[Tagging](https://github.com/basho/yokozuna/blob/develop/docs/TAGGING.md) +for more information. + +## Coverage + +Yokozuna uses _doc-based partitioning_. This means that all index +entries for a given Riak Object are co-located on the same physical +machine. To query the entire index all partitions must be contacted. +Adjacent partitions keep replicas of the same object. Replication allows +the entire index to be considered by only contacting a subset of the +partitions. The process of finding a covering set of partitions is known +as _coverage_. + +Each partition in the coverage plan has an owning node. Thus a plan can +be thought of as a unique set of nodes along with a covering set of +partitions. Yokozuna treats the node list as physical hostnames and +passes them to Solr's distributed search via the `shards` parameter. +Partitions, on the other hand, are treated logically in Yokozuna. All +partitions for a given node are stored in the same index; unlike KV +which uses _partition_ as a physical separation. To properly filter out +overlapping replicas the partition data from the cover plan is passed to +Solr via the filter query (`fq`) parameter. + +Calculating a coverage plan is handled by Riak Core. It can be a very +expensive operation as much computation is done symbolically, and the +process amounts to a knapsack problem. The larger the ring the more +expensive. Yokozuna takes advantage of the fact that it has no physical +partitions by computing a coverage plan asynchronously every few +seconds, caching the plan for query use. In the case of node failure or +ownership change this could mean a delay between cluster state and the +cached plan. This is, however, a good trade-off given the performance +benefits, especially since even without caching there is a race, albeit +one with a smaller window. + +## Statistics + +The Riak Search batching subsystem provides statistics on run-time characteristics of search system components. These statistics are accessible via the standard Riak KV stats interfaces and can be monitored through standard enterprise management tools. + + +* `search_index_throughput_(count|one)` - The total count of objects that have been indexed, per Riak node, and the count of objects that have been indexed within the metric measurement window. + +* `search_index_latency_(min|mean|max|median|95|99|999)` - The minimum, mean, maximum, median, 95th percentile, 99th percentile, and 99.9th percentile measurements of indexing latency, as measured from the time it takes to send a batch to Solr to the time the response is received from Solr, divided by the batch size. + +* `search_queue_batch_latency_(min|mean|max|median|95|99|999)` - The minimum, mean, maximum, median, 95th percentile, 99th percentile, and 99.9th percentile measurements of batch latency, as measured from the time it takes to send a batch to Solr to the time the response is received from Solr. + +* `search_queue_batch_throughput_(count|one)` - The total number of batches delivered into Solr, per Riak node, and the number of batches that have been indexed within the metric measurement window. + +* `search_queue_batchsize_(min|mean|max|median)` - The minimum, mean, maximum, and median measurements of the batch size across all indices and Solrq worker processes. + +* `search_queue_hwm_purged_(count|one)` - The total number of purged objects, and the number of purged objects within the metric measurement window. + +* `search_queue_capacity` - The capacity of the existing queues, expressed as a integral percentage value between 0 and 100. This measurement is based on the ratio of equeued objects and the configured high water mark. + +* `search_queue_drain_(count|one)` - The total number of drain operations, and the number of drain operations within the metric measurement window. + +* `search_queue_drain_fail_(count|one)` - The total number of drain failures, and the number of drain failures within the metric measurement window. + +* `search_queue_drain_timeout_(count|one)` - The total number of drain timeouts, and the number of drain timeouts within the metric measurement window. + +* `search_queue_drain_latency_(min|mean|max|median|95|99|999)` - The minimum, mean, maximum, median, 95th percentile, 99th percentile, and 99.9th percentile measurements of drain latency, as measured from the time it takes to initiate a drain to the time the drain is completed. + +* `search_detected_repairs_count` - The total number of AAE repairs that have been detected when comparing YZ and Riak/KV AAE trees. Note that this statistic is a measurement of the differences found in the AAE trees; there may be some latency between the time the trees are compared and the time that the repair is written to Solr. + +* `search_blockedvnode_(count|one)` - The total count of vnodes that have been blocked, per Riak node, and the count of blocked vnodes within the metric measurement window. Vnodes are blocked when a Solrq worker exceeds its high water mark, as defined by the [`search.queue.high_watermark`][configuring search] configuration setting. + +* `search_index_fail_(count|one)` - The total count of failed attempts to index, per Riak node, and the count of index failures within the metric measurement window. + +* `search_query_throughput_(count|one)` - The total count of queries, per Riak node, and the count of queries within the metric measurement window. + +* `search_query_latency_(min|mean|max|median|95|99|999)` - The minimum, mean, maximum, median, 95th percentile, 99th percentile, and 99.9th percentile measurements of querying latency, as measured from the time it takes to send a request to Solr to the time the response is received from Solr. + +* `search_query_fail_(count|one)` - The total count of failed queries, per Riak node, and the count of query failures within the metric measurement window. + +* `search_index_bad_entry_count` - the number of writes to Solr that have resulted in an error due to the format of the data (e.g., non-unicode data) since the last restart of Riak. + +* `search_index_bad_entry_one` - the number of writes to Solr that have resulted in an error due to the format of the data (e.g., non-unicode data) within the past minute. + +* `search_index_extract_fail_count` - the number of failures that have occurred extracting data into a format suitable for Solr (e.g., badly formatted JSON) since the last start of Riak. + +* `search_index_extract_fail_one` - the number of failures that have occurred extracting data into a format suitable for Solr (e.g., badly formatted JSON) within the past minute. + +While most of the default values are sufficient, you may have to +increase [`search.solr.start_timeout`][configuring search] as more data is indexed, which may cause Solr to require more time to start. \ No newline at end of file diff --git a/content/riak/kv/2.2.6/using/reference/secondary-indexes.md b/content/riak/kv/2.2.6/using/reference/secondary-indexes.md new file mode 100644 index 0000000000..5f5e9d399d --- /dev/null +++ b/content/riak/kv/2.2.6/using/reference/secondary-indexes.md @@ -0,0 +1,72 @@ +--- +title: "Secondary Indexes Reference" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Secondary Indexes" + identifier: "managing_ref_2i" + weight: 110 + parent: "managing_ref" +toc: true +aliases: + - /riak-docs/riak/2.2.6/dev/advanced/2i + - /riak-docs/riak/kv/2.2.6/dev/advanced/2i +--- + +[usage bucket types]: {{<baseurl>}}riak/kv/2.2.6/developing/usage/bucket-types +[use ref strong consistency]: {{<baseurl>}}riak/kv/2.2.6/using/reference/strong-consistency + +> **Note: Riak Search preferred for querying** +> +> If you're interested in non-primary-key-based querying in Riak, i.e. if +you're looking to go beyond straightforward K/V operations, we now +recommend [Riak Search]({{<baseurl>}}riak/kv/2.2.6/developing/usage/search/) rather than secondary indexes for a variety of reasons. Riak Search has a far more capacious querying API and can be used with all of Riak's storage backends. + +This document provides implementation and other details for Riak's +[secondary indexes]({{<baseurl>}}riak/kv/2.2.6/developing/usage/secondary-indexes/) \(2i) feature. + +## How It Works + +Secondary indexes use **document-based partitioning**, a system where +indexes reside with each document, local to the [vnode]({{<baseurl>}}riak/kv/2.2.6/learn/glossary/#vnode). This +system is also a local index. Secondary indexes are a list of key/value +pairs that are similar to HTTP headers. At write time, objects are +tagged with index entries consisting of key/value metadata. This +metadata can be queried to retrieve the matching keys. + +![Secondary Index]({{<baseurl>}}images/Secondary-index-example.png) + +Indexes reside on multiple machines. Since indexes for an object are +stored on the same partition as the object itself, query-time +performance issues might arise. When issuing a query, the system must +read from a "covering" set of partitions and then merge the results. +The system looks at how many replicas of data are stored---the N value +or `n_val`---and determines the minimum number of partitions that it +must examine (1 / `n_val`) to retrieve a full set of results, also +taking into account any offline nodes. + +An application can modify the indexes for an object by reading an +object, adding or removing index entries, and then writing the object. +Finally, an object is automatically removed from all indexes when it is +deleted. The object's value and its indexes should be thought of as a +single unit. There is no way to alter the indexes of an object +independently from the value of an object, and vice versa. Indexing is +atomic, and is updated in real time when writing an object. This means +that an object will be present in future index queries as soon as the +write operation completes. + +Riak stores 3 replicas of all objects by default, although this can be +changed [using bucket types][usage bucket types], which manage buckets' [replication properties]({{<baseurl>}}riak/kv/2.2.6/developing/app-guide/replication-properties). The system is capable of generating a full set of results +from one third of the system’s partitions as long as it chooses the +right set of partitions. The query is sent to each partition, the index +data is read, and a list of keys is generated and then sent back to the +requesting node. + +> **Note on 2i and strong consistency** +> +> Secondary indexes do not currently work with the [strong consistency][use ref strong consistency] feature introduced in Riak version 2.0. If you store objects in [strongly consistent buckets]({{<baseurl>}}riak/kv/2.2.6/developing/app-guide/strong-consistency/#creating-a-strongly-consistent-bucket-type) and attach +secondary index metadata to those objects, you can still perform +strongly consistent operations on those objects but the secondary +indexes will be ignored. diff --git a/content/riak/kv/2.2.6/using/reference/snmp.md b/content/riak/kv/2.2.6/using/reference/snmp.md new file mode 100644 index 0000000000..e76af599c2 --- /dev/null +++ b/content/riak/kv/2.2.6/using/reference/snmp.md @@ -0,0 +1,162 @@ +--- +title: "Simple Network Management Protocol" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "SNMP" + identifier: "managing_ref_snmp" + weight: 107 + parent: "managing_ref" +toc: true +commercial_offering: true +aliases: + - /riak-docs/riak/2.2.6/ops/running/monitoring/snmp + - /riak-docs/riak/kv/2.2.6/ops/running/monitoring/snmp +--- + +Riak Enterprise provided a built-in SNMP server that allows an external system, such as Hyperic, to query the Riak node for statistics such as the average get and put times as well as the number of puts and gets. This document only covers SNMP v2c support at this time which was the last supported version. After the release of Riak KV 2.2.3 Enterprise Edition, support for SNMP has been dropped. The below configuration examples are left for people analysing legacy settings and only work with the Enterprise Edition of Riak KV 2.2.3 or lower. + +## Configuration + +The first step in configuring your SNMP setup is to edit the appropriate files in the Riak node's `etc/snmp/agent/conf/` directory. + +First, edit the `agent.conf` file and set the appropriate IP on which the SNMP server should listen (Ex: `192.168.1.20`): + +```erlang +{intAgentIpAddress, [192,168,1,20]}. +{intAgentUDPPort, 4000}. +{snmpEngineID, "agent's engine"}. +{snmpEngineMaxMessageSize, 484}. + +%% Note: The commas in the IP are in the correct format +``` + +Next, edit the `community.conf` file if you would like to change your community from public to a different string. + +Finally, edit the `standard.conf` file and update it with the proper information: + +```erlang +{sysName, "Riak Node 1"}. +{sysDescr, "Riak Agent"}. +{sysContact, "syadmin@company.com"}. +{sysLocation, "System and Rack Location"}. +{sysObjectID, [3,6,1,4,1,193,19]}. %% {ericsson otp} - don't change +{sysServices, 72}. %% don't change +``` + +Riak needs to be restarted for configuration changes to take affect. + +**Note**: Prior to Riak Enterprise 0.13, SNMP configuration values were not reloaded during a restart. + +To force Riak to reload SNMP configuration files on startup: + + 1. Open `app.config` (most package installs place this file in `/etc/riak/`; Solaris package installs place this file in `/opt/riak/etc/`). + + 2. Locate the SNMP term: + + ```erlang + {snmp, + [{agent, + [{config, [{dir, "/etc/riak/snmp/agent/conf/"}, + {force_load, true}]}, + {db_dir, "/var/lib/riak/snmp/agent/db/"}]}]} + {snmp, + [{agent, + [{config, [{dir, "/etc/riak/snmp/agent/conf/"}]}, + {db_dir, "/var/lib/riak/snmp/agent/db/"}]}]} + ``` + + 3. Add `{force_load, true}` to the `config` term: + + ```erlang + {snmp, + [{agent, + [{config, [{dir, "/etc/riak/snmp/agent/conf/"}, + {force_load, true}]}, + {db_dir, "/var/lib/riak/snmp/agent/db/"}]}]} + ``` + + 4. Save `app.config` + + 5. Restart Riak + +Once you have configured the SNMP settings you can start your Riak node and will be able to snmpwalk the node to verify that the setup is working: + +```bash +$ snmpwalk -OS -c public -v2c -m all 192.168.52.129:4000 . +``` + +If you would like to query the OIDs associated with Riak you will need to reference the MIB shipped with Riak. For example, the x86_64 packages have the MIB in the following folder: + +```bash +/usr/lib64/riak/lib/riak_snmp-0.2/priv/mibs +``` + +This folder can be referenced in the snmpwalk command as follows: + +```bash +$ snmpwalk -OS -c public -v 2c -m ALL \ + -M +/usr/lib64/riak/lib/riak_snmp-0.2/priv/mibs \ + 192.168.52.129:4000 RIAK +``` + + +## SNMP Counters + +**vnodeGets** +*Type:* Counter +Number of vnode-level GETs in past minute + +**vnodePuts** +*Type:* Counter +Number of vnode-level PUTs in past minute + +**nodeGets** +*Type:* Counter +Number of GETs in past minute + +**nodePuts** +*Type:* Counter +Number of PUTs in past minute + +**nodeGetTimeMean** +*Type:* Gauge +Mean GET time (microseconds) + +**nodeGetTimeMedian** +*Type:* Gauge +Median GET time (microseconds) + +**nodeGetTime95** +*Type:* Gauge +95th percentile GET time (microseconds) + +**nodeGetTime99** +*Type:* Gauge +99th percentile GET time (microseconds) + +**nodeGetTime100** +*Type:* Gauge +Maximum GET time (microseconds) + +**nodePutTime95** +*Type:* Gauge +95th percentile PUT time (microseconds) + +**nodePutTime99** +*Type:* Gauge +99th percentile PUT time (microseconds) + +**nodePutTime100** +*Type:* Gauge +Maximum PUT time (microseconds) + +**nodePutTimeMean** +*Type:* Gauge +Mean PUT time (microseconds) + +**nodePutTimeMedian** +*Type:* Gauge +Median PUT time (microseconds) diff --git a/content/riak/kv/2.2.6/using/reference/statistics-monitoring.md b/content/riak/kv/2.2.6/using/reference/statistics-monitoring.md new file mode 100644 index 0000000000..7fe8641fb6 --- /dev/null +++ b/content/riak/kv/2.2.6/using/reference/statistics-monitoring.md @@ -0,0 +1,391 @@ +--- +title: "Statistics & Monitoring Reference" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Monitoring" + identifier: "managing_ref_monitoring" + weight: 106 + parent: "managing_ref" +toc: true +aliases: + - /riak-docs/riak/2.2.6/ops/running/stats-and-monitoring + - /riak-docs/riak/kv/2.2.6/ops/running/stats-and-monitoring +--- + +Riak provides data related to current operating status, which includes +statistics in the form of counters and histograms. These statistics +are made available through the HTTP API via the [`/stats`]({{<baseurl>}}riak/kv/2.2.6/developing/api/http/status) endpoint, or through the [`riak-admin`]({{<baseurl>}}riak/kv/2.2.6/using/admin/riak-admin/) interface, in particular the `stat` and `status` commands. + +This page presents the most commonly monitored and gathered +statistics, as well as numerous solutions for monitoring and gathering +statistics that our customers and community report using successfully +in Riak cluster environments. You can learn more about the specific +Riak statistics provided in the [Inspecting a Node]({{<baseurl>}}riak/kv/2.2.6/using/cluster-operations/inspecting-node) and [HTTP Status]({{<baseurl>}}riak/kv/2.2.6/developing/api/http/status) documentation. + +## System Metrics To Graph + +Graphing general system metrics of Riak nodes will help with +diagnostics and early warnings of potential problems, as well as help +guide provisioning and scaling decisions. + +* CPU (user/system/wait/idle) +* Processor Load +* Available Memory +* Available disk space +* Used file descriptors +* Swap Usage +* IOWait +* Read operations +* Write operations +* Network throughput +* Network errors + +We also recommend tracking your system's virtual and +writebacks. Things like massive flushes of dirty pages or steadily +climbing writeback volumes can indicate poor virtual memory tuning. +More information can be found [here][sysctl_vm_txt] and in our +documentation on [system tuning]({{<baseurl>}}riak/kv/2.2.6/using/performance/#storage-and-file-system-tuning). + +## Riak Metrics to Graph +Riak metrics fall into several general categories: + +1. Throughput metrics +2. Latency metrics +3. Erlang resource usage metrics +4. General Riak load/health metrics + +If graphing all of the [available Riak metrics]({{<baseurl>}}riak/kv/2.2.6/using/cluster-operations/inspecting-node) is +not practical, you should pick a minimum relevant subset from these +categories. Some of the most helpful metrics are discussed below. + +### Throughput Metrics + +Graphing the throughput stats relevant to your use case is often +helpful for capacity planning and usage trend analysis. In addition, +it helps you establish an expected baseline -- that way, you can +investigate unexpected spikes or dips in the throughput. The +following stats are recorded for operations that happened *during the +last minute*. + +Metric | Relevance | Operations (for the last minute) +:--------|:--------|:-------------------------------- +```node_gets``` | K/V | Reads coordinated by this node +```node_puts``` | K/V | Writes coordinated by this node +```vnode_counter_update``` | Data Types | Update [Counters][data_types_counters] operations coordinated by local vnodes +```vnode_set_update``` | Data Types | Update [Sets][data_types_sets] operations coordinated by local vnodes +```vnode_map_update``` | Data Types | Update [Maps][data_types_maps] operations coordinated by local vnodes +```search_query_throughput_one``` | Search | Search queries on the node +```search_index_throughtput_one``` | Search | Documents indexed by Search +```consistent_gets``` | Strong Consistency | Consistent reads on this node +```consistent_puts``` | Strong Consistency | Consistent writes on this node +```vnode_index_reads``` | Secondary Indexes | Number of local replicas participating in secondary index reads + +Note that there are no separate stats for updates to Flags or +Registers, as these are included in ```vnode_map_update```. + +### Latency Metrics + +As with the throughput metrics, keeping an eye on average (and max) +latency times will help detect usage patterns, and provide advanced +warnings for potential problems. + +{{% note title="Note on FSM Time Stats" %}} +FSM Time Stats represent the amount of time in microseconds required to +traverse the GET or PUT Finite State Machine code, offering a picture of +general node health. From your application's perspective, FSM Time effectively +represents experienced latency. Mean, Median, and 95th-, 99th-, and +100th-percentile (Max) counters are displayed. These are one-minute stats. +{{% /note %}} + +Metric | Also | Relevance | Latency (in microseconds) +:------|:-----|:----------|:------------------------- +```node_get_fsm_time_mean``` | ```_median```, ```_95```, ```_99```, ```_100``` | K/V | Time between reception of client read request and subsequent response to client +```node_put_fsm_time_mean``` | ```_median```, ```_95```, ```_99```, ```_100``` | K/V | Time between reception of client write request and subsequent response to client +```object_counter_merge_time_mean``` | ```_median```, ```_95```, ```_99```, ```_100``` | Data Types | Time it takes to perform an Update Counter operation +```object_set_merge_time_mean``` | ```_median```, ```_95```, ```_99```, ```_100``` | Data Types | Time it takes to perform an Update Set operation +```object_map_merge_time_mean``` | ```_median```, ```_95```, ```_99```, ```_100``` | Data Types | Time it takes to perform an Update Map operation +```search_query_latency_median``` | ```_min```, ```_95```, ```_99```, ```_999```, ```_max``` | Search | Search query latency +```search_index_latency_median``` | ```_min```, ```_95```, ```_99```, ```_999```, ```_max``` | Search | Time it takes Search to index a new document +```consistent_get_time_mean``` | ```_median```, ```_95```, ```_99```, ```_100``` | Strong Consistency | Strongly consistent read latency +```consistent_put_time_mean``` | ```_median```, ```_95```, ```_99```, ```_100``` | Strong Consistency | Strongly consistent write latency + +### Erlang Resource Usage Metrics + +These are system metrics from the perspective of the Erlang VM, +measuring resources allocated and used by Erlang. + +Metric | Notes +:------|:------------------------- +```sys_process_count``` | Number of processes currently running in the Erlang VM +```memory_processes``` | Total amount of memory allocated for Erlang processes (in bytes) +```memory_processes_used``` | Total amount of memory used by Erlang processes (in bytes) + +### General Riak Load/Health Metrics + +These various stats give a picture of the general level of activity or +load on the Riak node at any given moment. + +Metric | Also | Notes +:------|:-----|:------------------ +```node_get_fsm_siblings_mean``` | ```_median```, ```_95```, ```_99```, ```_100``` | Number of siblings encountered during all GET operations by this node within the last minute. Watch for abnormally high sibling counts, especially max ones. +```node_get_fsm_objsize_mean``` | ```_median```, ```_95```, ```_99```, ```_100``` | Object size encountered by this node within the last minute. Abnormally large objects (especially paired with high sibling counts) can indicate sibling explosion. +```riak_search_vnodeq_mean``` | ```_median```, ```_95```, ```_99```, ```_100``` | Number of unprocessed messages in the vnode message queues of the Riak Search subsystem on this node in the last minute. The queues give you an idea of how backed up Solr is getting. +```search_index_fail_one``` | | Number of "Failed to index document" errors Search encountered for the last minute +```pbc_active``` | | Number of currently active protocol buffer connections +```pbc_connects``` | | Number of new protocol buffer connections established during the last minute +```read_repairs``` | | Number of read repair operations this node has coordinated in the last minute (determine baseline, watch for abnormal spikes) +```list_fsm_active``` | | Number of List Keys FSMs currently active (should be 0) +```node_get_fsm_rejected``` | | Number of GET FSMs actively being rejected by Sidejob's overload protection +```node_put_fsm_rejected``` | | Number of PUT FSMs actively being rejected by Sidejob's overload protection + +### General Riak Search Load/Health Metrics + +These various stats give a picture of the general level of activity or +load on the Riak node at any given moment. + +Metric | Description +:------|:------------ +`search_index_bad_entry_count` | Number of writes to Solr that have resulted in an error due to the format of the data (e.g., non-unicode data) since the last restart of Riak. +`search_index_bad_entry_one ` | Number of writes to Solr that have resulted in an error due to the format of the data (e.g., non-unicode data) in the past minute. +`search_index_extract_fail_count` | Number of failures that have occurred extracting data into a format suitable to insert into Solr (e.g., badly formatted JSON) since the last start of Riak. +`search_index_extract_fail_one` | Number of failures that have occurred extracting data into a format suitable to insert into Solr (e.g., badly formatted JSON) in the past minute. + + +## Command-line Interface + +The [`riak-admin`]({{<baseurl>}}riak/kv/2.2.6/using/admin/riak-admin/) tool provides two +interfaces for retrieving statistics and other information: `status` +and `stat`. + +### status + +Running the `riak-admin status` command will return all of the +currently available information from a running node. + +```bash +riak-admin status +``` + +This will return a list of over 300 key/value pairs, like this: + +``` +1-minute stats for 'dev1@127.0.0.1' +------------------------------------------- +connected_nodes : ['dev2@127.0.0.1','dev3@127.0.0.1'] +consistent_get_objsize_100 : 0 +consistent_get_objsize_195 : 0 +... etc ... +``` + +A comprehensive list of available stats can be found in the +[Inspecting a Node]({{<baseurl>}}riak/kv/2.2.6/using/cluster-operations/inspecting-node/#riak-admin-status) document. + +### stat + +The `riak-admin stat` command is related to the `riak-admin status` +command but provides a more fine-grained interface for interacting with +stats and information. Full documentation of this command can be found +in the [Inspecting a Node]({{<baseurl>}}riak/kv/2.2.6/using/cluster-operations/inspecting-node/#riak-admin-stat) document. + +## Statistics and Monitoring Tools + +There are many open source, self-hosted, and service-based solutions for +aggregating and analyzing statistics and log data for the purposes of +monitoring, alerting, and trend analysis on a Riak cluster. Some +solutions provide Riak-specific modules or plugins as noted. + +The following are solutions which customers and community members have +reported success with when used for monitoring the operational status of +their Riak clusters. Community and open source projects are presented +along with commercial and hosted services. + +{{% note title="Note on Riak 2.x Statistics Support" %}} +Many of the below tools were either created by third-parties or Basho +engineers for general usage, and have been passed to the community for further +updates. As such, many of the below only aggregate the statistics and messages +that were output by Riak 1.4.x. + +Like all code under [Basho Labs](https://github.com/basho-labs/), the below +tools are "best effort" and have no dedicated Basho support. We both +appreciate and need your contribution to keep these tools stable and up to +date. Please open up a GitHub issue on the repository if you'd like to be a +maintainer. + +Look for banners calling out the tools we've verified that support the latest +Riak 2.x statistics! +{{% /note %}} + +### Self-Hosted Monitoring Tools + +#### Riaknostic + +[Riaknostic](http://riaknostic.basho.com) is a growing suite of +diagnostic checks that can be run against your Riak node to discover +common problems and recommend how to resolve them. These checks are +derived from the experience of the Basho Client Services Team as well as +numerous public discussions on the mailing list, IRC room, and other +online media. + +Riaknostic integrates into the `riak-admin` command via a `diag` +subcommand, and is a great first step in the process of diagnosing and +troubleshooting issues on Riak nodes. + +#### Riak Control + +[Riak Control]({{<baseurl>}}riak/kv/2.2.6/using/admin/riak-control/) is Basho's REST-driven user-interface for managing Riak +clusters. It is designed to give you quick insight into the health of +your cluster and allow for easy management of nodes. + +While Riak Control does not currently offer specific monitoring and +statistics aggregation or analysis functionality, it does offer features +which provide immediate insight into overall cluster health, node +status, and handoff operations. + +#### collectd + +[collectd](http://collectd.org) gathers statistics about the system it +is running on and stores them. The statistics are then typically graphed +to find current performance bottlenecks, predict system load, and +analyze trends. + +#### Ganglia + +[Ganglia](http://ganglia.info) is a monitoring system specifically +designed for large, high-performance groups of computers, such as +clusters and grids. Customers and community members using Riak have +reported success in using Ganglia to monitor Riak clusters. + +A [Riak Ganglia module][riak_ganglia] for collecting statistics from +the Riak HTTP [`/stats`]({{<baseurl>}}riak/kv/2.2.6/developing/api/http/status) endpoint is also available. + +#### Nagios + +{{% note %}} +**Tested and Verified Support for Riak 2.x.** +{{% /note %}} + +[Nagios](http://www.nagios.org) is a monitoring and alerting solution +that can provide information on the status of Riak cluster nodes, in +addition to various types of alerting when particular events occur. +Nagios also offers logging and reporting of events and can be used for +identifying trends and capacity planning. + +A collection of [reusable Riak-specific scripts][riak_nagios] are +available to the community for use with Nagios. + +#### OpenTSDB + +[OpenTSDB](http://opentsdb.net) is a distributed, scalable Time Series Database +(TSDB) used to store, index, and serve metrics from various sources. It can +collect data at a large scale and graph these metrics on the fly. + +A [Riak collector for OpenTSDB][tcollector_riak_plugin] is available as part of +the [tcollector framework][tcollector]. + +#### Riemann + +[Riemann](http://github.com/riemann/riemann/) uses a powerful stream +processing language to aggregate events from client agents running on +Riak nodes, and can help track trends or report on events as they occur. +Statistics can be gathered from your nodes and forwarded to a solution +such as Graphite for producing related graphs. + +A [Riemann Tools](https://github.com/aphyr/riemann.git) project +consisting of small programs for sending data to Riemann provides a +module specifically designed to read Riak statistics. + +#### Zabbix + +{{% note %}} +**Tested and Verified Support for Riak 2.x Stats.** +{{% /note %}} + +[Zabbix](http://www.zabbix.com) is an open-source performance monitoring, +alerting, and graphing solution that can provide information on the state of +Riak cluster nodes. + +A [Zabbix plugin for Riak][riak_zabbix] is available to get you started +monitoring Riak using Zabbix. + + +### Hosted Service Monitoring Tools + +The following are some commercial tools which Basho customers have +reported successfully using for statistics gathering and monitoring +within their Riak clusters. + +#### Circonus +[Circonus](http://circonus.com) provides organization-wide monitoring, +trend analysis, alerting, notifications, and dashboards. It can been +used to provide trend analysis and help with troubleshooting and +capacity planning in a Riak cluster environment. + +#### New Relic + +{{% note %}} +**Tested and Verified Support for Riak 2.x Stats.** +{{% /note %}} + +[New Relic](http://newrelic.com) is a data analytics and visualization platform +that can provide information on the current and past states of Riak nodes and +visualizations of machine generated data such as log files. + +A [Riak New Relic Agent][riak_new_relic] for collecting statistics from the Riak HTTP [`/stats`]({{<baseurl>}}riak/kv/2.2.6/developing/api/http/status) endpoint is also available. + +#### Splunk + +[Splunk](http://www.splunk.com) is available as downloadable software or +as a service, and provides tools for visualization of machine generated +data such as log files. It can be connected to Riak's HTTP statistics +[`/stats`]({{<baseurl>}}riak/kv/2.2.6/developing/api/http/status) endpoint. + +Splunk can be used to aggregate all Riak cluster node operational log +files, including operating system and Riak-specific logs and Riak +statistics data. These data are then available for real time graphing, +search, and other visualization ideal for troubleshooting complex issues +and spotting trends. + +## Summary + +Riak exposes numerous forms of vital statistic information which can be +aggregated, monitored, analyzed, graphed, and reported on in a variety +of ways using numerous open source and commercial solutions. + +If you use a solution not listed here with Riak and would like to +include it (or would otherwise like to update the information on this +page), feel free to fork the docs, add it in the appropriate section, +and send a pull request to the [Riak +Docs](https://github.com/basho/basho_docs). + +## References + +* [Inspecting a Node]({{<baseurl>}}riak/kv/2.2.6/using/cluster-operations/inspecting-node) +* [Riaknostic](http://riaknostic.basho.com) +* [Riak Control]({{<baseurl>}}riak/kv/2.2.6/using/admin/riak-control/) +* [collectd](http://collectd.org) +* [Ganglia](http://ganglia.info) +* [Nagios](http://www.nagios.org) +* [OpenTSDB](http://opentsdb.net) +* [tcollector framework][tcollector] +* [Riemann](http://github.com/riemann/riemann/) +* [Riemann Github](https://github.com/aphyr/riemann) +* [Zabbix](http://www.zabbix.com) +* [Circonus](http://circonus.com) +* [New Relic](http://newrelic.com) +* [Splunk](http://www.splunk.com) +* [Riak Docs on Github](https://github.com/basho/basho_docs) + + +[sysctl_vm_txt]: https://www.kernel.org/doc/Documentation/sysctl/vm.txt +[data_types_counters]: http://docs.basho.com/riak/latest/dev/using/data-types/#Counters +[data_types_sets]: http://docs.basho.com/riak/latest/dev/using/data-types/#Sets +[data_types_maps]: http://docs.basho.com/riak/latest/dev/using/data-types/#Maps +[riak_nagios]: https://github.com/basho/riak_nagios +[tcollector]: https://github.com/stumbleupon/tcollector +[tcollector_riak_plugin]: https://github.com/stumbleupon/tcollector/blob/master/collectors/0/riak.py +[riak_zabbix]: https://github.com/basho/riak-zabbix +[riak_new_relic]: https://github.com/basho/riak_newrelic +[riak_ganglia]: https://github.com/jnewland/gmond_python_modules/tree/master/riak/ diff --git a/content/riak/kv/2.2.6/using/reference/strong-consistency.md b/content/riak/kv/2.2.6/using/reference/strong-consistency.md new file mode 100644 index 0000000000..9a82287098 --- /dev/null +++ b/content/riak/kv/2.2.6/using/reference/strong-consistency.md @@ -0,0 +1,145 @@ +--- +title: "Strong Consistency Reference" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Strong Consistency" + identifier: "managing_ref_strong_consistency" + weight: 112 + parent: "managing_ref" +toc: true +--- + +[usage bucket types]: {{<baseurl>}}riak/kv/2.2.6/developing/usage/bucket-types +[concept eventual consistency]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/eventual-consistency + +Riak was originally designed as an [eventually consistent]({{<baseurl>}}riak/kv/2.2.6/learn/concepts/eventual-consistency) system, fundamentally geared toward providing partition +(i.e. fault) tolerance and high read and write availability. + +While this focus on high availability is a great fit for many data +storage needs, there are also many use cases for which strong data +consistency is more important than availability. Basho introduced a new +strong consistency option in version 2.0 to address these use cases. +In Riak, strong consistency is applied [using bucket types][usage bucket types], which +enables developers to apply strong consistency guarantees on a per-key +basis. + +Elsewhere in the documentation there are instructions for [enabling and using]({{<baseurl>}}riak/kv/2.2.6/developing/app-guide/strong-consistency/) strong consistency, as well as a [guide for operators]({{<baseurl>}}riak/kv/2.2.6/configuring/strong-consistency) looking to manage, +configure, and monitor strong consistency. + +## Strong vs. Eventual Consistency + +If you successfully write a value to a key in a strongly consistent +system, the next successful read of that key is guaranteed to show that +write. A client will never see out-of-date values. The drawback is that +some operations may fail if an insufficient number of object replicas +are available. More on this in the section on [trade-offs](#trade-offs). + +In an eventually consistent system, on the other hand, a read may return +an out-of-date value, particularly during system or network failures. +The advantage of this approach is that reads and writes can succeed even +when a cluster is experiencing significant service degradation. + +### Example + +Building on the example presented in the [eventual consistency][concept eventual consistency] doc, +imagine that information about who manages Manchester United is stored +in Riak, in the key `manchester-manager`. In the eventual consistency +example, the value associated with this key was originally +`David Moyes`, meaning that that was the first successful write to that +key. But then `Louis van Gaal` became Man U's manager, and a write was +executed to change the value of `manchester-manager`. + +Now imagine that this write failed on one node in a multi-node cluster. +Thus, all nodes report that the value of `manchester-manager` is `Louis +van Gaal` except for one. On the errant node, the value of the +`manchester-manager` key is still `David Moyes`. An eventually +consistent system is one in which a get request will most likely return +`Louis van Gaal` but could return the outdated value `David Moyes`. + +In a strongly consistent system, conversely, any successful read on +`manchester-manager` will return `Louis van Gaal` and never `David Moyes`. +Reads will return `Louis van Gaal` every single time until Man U gets a new +manager and someone performs a successful write to `manchester-manager` +to change its value. + +It might also be useful to imagine it a bit more abstractly. The +following causal sequence would characterize a strongly consistent +system: + +1. The value of the key `k` is set to `v` +2. All successful reads on `k` return `v` +3. The value of `k` is changed to `v2` +4. All successful reads on `k` return `v2` +5. And so forth + +At no point in time does this system return an out-of-date value. + +The following sequence could characterize an eventually consistent +system: + +1. A write is made that sets the value of the key `k` to `v` +2. Nearly all reads to `k` return `v`, but a small percentage return + `not found` +3. A write to `k` changes the value to `v2` +4. Nearly all reads to `k` now return `v2`, but a small number return + the outdated `v` (or even `not found`) because the newer value hasn't + yet been replicated to all nodes + +## Making the Strong vs. Eventual Decision + +The first system described above may sound like the undisputed champion, +and the second system undesirable. However: + +1. Reads and writes on the first system will often be slower---if only + by a few milliseconds---because the system needs to manage reads and + writes more carefully. If performance is of primary concern, the + first system might not be worth the sacrifice. +2. Reads and writes on the first system may fail entirely if enough + servers are unavailable. If high availability is the top priority, + then the second system has a significant advantage. + +So when deciding whether to use strong consistency in Riak, the +following question needs to be asked: + +#### For the specific use case at hand, is it better for reads to fail than to return a potentially out-of-date value? + +If the answer is yes, then you should seriously consider using Riak in a +strongly consistent way for the data that demands it, while bearing in +mind that other data can still be stored in Riak in an eventually +consistent way. + +## Trade-offs + +Using Riak in a strongly consistent fashion comes with two unavoidable +trade-offs: + +1. Less availability +2. Slightly slower performance + +Strongly consistent operations are necessarily less highly available +than eventually consistent operations because they require a **quorum** +of available object replicas to succeed. Quorum is defined as N / 2 + 1, +or `n_val` / 2 + 1. If N is set to 7, at least 4 object replicas must be +available, 2 must be available if N=3, etc. + +If there is a network partition that leaves less than a quorum of object +replicas available within an ensemble, strongly consistent operations +against the keys managed by that ensemble will fail. + +Nonetheless, consistent operations do provide a great deal of fault +tolerance. Consistent operations can still succeed when a minority of +replicas in each ensemble can be offline, faulty, or unreachable. In +other words, **strongly consistent operations will succeed as long as +quorum is maintained**. A fuller discussion can be found in the +[operations]({{<baseurl>}}riak/kv/2.2.6/configuring/strong-consistency/#fault-tolerance) +documentation. + +A second trade-off regards performance. Riak's implementation of strong +consistency involves a complex [consensus subsystem]({{<baseurl>}}riak/kv/2.2.6/using/reference/strong-consistency/#implementation-details) that typically requires more communication between Riak nodes than eventually consistent operations, +which can entail a performance hit of varying proportions, depending on +a variety of factors. + +Ways to address this issue can be found in [strong consistency and performance]({{<baseurl>}}riak/kv/2.2.6/configuring/strong-consistency/#performance). diff --git a/content/riak/kv/2.2.6/using/reference/v2-multi-datacenter.md b/content/riak/kv/2.2.6/using/reference/v2-multi-datacenter.md new file mode 100644 index 0000000000..4e3407141a --- /dev/null +++ b/content/riak/kv/2.2.6/using/reference/v2-multi-datacenter.md @@ -0,0 +1,35 @@ +--- +title: "V2 Multi-Datacenter Replication Reference" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "V2 Replication" + identifier: "managing_ref_v2" + weight: 115 + parent: "managing_ref" +toc: true +--- + +[v2 mdc arch]: ./architecture +[v2 mdc fullsync]: ./scheduling-fullsync + +{{% note title="Deprecation Warning" %}} +v2 Multi-Datacenter Replication is deprecated and will be removed in a future version. Please use [v3]({{<baseurl>}}riak/kv/2.2.6/using/reference/v3-multi-datacenter/) instead. +{{% /note %}} + + +## In This Section + +#### [V2 Multi-Datacenter Replication Reference: Architecture][v2 mdc arch] + +Overview of the architecture undergirding Riak's Multi-Datacenter Replication capabilities. + +[Learn More >>][v2 mdc arch] + +#### [V2 Multi-Datacenter Replication Reference: Scheduling Fullsync][v2 mdc fullsync] + +Brief tutorial on scheduling fullsync operations. + +[Learn More >>][v2 mdc fullsync] diff --git a/content/riak/kv/2.2.6/using/reference/v2-multi-datacenter/architecture.md b/content/riak/kv/2.2.6/using/reference/v2-multi-datacenter/architecture.md new file mode 100644 index 0000000000..080db5112e --- /dev/null +++ b/content/riak/kv/2.2.6/using/reference/v2-multi-datacenter/architecture.md @@ -0,0 +1,126 @@ +--- +title: "V2 Multi-Datacenter Replication Reference: Architecture" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Architecture" + identifier: "managing_ref_v2_architecture" + weight: 100 + parent: "managing_ref_v2" +toc: true +aliases: + - /riak-docs/riak/2.2.6/ops/mdc/v2/architecture + - /riak-docs/riak/kv/2.2.6/ops/mdc/v2/architecture +--- + +{{% note title="Deprecation Warning" %}} +v2 Multi-Datacenter Replication is deprecated and will be removed in a future version. Please use [v3]({{<baseurl>}}riak/kv/2.2.6/using/reference/v3-multi-datacenter/architecture/) instead. +{{% /note %}} + + +This document provides a basic overview of the architecture undergirding +Riak's Multi-Datacenter Replication capabilities. + +## How Replication Works + +When Multi-Datacenter Replication is implemented, one Riak cluster acts +as a **primary cluster**. The primary cluster handles replication +requests from one or more **secondary clusters** (generally located in +datacenters in other regions or countries). If the datacenter with the +primary cluster goes down, a secondary cluster can take over as the +primary cluster. In this sense, Riak's multi-datacenter capabilities are +masterless. + +Multi-Datacenter Replication has two primary modes of operation: +**fullsync** and **realtime**. In fullsync mode, a complete +synchronization occurs between primary and secondary cluster(s); in +realtime mode, continual, incremental synchronization occurs, i.e. +replication is triggered by new updates. + +Fullsync is performed upon initial connection of a secondary cluster, +and then periodically thereafter (every 360 minutes is the default, but +this can be modified). Fullsync is also triggered if the TCP connection +between primary and secondary cluster is severed and then recovered. + +Both fullsync and realtime mode are described in detail below. +But first, a few key concepts. + +## Concepts + +### Listener Nodes + +Listeners, also called **servers**, are Riak nodes in the primary +cluster that listen on an external IP address for replication requests. +Any node in a Riak cluster can participate as a listener. Adding more +nodes will increase the fault tolerance of the replication process in +the event of individual node failures. If a listener node goes down, +another node can take its place. + +### Site Nodes + +Site nodes, also called **clients**, are Riak nodes on a secondary +cluster that connect to listener nodes and send replication initiation +requests. Site nodes are paired with a listener node when started. + +### Leadership + +Only one node in each cluster will serve as the lead site (client) or +listener (server) node. Riak replication uses a leadership-election +protocol to determine which node in the cluster will participate in +replication. If a site connects to a node in the primary cluster that is +not the leader, it will be redirected to the listener node that is +currently the leader. + +## Fullsync Replication + +Riak performs the following steps during fullsync +replication, as illustrated in the Figure below. + +1. A TCP connection is established between the primary and secondary + clusters +2. The site node in the secondary cluster initiates fullsync replication + with the primary node by sending a message to the listener node in + the primary cluster +3. The site and listener nodes iterate through each [vnode]({{<baseurl>}}riak/kv/2.2.6/learn/glossary/#vnode) in their respective clusters and compute a hash for + each key's object value. The site node on the secondary cluster sends + its complete list of key/hash pairs to the listener node in the + primary cluster. The listener node then sequentially compares its + key/hash pairs with the primary cluster's pairs, identifying any + missing objects or updates needed in the secondary cluster. +4. The listener node streams the missing objects/updates to the + secondary cluster. +5. The secondary cluster replicates the updates within the cluster to + achieve the new object values, completing the fullsync cycle + +<br> +![MDC Fullsync]({{<baseurl>}}images/MDC_Full-sync-small.png) +<br> + +## Realtime Replication + +Riak performs the following steps during realtime +replication, as illustrated in the Figure below. + +1. The secondary cluster establishes a TCP connection to the primary +2. Realtime replication of a key/object is initiated when an update is + sent from a client to the primary cluster +3. The primary cluster replicates the object locally +4. The listener node on the primary cluster streams an update to the + secondary cluster +5. The site node within the secondary cluster receives and replicates + the update + +<br> +![MDC Realtime]({{<baseurl>}}images/MDC-real-time-sync-small.png) +<br> + +## Restrictions + +It is important to note that both clusters must have certain attributes +in common for Multi-Datacenter Replication to work. If you are using +either fullsync or realtime replication, both clusters must have the +same [ring size]({{<baseurl>}}riak/kv/2.2.6/learn/concepts/clusters/#the-ring); if you are using fullsync +replication, every bucket's [`n_val`]({{<baseurl>}}riak/kv/2.2.6/developing/app-guide/replication-properties#n-value-and-replication) must be the same in both the +source and sink cluster. diff --git a/content/riak/kv/2.2.6/using/reference/v2-multi-datacenter/scheduling-fullsync.md b/content/riak/kv/2.2.6/using/reference/v2-multi-datacenter/scheduling-fullsync.md new file mode 100644 index 0000000000..1789b050fd --- /dev/null +++ b/content/riak/kv/2.2.6/using/reference/v2-multi-datacenter/scheduling-fullsync.md @@ -0,0 +1,49 @@ +--- +title: "V2 Multi-Datacenter Replication Reference: Scheduling Fullsync" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Scheduling Fullsync" + identifier: "managing_ref_v2_fullsync" + weight: 101 + parent: "managing_ref_v2" +toc: true +aliases: + - /riak-docs/riak/2.2.6/ops/mdc/v2/scheduling-fullsync + - /riak-docs/riak/kv/2.2.6/ops/mdc/v2/scheduling-fullsync +--- + +{{% note title="Deprecation Warning" %}} +v2 Multi-Datacenter Replication is deprecated and will be removed in a future version. Please use [v3]({{<baseurl>}}riak/kv/2.2.6/using/reference/v3-multi-datacenter/scheduling-fullsync/) instead. +{{% /note %}} + + +## Scheduling Fullsync Operation + +With the `pause` and `resume` commands it is possible to limit the +fullsync operation to off-peak times. First, disable `fullsync_interval` +and set `fullsync_on_connect` to `false`. Then, using cron or something +similar, execute the commands below at the start of the sync window. +In these examples, the commands are combined in a `.sh` or analogous +file: + +```bash +#!/bin/sh + +## Resume from where we left off +riak-repl resume-fullsync + +## Start fullsync if nothing is running +riak-repl start-fullsync +``` + +At the end of the sync window: + +```bash +#!/bin/sh + +## Stop fullsync until start of next sync window +riak-repl pause-fullsync +``` diff --git a/content/riak/kv/2.2.6/using/reference/v3-multi-datacenter.md b/content/riak/kv/2.2.6/using/reference/v3-multi-datacenter.md new file mode 100644 index 0000000000..78ba75308a --- /dev/null +++ b/content/riak/kv/2.2.6/using/reference/v3-multi-datacenter.md @@ -0,0 +1,47 @@ +--- +title: "V3 Multi-Datacenter Replication Reference" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "V3 Replication" + identifier: "managing_ref_v3" + weight: 114 + parent: "managing_ref" +toc: true +--- + +[v3 mdc arch]: ./architecture +[v3 mdc aae]: ./aae +[v3 mdc cascade]: ./cascading-writes +[v3 mdc fullsync]: ./scheduling-fullsync + +## In This Section + +#### [V3 Multi-Datacenter Replication Reference: Architecture][v3 mdc arch] + +Overview of the architecture undergirding Riak's Version 3 Multi-Datacenter Replication capabilities. + +[Learn More >>][v3 mdc arch] + + +#### [V3 Multi-Datacenter Replication Reference: With Active Anti-Entropy][v3 mdc aae] + +Overview of using Riak KV's active anti-entropy (AAE) subsystem with Multi-Datacenter. + +[Learn More >>][v3 mdc aae] + + +#### [V3 Multi-Datacenter Replication Reference: Cascading Realtime Writes][v3 mdc cascade] + +Details the cascading realtime writes feature. + +[Learn More >>][v3 mdc cascade] + + +#### [V3 Multi-Datacenter Replication Reference: Scheduling Fullsync][v3 mdc fullsync] + +Brief tutorial on scheduling fullsync operations. + +[Learn More >>][v3 mdc fullsync] diff --git a/content/riak/kv/2.2.6/using/reference/v3-multi-datacenter/aae.md b/content/riak/kv/2.2.6/using/reference/v3-multi-datacenter/aae.md new file mode 100644 index 0000000000..742e3f3e8e --- /dev/null +++ b/content/riak/kv/2.2.6/using/reference/v3-multi-datacenter/aae.md @@ -0,0 +1,125 @@ +--- +title_supertext: "V3 Multi-Datacenter Replication Reference:" +title: "Fullsync via Active Anti-Entropy" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Active Anti-Entropy" + identifier: "managing_ref_v3_aae" + weight: 101 + parent: "managing_ref_v3" +toc: true +commercial_offering: true +aliases: + - /riak-docs/riak/2.2.6/ops/mdc/v3/aae + - /riak-docs/riak/kv/2.2.6/ops/mdc/v3/aae +--- + +[glossary aae]: {{<baseurl>}}riak/kv/2.2.6/learn/glossary/#active-anti-entropy-aae +[config reference#advanced]: {{<baseurl>}}riak/kv/2.2.6/configuring/reference/#advanced-configuration +[concept clusters]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/clusters + +> **Note: Technical preview** +> +> The active anti-entropy fullsync strategy, as it pertains to +replication, is currently in **technical preview** mode. This means that +it hasn't been tested at large scale and that there may be issues that +Basho must address prior to a general release. Please don't use this +feature on a production system without professional services or customer +service engineering support. + +## Overview + +Riak Multi-Datacenter (MDC) Replication version 3 (originally limited to Riak +Enterprise versions 1.4.0 - 2.2.3 and now available to all versions post 2.2.3) can now take advantage of Riak's [active anti-entropy][glossary aae] \(AAE) subsystem, which was first introduced as a +technology preview in Riak 1.3.0. + +AAE plus Replication uses existing Riak AAE hash trees stored in +LevelDB, so if AAE is already active, there is no additional startup +delay for enabling the `aae` fullsync strategy. AAE can also be enabled +for the first time on a cluster, although some custom settings can +enhance performance in this case to help AAE trees be built more +quickly. See [Configuration/AAE Tree Build Optimization](#aae-tree-build-optimization). + +## Requirements: + +* Riak Enterprise version 1.4.0 or later installed on source and sink + clusters +* Riak MDC Replication Version 3 enabled on source and sink + clusters +* Both source and sink clusters must be of the same ring size +* AAE must be enabled on both source and sink clusters +* `fullsync_strategy` in the `riak_repl` section of the + `advanced.config` configuration file must be set to `aae` on both + source and sink clusters +* AAE trees must have been built on both source and sink clusters. In + the event that an AAE tree is not built on both the source and sink, + fullsync will default to the `keylist` fullsync strategy for that + partition. + +## Configuration + +If you are using Riak version 2.0, configuration is managed +using the `advanced.config` files on +each node. The semantics of the `advanced.config` file are similar to +the formerly used `app.config` file. For more information and for a list +of configurable parameters, see our documentation on [Advanced Configuration][config reference#advanced]. + +## Enable Active Anti-Entropy + +To enable [active anti-entropy][glossary aae] \(AAE) in Riak, you must enable it in Riak in both source and sink clusters. If it is not +enabled, the `keylist` strategy will be used. + +To enable AAE in Riak KV: + +```riakconf +anti_entropy = active +``` + +By default, it could take a couple of days for the cluster to build all +of the necessary hash trees because the default **build rate** of trees +is to build 1 partition per hour, per node. With a +[ring size][concept clusters] of 256 and 5 nodes, that is 2 days. + +Changing the rate of tree building can speed up this process, with the +caveat that rebuilding a tree takes processing time from the cluster, +and this should not be done without assessing the possible impact on +get/put latencies for normal cluster operations. For a production +cluster, we recommend leaving the default in place. + +For a test cluster, the build rate can be changed in `riak.conf`. If a +partition has not had its AAE tree built yet, it will default to using +the `keylist` replication strategy. Instructions on these settings can +be found in the section directly below. + +<div id="aae-tree-build-optimization"></div> + +### AAE Tree Build Optimization + +You can speed up the build rate for AAE-related hash trees by adjusting +the `anti_entropy.tree.build_limit.*` and `anti_entropy.concurrency_limit` +settings. + +```riakconf +anti_entropy.tree.build_limit.number = 10 +anti_entropy.tree.build_limit.per_timespan = 1h +anti_entropy.concurrency_limit = 10 +``` + +### Enable AAE Fullsync Replication Strategy + +Finally, the replication fullsync strategy must be set to use `aae` on +both source and sink clusters. If not, the `keylist` replication +strategy will be used. + +To enable AAE w/ Version 3 MDC Replication: + +```advancedconfig +{riak_repl, [ + % ... + {fullsync_strategy, aae}, + % ... + ]} +``` diff --git a/content/riak/kv/2.2.6/using/reference/v3-multi-datacenter/architecture.md b/content/riak/kv/2.2.6/using/reference/v3-multi-datacenter/architecture.md new file mode 100644 index 0000000000..c0119cf0e6 --- /dev/null +++ b/content/riak/kv/2.2.6/using/reference/v3-multi-datacenter/architecture.md @@ -0,0 +1,182 @@ +--- +title_supertext: "V3 Multi-Datacenter Replication Reference:" +title: "Architecture" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Architecture" + identifier: "managing_ref_v3_architecture" + weight: 100 + parent: "managing_ref_v3" +toc: true +commercial_offering: true +aliases: + - /riak-docs/riak/2.2.6/ops/mdc/v3/architecture + - /riak-docs/riak/kv/2.2.6/ops/mdc/v3/architecture +--- + +[glossary vnode]: {{<baseurl>}}riak/kv/2.2.6/learn/glossary/#vnode +[concept clusters]: {{<baseurl>}}riak/kv/2.2.6/learn/concepts/clusters + +## How Version 3 Replication Works + +In Multi-Datacenter (MDC) Replication, a cluster can act as either the + +* **source cluster**, which sends replication data to one or +* **sink clusters**, which are generally located in datacenters in other + regions or countries. + +Bidirectional replication can easily be established by making a cluster +both a source and sink to other clusters. Riak +Multi-Datacenter Replication is considered "masterless" in that all +clusters participating will resolve replicated writes via the normal +resolution methods available in Riak. + +In Multi-Datacenter Replication, there are two primary modes of +operation: + +* **Fullsync** replication is a complete synchronization that occurs + between source and sink cluster(s), which can be performed upon + initial connection of a sink cluster if you wish +* **Realtime** replication is a continual, incremental synchronization + triggered by successful writing of new updates on the source cluster + +Fullsync and realtime replication modes are described in detail below. + +## Concepts + +### Sources + +A source refers to a cluster that is the primary producer of replication +data. A source can also refer to any node that is part of the source +cluster. Source clusters push data to sink clusters. + +### Sinks + +A sink refers to a cluster that is the primary consumer of replication +data. A sink can also refer to any node that is part of the sink +cluster. Sink clusters receive data from source clusters. + +### Cluster Manager + +The cluster manager is a Riak service that provides +information regarding nodes and protocols supported by the sink and +source clusters. This information is primarily consumed by the +`riak-repl connect` command. + +### Fullsync Coordinator + +In fullsync replication, a node on the source cluster is elected to be +the *fullsync coordinator*. This node is responsible for starting and +stopping replication to the sink cluster. It also communicates with the +sink cluster to exchange key lists and ultimately transfer data across a +TCP connection. If a fullsync coordinator is terminated as the result of +an error, it will automatically restart on the current node. If the node +becomes unresponsive, a leader election will take place within 5 seconds +to select a new node from the cluster to become the coordinator. In the +event of a coordinator restart, a fullsync will have to restart. + +## Fullsync Replication + +Fullsync replication scans through the list of partitions in a Riak +cluster and determines which objects in the sink cluster need to be +updated. A source partition is synchronized to a node on the sink +cluster containing the current partition. + +## Realtime Replication + +In realtime replication, a node in the source cluster will forward data +to the sink cluster. A node in the source cluster does not necessarily +connect to a node containing the same [vnode][glossary vnode] on +the sink cluster. This allows Riak to spread out realtime replication +across the entire cluster, thus improving throughput and making +replication more fault tolerant. + +### Initialization + +Before a source cluster can begin pushing realtime updates to a sink, +the following commands must be issued: + +1. `riak-repl realtime enable <sink_cluster>` + + After this command, the realtime queues (one for each Riak node) are + populated with updates to the source cluster, ready to be pushed to + the sink. + +2. `riak-repl realtime start <sink_cluster>` + + This instructs the Riak connection manager to contact the sink + cluster. + + <br /> + ![MDC fullsync]({{<baseurl>}}images/MDC-v3-realtime1.png) + <br /> + + At this point realtime replication commences. + +<ol start="3"> +<li>Nodes with queued updates establish connections to the sink cluster +and replication begins.</li> +</ol> + +<br /> +![MDC fullsync]({{<baseurl>}}images/MDC-v3-realtime2.png) +<br /> + +### Realtime queueing and synchronization + +Once initialized, realtime replication continues to use the queues to +store data updates for synchronization. + +<ol start="4"> +<li>The client sends an object to store on the source cluster.</li> +<li>Riak writes N replicas on the source cluster.</li> +</ol> + +<br /> +![MDC fullsync]({{<baseurl>}}images/MDC-v3-realtime3.png) +<br /> + +<ol start="6"> +<li>The new object is stored in the realtime queue.</li> +<li>The object is copied to the sink cluster.</li> +</ol> + +<br /> +![MDC fullsync]({{<baseurl>}}images/MDC-v3-realtime4.png) +<br /> + +<ol start="8"> +<li>The destination node on the sink cluster writes the object to N +nodes.</li> +</ol> + +<br /> +![MDC fullsync]({{<baseurl>}}images/MDC-v3-realtime5.png) +<br /> + +<ol start="9"> +<li>The successful write of the object to the sink cluster is +acknowledged and the object removed from the realtime queue.</li> +</ol> + +<br /> +![MDC fullsync]({{<baseurl>}}images/MDC-v3-realtime6.png) +<br /> + +## Restrictions + +It is important to note that both clusters must have certain attributes +in common for Multi-Datacenter Replication to work. If you are using +either fullsync or realtime replication, both clusters must have the +same [ring size][concept clusters]; if you are using fullsync +replication, every bucket's `n_val` must be the same in both the +source and sink cluster. + + +<script type="text/javascript"> +document.write(unescape("%3Cscript src='//munchkin.marketo.net/munchkin.js' type='text/javascript'%3E%3C/script%3E")); +</script> +<script>Munchkin.init('721-DGT-611');</script> diff --git a/content/riak/kv/2.2.6/using/reference/v3-multi-datacenter/cascading-writes.md b/content/riak/kv/2.2.6/using/reference/v3-multi-datacenter/cascading-writes.md new file mode 100644 index 0000000000..9bf99097e3 --- /dev/null +++ b/content/riak/kv/2.2.6/using/reference/v3-multi-datacenter/cascading-writes.md @@ -0,0 +1,98 @@ +--- +title_supertext: "V3 Multi-Datacenter Replication Reference:" +title: "Cascading Realtime Writes" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Cascading Writes" + identifier: "managing_ref_v3_cascading_writes" + weight: 102 + parent: "managing_ref_v3" +toc: true +commercial_offering: true +aliases: + - /riak-docs/riak/2.2.6/ops/mdc/v3/cascading-writes + - /riak-docs/riak/kv/2.2.6/ops/mdc/v3/cascading-writes +--- + +## Introduction + +Riak includes a feature that cascades realtime writes across +multiple clusters. + +Cascading Realtime Writes is enabled by default on new clusters running +Riak. It will need to be manually enabled on existing clusters. + +Cascading realtime requires the `{riak_repl, rtq_meta}` capability to +function. + +{{% note title="Note on cascading tracking" %}} +Cascading tracking is a simple list of where an object has been written. This +works well for most common configurations. Larger installations, however, may +have writes cascade to clusters to which other clusters have already written. +{{% /note %}} + + +``` ++---+ +---+ +---+ +| A | <-> | B | <-> | C | ++---+ +---+ +---+ + ^ ^ + | | + V V ++---+ +---+ +---+ +| F | <-> | E | <-> | D | ++---+ +---+ +---+ +``` + +In the diagram above, a write at cluster A will begin two cascades. One +goes to B, C, D, E, and finally F; the other goes to F, E, D, C, and +finally B. Each cascade will loop around to A again, sending a +replication request even if the same request has already occurred from +the opposite direction, creating 3 extra write requests. + +This can be mitigated by disabling cascading in a cluster. If cascading +were disabled on cluster D, a write at A would begin two cascades. One +would go through B, C, and D, the other through F, E, and D. This +reduces the number of extraneous write requests to 1. + +A different topology can also prevent extra write requests: + +``` ++---+ +---+ +| A | | E | ++---+ +---+ + ^ ^ ^ ^ + | \ +---+ +---+ / | + | > | C | <-> | D | < | + | / +---+ +---+ \ | + V V V V ++---+ +---+ +| B | | F | ++---+ +---+ +``` + +A write at A will cascade to C and B. B will not cascade to C because +A will have already added C to the list of clusters where the write has +occurred. C will then cascade to D. D then cascades to E and F. E and F +see that the other was sent a write request (by D), and so they do not +cascade. + +## Usage + +Riak Cascading Writes can be enabled and disabled using the +`riak-repl` command. Please see the [Version 3 Operations guide]({{<baseurl>}}riak/kv/2.2.6/using/cluster-operations/v3-multi-datacenter) for more information. + +To show current the settings: + +`riak-repl realtime cascades` + +To enable cascading: + +`riak-repl realtime cascades always` + +To disable cascading: + +`riak-repl realtime cascades never` diff --git a/content/riak/kv/2.2.6/using/reference/v3-multi-datacenter/scheduling-fullsync.md b/content/riak/kv/2.2.6/using/reference/v3-multi-datacenter/scheduling-fullsync.md new file mode 100644 index 0000000000..b29b402ad2 --- /dev/null +++ b/content/riak/kv/2.2.6/using/reference/v3-multi-datacenter/scheduling-fullsync.md @@ -0,0 +1,68 @@ +--- +title_supertext: "V3 Multi-Datacenter Replication Reference:" +title: "Scheduling Fullsync" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Scheduling Fullsync" + identifier: "managing_ref_v3_fullsync" + weight: 103 + parent: "managing_ref_v3" +toc: true +commercial_offering: true +aliases: + - /riak-docs/riak/2.2.6/ops/mdc/v3/scheduling-fullsync + - /riak-docs/riak/kv/2.2.6/ops/mdc/v3/scheduling-fullsync +--- + +[config reference#advanced]: {{<baseurl>}}riak/kv/2.2.6/configuring/reference/#advanced-configuration + +The `fullsync_interval` parameter can be configured in the `riak-repl` +section of [`advanced.config`][config reference#advanced] with either: + +* a single integer value representing the duration to wait, in minutes, + between fullsyncs, _or_ +* a list of pairs of the form `[{"clustername", time_in_minutes}, + {"clustername", time_in_minutes}, ...]` pairs for each sink + participating in fullsync replication. Note the commas separating each + pair, and `[ ]` surrounding the entire list. + +## Examples + +Sharing a fullsync time (in minutes) for all sinks: + +```advancedconfig +{riak_repl, [ + % ... + {data_root, "/configured/repl/data/root"}, + {fullsync_interval, 90} %% fullsync runs every 90 minutes + % ... + ]} +``` + +List of multiple sinks with separate times in minutes: + +```advancedconfig +{riak_repl, [ + % ... + {data_root, "/configured/repl/data/root"}, + % clusters sink_boston + sink_newyork have difference intervals (in minutes) + {fullsync_interval, [ + {"sink_boston", 120}, %% fullsync to sink_boston with run every 120 minutes + {"sink_newyork", 90}]} %% fullsync to sink_newyork with run every 90 minutes + + ]} +``` + +## Additional Fullsync Stats + +Additional fullsync stats per sink have been added in Riak. + +* `fullsyncs_completed` — The number of fullsyncs that have been + completed to the specified sink cluster. +* `fullsync_start_time` — The time the current fullsink to the + specified cluster began. +* `last_fullsync_duration` — The duration (in seconds) of the last + completed fullsync. diff --git a/content/riak/kv/2.2.6/using/repair-recovery.md b/content/riak/kv/2.2.6/using/repair-recovery.md new file mode 100644 index 0000000000..1482b98fd6 --- /dev/null +++ b/content/riak/kv/2.2.6/using/repair-recovery.md @@ -0,0 +1,48 @@ +--- +title: "Repair & Recovery" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Repair & Recovery" + identifier: "managing_repair_recover" + weight: 204 + parent: "managing" +toc: true +--- + +[repair recover fail]: ./failure-recovery/ +[repair recover errors]: ./errors/ +[repair recover repairs]: ./repairs/ +[repair recover restart]: ./rolling-restart/ + +## In This Section + +#### [Failure & Recovery][repair recover fail] + +Lists steps that can be taken to minimize the harm caused by a general +cluster failure. + +[Learn More >>][repair recover fail] + + +#### [Errors & Messages][repair recover errors] + +Details most common errors & messages. + +[Learn More >>][repair recover errors] + + +#### [Repairs][repair recover repairs] + +Tutorials on running various repair operations. + +[Learn More >>][repair recover repairs] + + +#### [Rolling Restarts][repair recover restart] + +Brief guide on performing node-by-node restarts. + +[Learn More >>][repair recover restart] diff --git a/content/riak/kv/2.2.6/using/repair-recovery/errors.md b/content/riak/kv/2.2.6/using/repair-recovery/errors.md new file mode 100644 index 0000000000..609e3ba941 --- /dev/null +++ b/content/riak/kv/2.2.6/using/repair-recovery/errors.md @@ -0,0 +1,362 @@ +--- +title: "Errors & Messages" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Errors" + identifier: "repair_recover_errors" + weight: 101 + parent: "managing_repair_recover" +toc: true +aliases: + - /riak-docs/riak/2.2.6/ops/running/recovery/errors + - /riak-docs/riak/kv/2.2.6/ops/running/recovery/errors +--- + +[config reference]: {{<baseurl>}}riak/kv/2.2.6/configuring/reference + +This is not a comprehensive listing of every error that Riak may +encounter -- screws fall out all of the time, the world is an imperfect +place. This is an attempt at capturing the most common recent errors +that users do encounter, as well as give some description to non +critical error atoms which you may find in the logs. + +Discovering the source of an error can take some detective work, since +one error can cause a cascade of errors. + +The tables in this document do not specify which logs these error +messages may appear in. Depending upon your log configuration some may +appear more often (i.e., if you set the log to debug), while others may +output to your console (eg. if you tee'd your output or started as `riak +console`). + +You can optionally customize your log message format via the +`lager_default_formatter` field under `lager` in `app.config`. If you +do, your messages will look different from those shown in this document. + +Finally, this document is organized to be able to lookup portions of a +log message, since printing every variation would be a bit unwieldy. For +example, this message: + +``` +12:34:27.999 [error] gen_server riak_core_capability terminated with reason:\ +no function clause matching orddict:fetch('riak@192.168.2.81', []) line 72 +``` + +Starts with a date (`12:34:27.999`), followed by the log severity +(`[error]`), with a message formatted by lager (found in the Lager table +below as *gen_server `Mod` terminated with reason: `Reason`*) + +### Lager Formats + +Riak's main logging mechanism is the project Lager, so it's good to note +some of the more common message formats. In almost every case the +reasons for the error are described as variables, such as `Reason` of +`Mod` (meaning the Erlang module which is generally the source of the +error). + +Riak does not format all error messages that it receives into +human-readable sentences. However, It does output errors as objects. + +The above example error message corresponds with the first message in +this table, where the Erlang `Mod` value is `riak_core_capability` and +the reason was an Erlang error: `no function clause matching +orddict:fetch('riak@192.168.2.81', []) line 72`. + +Error | Message +------|-------- + | `gen_server <Mod> terminated with reason: <Reason>` + | `gen_fsm <Mod> in state <State> terminated with reason: <Reason>` + | `gen_event <ID> installed in <Mod> terminated with reason: <Reason>` +`badarg` | `bad argument in call to <Mod1> in <Mod2>` +`badarith` | `bad arithmetic expression in <Mod>` +`badarity` | `fun called with wrong arity of <Ar1> instead of <Ar2> in <Mod>` +`badmatch` | `no match of right hand value <Val> in <Mod>` +`bad_return` | `bad return value <Value> from <Mod>` +`bad_return_value` | `bad return value: <Val> in <Mod>` +`badrecord` | `bad record <Record> in <Mod>` +`case_clause` | `no case clause matching <Val> in <Mod>` +`emfile` | `maximum number of file descriptors exhausted, check ulimit -n` +`function_clause` | `no function clause matching <Mod>` +`function not exported` | `call to undefined function <Func> from <Mod>` +`if_clause` | `no true branch found while evaluating if expression in <Mod>` +`noproc` | `no such process or port in call to <Mod>` +`{system_limit, {erlang, open_port}}` | `maximum number of ports exceeded` +`{system_limit, {erlang, spawn}}` | `maximum number of processes exceeded` +`{system_limit, {erlang, spawn_opt}}` | `maximum number of processes exceeded` +`{system_limit, {erlang, list_to_atom}}` | `tried to create an atom larger than 255, or maximum atom count exceeded` +`{system_limit, {ets, new}}` | `maximum number of Erlang Term Storage (ETS) tables exceeded` +`try_clause` | `no try clause matching <Val> in <Mod>` +`undef` | `call to undefined function <Mod>` + +### Error Atoms + +Since Erlang programming support is a "happy path/fail fast" style, one +of the more common error log strings you might encounter contain +`{error,{badmatch,{...`. This is Erlang's way of telling you that an +unexpected value was assigned, so these errors can prefix the more +descriptive parts. In this case, `{error,{badmatch,{...` prefixes the +more interesting `insufficient_vnodes_available` error, which can be +found in the `riak_kv` table later on in this document. + +```log +2012-01-13 02:30:37.015 [error] <0.116.0> webmachine error: path="/riak-docs/riak/contexts"\ +{error,{error,{badmatch,{error,insufficient_vnodes_available}},\ +[{riak_kv_wm_keylist,produce_bucket_body,2},{webmachine_resource,resource_call,3},\ +{webmachine_resour,resource_call,1},{webmachine_decision_core,decision,1},\ +{webmachine_decision_core,handle_request,2},\ +{webmachine_mochiweb,loop,1},{mochiweb_http,headers,5}]}} +``` + +## Erlang Errors + +Although relatively rare once a Riak cluster is running in production, +users new to Riak or Erlang occasionally encounter errors on initial +installation. These spring from a setup Erlang does not expect, +generally due to network, permission, or configuration problems. + +Error | Description | Resolution +:-----|:------------|:---------- +`{error,duplicate_name}` | You are trying to start a new Erlang node, but another node with the same name is already running | You might be attempting to start multiple nodes on the same machine with the same `vm.args` `-name` value; or if Riak is already running, check for `beam.smp`; or epmd thinks Riak is running, check/kill epmd +`{error,econnrefused}` | Remote Erlang node connection refused | Ensure your cluster is up and nodes are able to communicate with each other. See <a href="http://docs.basho.com/riak/kv/2.2.6/using/repair-recovery/errors/#more">Step 1</a>. +`{error,ehostunreach}` | Remote node cannot be connected to | Ensure that nodes are able to communicate with each other. See <a href="http://docs.basho.com/riak/kv/2.2.6/using/repair-recovery/errors/#more">Step 1</a>. +`{error,eacces}` | Cannot write a given file | Ensure the Riak beam process has permission to write to all `*_dir` values in `app.config`, for example, `ring_state_dir`, `platform_data_dir`, and others +`{error,enoent}` | Missing an expected file or directory | Ensure all `*_dir` values in `app.config` exist, for example, `ring_state_dir`, `platform_data_dir`, and others +`{error,erofs}` | A file/directory is attempted to be written to a read-only filesystem | Only set Riak directories to read/write filesystems +`system_memory_high_watermark` | Often a sign than an <a href="http://www.erlang.org/doc/man/ets.html">ETS table</a> has grown too large | Check that you are using a backend appropriate for your needs (LevelDB for very large key counts) and that your vnode count is reasonable (measured in dozens per node rather than hundreds) +`temp_alloc` | Erlang attempting to allocate memory | Often associated with `Cannot allocate X bytes of memory`, which means that you're either creating too large of an object or that you simply don't have enough RAM. Base minimum suggested RAM per node is 4GB. + +## Riak Errors and Messages + +Many KV errors have prescriptive messages. For such cases we leave it to +Riak to explain the correct course of action. For example, the +`map/reduce` `parse_input` phase will respond like this when it +encounters an invalid input: + +{{% note title="Note on inputs" %}} +Inputs must be a binary bucket, a tuple of bucket and key-filters, a list of +target tuples, a search index, or modfun tuple: `INPUT`. +{{% /note %}} + +For the remaining common error codes, they are often marked by Erlang +atoms (and quite often wrapped within an `{error,{badmatch,{...` tuple, +as described in the [Error](#erlang-errors) section +above). This table lays out those terse error codes and related log +messages, if they exist. + +### Riak Core + +Riak Core is the underlying implementation for KV. These are errors +originating from that framework, and can appear whether you use KV, +Search, or any Core implementation. + +Error | Message | Description | Resolution +:-----|:--------|:------------|:---------- +`behavior` | | Attempting to execute an unknown behavior | Ensure that your configuration file choices (e.g. backends) support the behaviors you're attempting to use, such as configuring LevelDB to use secondary indexes +`already_leaving` | `Node is already in the process of leaving the cluster` | An error marking a node to leave when it is already leaving | No need to duplicate the `leave` command +`already_replacement` | | This node is already in the replacements request list | You cannot replace the same node twice +`{different_owners, N1, N2}` | | Two nodes list different partition owners, meaning the ring is not ready | When the ring is ready, the status should be ok +`different_ring_sizes` | | The joining ring is a different size from the existing cluster ring | Don't join a node already joined to a cluster +`insufficient_vnodes_available` | | When creating a query coverage plan, not enough vnodes are available | Check the `riak-admin ring-status` and ensure all of your nodes are healthy and connected +`invalid_replacement` | | A new node is currently joining from a previous operation, so a replacement request is invalid until it is no longer joining | Wait until the node is finished joining +`invalid_ring_state_dir` | `Ring state directory <RingDir> does not exist, and could not be created: <Reason>` | The ring directory does not exist and no new dir can be created in expected location | Ensure that the Erlang proc can write to `ring_state_dir`or has permission to create that dir +`is_claimant` | | A node cannot be the claimant of its own remove request | Remove/replace nodes from another node +`is_up` | | Node is expected to be down but is up | When a node is downed, it should be down +`legacy` | | Attempting to stage a plan against a legacy ring | Staging is a feature only of Riak versions 1.2.0+ +`max_concurrency` | `Handoff receiver for partition <Partition> exited abnormally after processing <Count> objects: <Reason>` | Disallow more handoff processes than the `riak_core` `handoff_concurrency` setting (defaults to 2) | If this routinely kills vnodes, this issue has been linked to LevelDB compactions which can build up and block writing, which will also be accompanied by LevelDB logs saying `Waiting...` or `Compacting` +`{nodes_down, Down}` | | All nodes must be up to check | +`not_member` | | This node is not a member of the ring | Cannot leave/remove/down when this is not a ring member +`not_reachable` | | Cannot join unreachable node | Check your network connections, ensure Erlang cookie setting `vm.args` `-setcookie` +`{not_registered, App}` | | Attempting to use an unregistered process | Ensure that your `app.config` choices contain the app you're attempting to use `{riak_kv_stat, true}` +`not_single_node` | | There are no other members to join | Join with at least one other node +`nothing_planned` | | Cannot commit a plan without changes | Ensure at least one ring change is planned before running commit +`only_member` | | This is the only member of the ring | Cannot leave/remove/down when this is the only member of the ring +`ring_not_ready` | | Ring not ready to perform command | Attempting to plan a ring change before the ring is ready to do so +`self_join` | | Cannot join node with itself | Join another node to form a valid cluster +`timeout` | `<Type> transfer of <Module> from <SrcNode> <SrcPartition> to <TargetNode> <TargetPartition> failed because of TCP recv timeout` | | Ensure that ports chosen in your configuration files do not overlap with ports being used by your system, or with each other +`unable_to_get_join_ring` | | Cannot access cluster ring to join | Possible corrupted ring +`{unknown_capability, Capability}` | | Attempting to use a capability unsupported by this implementation | Ensure that your configuration choices support the capability you're attempting to use, such as Pipe MapReduce (setting a `mapred_2i_pipe` value in `app.config`) +`vnode_exiting` | `<Mod> failed to store handoff obj: <Err>` | | A vnode fails to hand off data because the handoff state is deleted +`vnode_shutdown` | | The vnode worker pool is shutting down | Various reasons can cause a shutdown, check other log messages + | `Bucket validation failed <Detail>` | | Only set value bucket properties + | `set_recv_data called for non-existing receiver` | Cannot connect to receiver during handoff | Ensure receiver node is still up and running, and that the standard + | `An <Dir> handoff of partition <M> was terminated because the vnode died` | Handoff stopped because of vnode was `DOWN` and sender must be killed | An expected message if a vnode dies during handoff. Check the logs for other causes. + | `status_update for non-existing handoff <Target>` | Cannot get the status of a handoff `Target` module that doesn't exist | An expected message. Check the logs for other causes. + | `SSL handoff config error: property <FailProp>: <BadMat>.` | The receiver may reject the senders attempt to start a handoff | Ensure your SSL settings and certificates are proper + | `Failure processing SSL handoff config <Props>:<X>:<Y>` | | Ensure your SSL settings and certificates are proper + | `<Type> transfer of <Module> from <SrcNode> <SrcPartition> to <TargetNode> <TargetPartition> failed because of <Reason>` | Nodes cannot hand off data | Ensure that your cluster is up and nodes are able to communicate with each other. See <a href="http://docs.basho.com/riak/kv/2.2.6/using/repair-recovery/errors/#more"> Step 1</a>. + | `Failed to start application: <App>` | Expected application cannot load | This relates to an Erlang application, and not necessarily the Riak application in general. The app may fail to load for many reasons, such as a missing native library. Read other log messages for clues + | `Failed to read ring file: <Reason>` | Gives a reason why the ring file cannot be read on startup | The reason given explains the problem, such as `eacces` meaning the Erlang process does not have permission to read + | `Failed to load ring file: <Reason>` | Gives a reason why the ring file cannot be loaded on startup | The reason given explains the problem, such as `enoent` meaning the expected file cannot be found + | `ring_trans: invalid return value: <Other>` | Transferring ring data between nodes received an invalid value | Often associated with ring corruption, or an unexpected exit from the transferring node + | `Error while running bucket fixup module <Fixup> from application <App> on bucket <BucketName>: <Reason>` | | Various sources for a fixup error, read associated errors + | `Crash while running bucket fixup module <Fixup> from application <App> on bucket <BucketName> : <What>:<Why>` | | Various source for a fixup error, read associated errors + | `<Index> <Mod> worker pool crashed <Reason>` | | Various reasons can be the source of a worker pool crash, read associated errors + | `Received xfer_complete for non-existing repair: <ModPartition>` | Unexpected repair message | Not much to do here, but a node did not expect to receive a `xfer_complete` status + +### Riak KV + +Riak KV is the key/value implementation, generally just considered to be +Riak proper. This is the source of most of the code, and consequently, +most of the error messages. + +Error | Message | Description | Resolution +:-----|:--------|:------------|:---------- +`all_nodes_down` | | No nodes are available | Check `riak-admin member-status` and ensure that all expected nodes in the cluster are of `valid` Status +`{bad_qterm, QueryTerm}` | | Bad query when performing MapReduce | Fix your MapReduce query +`{coord_handoff_failed, Reason}` | `Unable to forward put for <Key> to <CoordNode> - <Reason>` | Vnodes unable to communicate | Check that coordinating vnode is not down. Ensure your cluster is up and nodes are able to communicate with each other. See <a href="http://docs.basho.com/riak/kv/2.2.6/using/repair-recovery/errors/#more"> Step 1</a>. +`{could_not_reach_node, Node}` | | Erlang process was not reachable | Check network settings; ensure remote nodes are running and reachable; ensure all nodes have the same Erlang cookie setting `vm.args` `-setcookie`. See <a href="http://docs.basho.com/riak/kv/2.2.6/using/repair-recovery/errors/#more"> Step 1</a>. +`{deleted, Vclock}` | | The value was already deleted, includes the current vector clock | Riak will eventually clean up this tombstone +`{dw_val_violation, DW}` | | Same as `w_val_violation` but concerning durable writes | Set a valid DW value +`{field_parsing_failed, {Field, Value}}` | `Could not parse field +<Field>, value <Value>.` | Could not parse an index field | Most commonly an `_int` field which cannot be parsed. For example a query like this is invalid: `/buckets/X/index/Y_int/BADVAL`, since BADVAL should instead be an integer +`{hook_crashed, {Mod, Fun, Class, Exception}}` | `Problem invoking pre-commit hook` | Precommit process exited due to some failure | Fix the precommit function code, follow the message's exception and stacktrace to help debug +`{indexes_not_supported, Mod}` | | The chosen backend does not support indexes (only LevelDB currently supports secondary indexes) | Set your configuration to use the LevelDB backend +`{insufficient_vnodes, NumVnodes, need, R}` | | R was set greater than the total vnodes | Set a proper R value; or too many nodes are down; or too many nodes are unavailable due to crash or network partition. Ensure all nodes are available by running riak-admin ring-status. +`{invalid_hook_def, HookDef}` | `Invalid post-commit hook definition <Def>` | No Erlang module and function or JavaScript function name | Define the hook with the correct settings +`{invalid_inputdef, InputDef}` | | Bad inputs definitions when running MapReduce | Fix inputs settings; set `mapred_system` from `legacy` to `pipe` +`invalid_message` | | Unknown event sent to module | Ensure you're running similar versions of Riak across (and specifically poolboy) across all nodes +`{invalid_range, Args}` | | Index range query hasStart > End | Fix your query +`{invalid_return, {Mod, Fun, Result}}` | `Problem invoking pre-commit hook <Mod>:<Fun>, invalid return <Result>` | The given precommit function gave an invalid return for the given `Result` | Ensure your pre-commit functions return a valid result +`invalid_storage_backend` | `storage_backend <Backend> is non-loadable.` | Invalid backend choice when starting up Riak | Set a valid backend in your configuration files +`key_too_large` | | The key was larger than 65536 bytes | Use a smaller key +`local_put_failed` | | A local vnode PUT operation failed | This has been linked to a LevelDB issue related to restricted memory usage and inability to flush a write to disk. If this happens repetitively, stop/start the riak node, forcing a memory realloc +`{n_val_violation, N}` | | (W > N) or (DW > N) or (PW > N) or (R > N) or (PR > N) | No W or R values may be greater than N +`{nodes_not_synchronized, Members}` | | Rings of all members are not synchronized | Backups will fail if nodes are not synchronized +`{not_supported, mapred_index, FlowPid}` | | Index lookups for MapReduce are only supported with Pipe | Set mapred_system from legacy to pipe +`notfound` | | No value found | Value was deleted, or was not yet stored or replicated +`{pr_val_unsatisfied, PR, Primaries}` | | Same as `r_val_unsatisfied` but only counts `Primary` node replies | Too many primary nodes are down or the `PR` value was set too high +`{pr_val_violation, R}` | | Same as `r_val_violation` but concerning `Primary` reads | Set a valid `PR` value +`precommit_fail` | `Pre-commit hook <Mod>:<Fun> failed with reason <Reason>` | The given precommit function failed for the given `Reason` | Fix the precommit function code +`{pw_val_unsatisfied, PR, Primaries}` | | Same as `w_val_unsatisfied` but only counts `Primary` node replies | Too many primary nodes are down or the `PW` value was set too high +`{pw_val_violation, PW}` | | Same as `w_val_violation` but concerning primary writes | Set a valid `PW` value +`{r_val_unsatisfied, R, Replies}` | | Not enough nodes replied to satisfy the `R` value, contains the given `R` value and the actual number of `Replies` | Too many nodes are down or the R value was set too high +`{r_val_violation, R}` | | The given R value was non-numeric and not a valid setting (`on`, `all`, `quorum`) | Set a valid R value +`receiver_down` | | Remote process failed to acknowledge request | Can occur when listkeys is called +`{rw_val_violation, RW}` | | The given `RW` property was non-numeric and not a valid setting (`one`, `all`, `quorum`) | Set a valid `RW` value +`{siblings_not_allowed, Object}` | `Siblings not allowed: <Object>` | The hook to index cannot abide siblings | Set the buckets `allow_mult` property to `false` +`timeout`| | The given action took too long to reply | Ensure your cluster is up and nodes are able to communicate with each other. See <a href="http://docs.basho.com/riak/kv/2.2.6/using/repair-recovery/errors/#more"> Step 1</a>. Or check you have a reasonable `ulimit` size. Note that listkeys commands can easily timeout and shouldn't be used in production. +`{too_few_arguments, Args}` | | Index query requires at least one argument | Fix your query format +`{too_many_arguments, Args}` | | Index query is malformed with more than 1 (exact) or 2 (range) values | Fix your query format +`too_many_fails` | | Too many write failures to satisfy W or DW | Try writing again. Or ensure your nodes/network is healthy. Or set a lower W or DW value +`too_many_results` | | Too many results are attempted to be returned | This is a protective error. Either change your query to return fewer results, or change your `max_search_results` setting in `app.config` (it defaults to 100,000) +`{unknown_field_type, Field}` | `Unknown field type for field: <Field>.` | Unknown index field extension (begins with underscore) | The only value field types are `_int` and `_bin` +`{w_val_unsatisfied, RepliesW, RepliesDW, W, DW}` | | Not enough nodes replied to satisfy the W value, contains the given W value and the actual number of `Replies*` for either `W` or `DW` | Too many nodes are down or the `W` or `DW` value was set too high +`{w_val_violation, W}` | | The given W property was non-numeric and not a valid setting (on, all, quorum) | Set a valid W value + | `Invalid equality query <SKey>` | Equality query is required and must be binary for an index call | Pass in an equality value when performing a 2i equality query + | `Invalid range query: <Min> -> <Max>` | Both range query values are required and must be binary an index call | Pass in both range values when performing a 2i equality query + | `Failed to start <Mod> <Reason>:<Reason>` | Riak KV failed to start for given `Reason` | Several possible reasons for failure, read the attached reason for insight into resolution + +### Backend Errors + +These errors tend to stem from server-based problems. Backends are +sensitive to low or corrupt disk or memory resources, native code, and +configuration differences between nodes. Conversely, a network issue is +unlikely to affect a backend. + +Error | Message | Description | Resolution +:-----|:--------|:------------|:---------- +`data_root_not_set` | | Same as `data_root_unset` | Set the `data_root` directory in config +`data_root_unset` | `Failed to create bitcask dir: data_root is not set` | The `data_root` config setting is required | Set `data_root` as the base directory where to store bitcask data, under the `bitcask` section +`{invalid_config_setting, multi_backend, list_expected}` | | Multi backend configuration requires a list | Wrap `multi_backend` config value in a list +`{invalid_config_setting, multi_backend, list_is_empty`} | | Multi backend configuration requires a value | Configure at least one backend under `multi_backend` in `app.config` +`{invalid_config_setting, multi_backend_default, backend_not_found}` | | | Must choose a valid backend type to configure +`multi_backend_config_unset` | | No configuration for Multi backend | Configure at least one backend under `multi_backend` in `app.config` +`not_loaded` | | Native driver not loading | Ensure your native drivers exist (.dll or .so files {riak_kv_multi_backend, undefined_backend, BackendName} | | Backend defined for a bucket is invalid | Define a valid backed before using this bucket under lib/`project`/priv, where `project` is most likely eleveldb). +`reset_disabled` | | Attempted to reset a Memory backend in production | Don't use this in production + +### JavaScript + +These are some errors related to JavaScript pre-commit functions, +MapReduce functions, or simply the management of the pool of JavaScript +VMs. If you do not use JavaScript, these should not be encountered. If +they are, check your configuration for high `*js_vm*` values or as an +epiphenomenon to a real issue, such as low resources. + +Error | Message | Description | Resolution +---------|---------|-------------|------- +`no_vms` | `JS call failed: All VMs are busy.` | All JavaScript VMs are in use | Wait and run again; increase JavaScript VMs in `app.config` (`map_js_vm_count`, `reduce_js_vm_count`, or `hook_js_vm_count`) +`bad_utf8_character_code` | `Error JSON encoding arguments: <Args>` | A UTF-8 character give was a bad format | Only use correct UTF-8 characters for JavaScript code and arguments +`bad_json` | | Bad JSON formatting | Only use correctly formatted JSON for JavaScript command arguments + | `Invalid bucket properties: <Details>` | Listing bucket properties will fail if invalid | Fix bucket properties +`{load_error, "Failed to load spidermonkey_drv.so"}` | | The JavaScript driver is corrupted or missing | In OS X you may have compiled with `llvm-gcc` rather than `gcc`. + +### MapReduce + +These are possible errors logged by Riak's MapReduce implementation, +both legacy as well as Pipe. If you never use or call MapReduce, you +should not run across these. + +Error | Message | Description | Resolution +:-----|:--------|:------------|:---------- +`bad_mapper_props_no_keys` | | At least one property should be found by default. *Unused in Riak 1.3+* | Set mapper properties, or don't use it +`bad_mapred_inputs` | | A bad value sent to MapReduce. *Unused in Riak 1.3+* | When using the Erlang client interface, ensure all MapReduce and search queries are correctly binary +`bad_fetch` | | An expected local query was not retrievable. *Unused in Riak 1.3+* | Placing javascript MapReduce query code as a riak value must first be stored before execution +`{bad_filter, <Filter>}` | | An invalid keyfilter was used | Ensure your MapReduce keyfilter is correct +`{dead_mapper, <Stacktrace>, <MapperData>}` | | Getting a reply from a mapper for a job that has already exited. *Unused in Riak 1.3+* | Check for a stuck Erlang process, or if using legacy MR ensure `map_cache_size` is set (Both issues may require a node restart) +`{inputs, Reason}` | `An error occurred parsing the "inputs" field.` | MapReduce request has invalid input field | Fix MapReduce fields +`{invalid_json, Message}` | `The POST body was not valid JSON. The error from the parser was: <Message>` | Posting a MapReduce command requires correct JSON | Format MapReduce requests correctly +`javascript_reduce_timeout` | | JavaScript reduce function taking too long | For large numbers of objects, your JavaScript functions may become bottlenecks. Decrease the quantity of values being passed to and returned from the reduce functions, or rewrite as Erlang functions +`missing_field` | `The post body was missing the "inputs" or "query" field.` | Either an inputs or query field is required | Post MapReduce request with at least one +`{error,notfound}` | | Used in place of a RiakObject in the mapping phase | Your custom Erlang map function should deal with this type of value +`not_json` | `The POST body was not a JSON object.` | Posting a MapReduce command requires correct JSON | Format MapReduce requests correctly +`{no_candidate_nodes, exhausted_prefist, <Stacktrace>, <MapperData>}` | | Some map phase workers died | Possibly a long running job hitting MapReduce timeout, upgrade to Pipe +`{<query>, Reason}` | `An error occurred parsing the "query" field.` | MapReduce request has invalid query field | Fix MapReduce query +`{unhandled_entry, Other}` | `Unhandled entry: <Other>` | The `reduce_identity` function is unused | If you don't need `reduce_identity`, just don't set reduce phase at all +`{unknown_content_type, ContentType}` | | Bad content type for MapReduce query | Only `application/json` and `application/x-erlang-binary` are accepted + | `Phase <Fitting>: <Reason>` | A general error when something happens using the Pipe MapReduce implementation with a bad argument or configuration | Can happen with a bad map or reduce implementation, most recent known gotcha is when a JavaScript function improperly deals with tombstoned objects + | `riak_kv_w_reduce requires a function as argument, not a <Type>` | Reduce requires a function object, not any other type | This shouldn't happen +  +## Specific messages + +Although you can put together many error causes with the tables above, +here are some common yet esoteric messages with known causes and +solutions. + + Message | Resolution +:--------|:---------- +gen_server riak_core_capability terminated with reason: no function clause matching orddict:fetch('`Node`', []) | The Node has been changed, either through change of IP or `vm.args` `-name` without notifying the ring. Either use the `riak-admin cluster replace` command, or remove the corrupted ring files `rm -rf /var/lib/riak/ring/*` and rejoin to the cluster +gen_server <`PID`> terminated with reason: no function clause matching riak_core_pb:encode(`Args`) line 40 | Ensure you do not have different settings on different nodes (for example, a ttl mem setting on one node's mem backend, and another without) +monitor `busy_dist_port` `Pid` [...{almost_current_function,...] | This message means distributed Erlang buffers are filling up. Try setting zdbbl higher in `vm.args`, such as `+zdbbl 16384`. Or check that your network is not slow. Or ensure you are not slinging large values. If a high bandwidth network is congested, try setting RTO_min down to 0 msec (or 1msec). +<`PID`>@riak_core_sysmon___handler:handle_event:89 Monitor got {suppressed,port_events,1} | Logged as info, you can add `+swt very_low` to your `vm.args` +(in LevelDB LOG files) Compaction error | Turn off the node and run repair on the LevelDB partition. See <a href="http://docs.basho.com/riak/kv/2.2.6/using/repair-recovery/errors/#more">Step 2</a>. +enif_send: env==NULL on non-SMP VM/usr/lib/riak/lib/os_mon-2.2.9/priv/bin/memsup: Erlang has closed. | Riak's Erlang VM is built with SMP support and if Riak is started on a non-SMP system, an error like this one is logged. This is commonly seen in virtualized environments configured for only one CPU core. +exit with reason bad return value: {error,eaddrinuse} in context start_error | An error like this example can occur when another process is already bound to the same address as the process being started is attempting to bind to. Use operating system tools like `netstat`, `ps`, and `lsof` to determine the root cause for resolving this kind of errors; check for existence of stale `beam.smp` processes. +exited with reason: eaddrnotavail in gen_server:init_it/6 line 320 | An error like this example can result when Riak cannot bind to the addresses specified in the configuration. In this case, you should verify HTTP and Protocol Buffers addresses in `app.config` and ensure that the ports being used are not in the privileged (1-1024) range as the `riak` user will not have access to such ports. +gen_server riak_core_capability terminated with reason: no function clause matching orddict:fetch('riak@192.168.2.2', []) line 72 | Error output like this example can indicate that a previously running Riak node with an original `-name` value in `vm.args` has been modified by simply changing the value in `vm.args` and not properly through `riak-admin cluster replace`. +** Configuration error: [FRAMEWORK-MIB]: missing context.conf file => generating a default file | This error is commonly encountered when starting Riak Enterprise without prior [SNMP]({{<baseurl>}}riak/kv/2.2.6/using/reference/snmp) configuration. +RPC to 'node@example.com' failed: {'EXIT', {badarg, [{ets,lookup, [schema_table,<<"search-example">>], []} {riak_search_config,get_schema,1, [{file,"src/riak_search_config.erl"}, {line,69}]} ...| This error can be caused when attempting to use Riak Search without first enabling it in each node's `app.config`. See the [configuration files][config reference] documentation for more information on enabling Riak Search. + + +### More + +1. <a name="f1"></a>Ensure node inter-communication + - Check `riak-admin member-status` and ensure the cluster is valid. + - Check `riak-admin ring-status` and ensure the ring and vnodes are communicating as expected. + - Ensure your machine does not have a firewall or other issue that prevents traffic to the remote node. + - Your remote `vm.args` `-setcookie` must be the same value for every node in the cluster. + - The `vm.args` `-name` value must not change after joining the node (unless you use `riak-admin cluster replace`). + +2. <a name="f2"></a>Run LevelDB compaction + 1. `find . -name "LOG" -exec grep -l 'Compaction error' {} \;` *(Finding one compaction error is interesting, more than one might be a strong indication of a hardware or OS bug)* + 2. Stop Riak on the node: `riak stop` + 3. Start an Erlang session (do not start riak, we just want Erlang) + 4. From the Erlang console perform the following command to open the LevelDB database + + ```erlang + [application:set_env(eleveldb, Var, Val) || {Var, Val} <- + [{max_open_files, 2000}, + {block_size, 1048576}, + {cache_size, 20*1024*1024*1024}, + {sync, false}, + {data_root, "/var/db/riak/leveldb"}]]. + ``` + 5. For each of the corrupted LevelDB databases (found by `find . -name "LOG" -exec` | `grep -l 'Compaction error' {} \; `) run this command substituting in the proper vnode number. + + ```erlang + eleveldb:repair("/var/db/riak/leveldb/442446784738847563128068650529343492278651453440", []). + ``` + 6. When all have finished successfully you may restart the node: `riak start` + 7. Check for proper operation by looking at log files in /var/log/riak and in the LOG files in the effected LevelDB vnodes. diff --git a/content/riak/kv/2.2.6/using/repair-recovery/failed-node.md b/content/riak/kv/2.2.6/using/repair-recovery/failed-node.md new file mode 100644 index 0000000000..b6850aa55f --- /dev/null +++ b/content/riak/kv/2.2.6/using/repair-recovery/failed-node.md @@ -0,0 +1,110 @@ +--- +title: "Recovering a Failed Node" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Recover a Failed Node" + identifier: "repair_recover_failed_node" + weight: 104 + parent: "managing_repair_recover" +toc: true +aliases: + - /riak-docs/riak/2.2.6/ops/running/recovery/failed-node + - /riak-docs/riak/kv/2.2.6/ops/running/recovery/failed-node +--- + +## General Recovery Notes + +A Riak node can fail for many reasons, but a handful of checks enable you to +uncover some of the most common problems that can lead to node failure, +such as checking for RAID and filesystem consistency or faulty memory and +ensuring that your network connections are fully functioning. + +When a node fails and is then brought back into the cluster, make sure that it has the same node name that it did before it crashed. If the name has changed, the cluster will assume that the node is entirely new and that the crashed node is still part of the cluster. + +During the recovery process, hinted handoff will kick in and update the data on +the recovered node with updates accepted from other nodes in the cluster. Your +cluster may temporarily return `not found` for objects that are currently +being handed off (see our page on [Eventual Consistency](../../../learn/concepts/eventual-consistency) for more details on +these scenarios, in particular how the system behaves while the failed node is +not part of the cluster). + +## Node Name Changed + +If you are recovering from a scenario in which node name changes are out of +your control, you'll want to notify the cluster of its *new* name using the +following steps: + +1. Stop the node you wish to rename: + + ```bash + riak stop + ``` + + +2. Mark the node down from another node in the cluster: + + ```bash + riak-admin down <previous_node_name> + ``` + +3. Update the node name in Riak's configuration files: + + ```riakconf + nodename = <updated_node_name> + ``` + + ```vmargs + -name <updated_node_name> + ``` + +4. Delete the ring state directory (usually `/var/lib/riak/ring`). + +5. Start the node again: + + ```bash + riak start + ``` + +6. Ensure that the node comes up as a single instance: + + ```bash + riak-admin member-status + ``` + + The output should look something like this: + + ``` + ========================= Membership ========================== +Status Ring Pending Node +--------------------------------------------------------------- +valid 100.0% -- 'dev-rel@127.0.0.1' +--------------------------------------------------------------- +Valid:1 / Leaving:0 / Exiting:0 / Joining:0 / Down:0 + ``` + +7. Join the node to the cluster: + + ```bash + riak-admin cluster join <node_name_of_a_member_of_the_cluster> + ``` + +8. Replace the old instance of the node with the new: + + ```bash + riak-admin cluster force-replace <previous_node_name> <new_node_name> + ``` + +9. Review the changes: + + ```bash + riak-admin cluster plan + ``` + + Finally, commit those changes: + + ```bash + riak-admin cluster commit + ``` diff --git a/content/riak/kv/2.2.6/using/repair-recovery/failure-recovery.md b/content/riak/kv/2.2.6/using/repair-recovery/failure-recovery.md new file mode 100644 index 0000000000..b812adaab0 --- /dev/null +++ b/content/riak/kv/2.2.6/using/repair-recovery/failure-recovery.md @@ -0,0 +1,125 @@ +--- +title: "Failure & Recovery" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Failure & Recovery" + identifier: "repair_recover_failure" + weight: 100 + parent: "managing_repair_recover" +toc: true +aliases: + - /riak-docs/riak/2.2.6/ops/running/recovery/failure-recovery + - /riak-docs/riak/kv/2.2.6/ops/running/recovery/failure-recovery +--- + +Riak was built to withstand---or at the very least reduce the severity +of---many types of system failure. Nonetheless, bugs are a reality, +hardware does break, and occasionally Riak itself will fail. Here, we'll +list some steps that can be taken to minimize the harm caused by a general +cluster failure. + +## Forensics + +When a failure occurs, collect as much information as possible. Check +monitoring systems, backup log and configuration files if they are +available, including system logs like `dmesg` and `syslog`. Make sure +that the other nodes in the Riak cluster are still operating normally and +are not affected by a wider problem like a virtualization or network outage. +Try to determine the cause of the problem from the data you have collected. + +## Data Loss + +Many failures incur no data loss or minimal loss that can be +repaired automatically, without intervention. Outage of a single node +does not necessarily cause data loss, as other replicas of every key are +available elsewhere in the cluster. Once the node is detected as down, +other nodes in the cluster will take over its responsibilities +temporarily and transmit the updated data to it when it eventually +returns to service (also called [hinted handoff]({{<baseurl>}}riak/kv/2.2.6/learn/glossary/#hinted-handoff)). + +More severe data loss scenarios usually relate to hardware failure. +If data is lost, several options are available for restoring it. + +1. **Restore from backup** --- A daily backup of Riak nodes can be helpful. + The data in this backup may be stale depending on the time at which + the node failed, but it can be used to partially restore data from + lost storage volumes. If running in a RAID configuration, rebuilding + the array may also be possible. +2. **Restore from multi-cluster replication** --- If replication is enabled + between two or more clusters, the missing data will gradually be + restored via realtime replication and fullsync replication. A + fullsync operation can also be triggered manually via the `riak-repl` + command. +3. **Restore using intra-cluster repair** --- Riak versions 1.2 and greater + include a repair feature which will restore lost partitions with + data from other replicas. Currently, this must be invoked manually + using the Riak console and should be performed with guidance from a + Basho Client Services Engineer. + +Once data has been restored, normal operations should continue. If +multiple nodes completely lose their data, consultation and assistance +from Basho are strongly recommended. + +## Data Corruption + +Data at rest on disk can become corrupted by hardware failure or other +events. Generally, the Riak storage backends are designed to handle +cases of corruption in individual files or entries within files, and can +repair them automatically or simply ignore the corrupted parts. +Otherwise, clusters can recover from data corruption in roughly the same +way that they recover from data loss. + +## Out-of-Memory + +Sometimes, Riak will exit when it runs out of available RAM. While this +does not necessarily cause data loss, it may indicate that the cluster +needs to be scaled out. If free capacity is low on the rest of the cluster while the node is out, other nodes may also be at risk, so monitor carefully. + +Replacing the node with one that has greater RAM capacity may temporarily +alleviate the problem, but out-of-memory (OOM) issues tend to be an indication +that the cluster is under-provisioned. + +## High Latency / Request Timeout + +High latencies and timeouts can be caused by slow disks or networks or an +overloaded node. Check `iostat` and `vmstat` or your monitoring system to +determine the state of resource usage. If I/O utilization is high but +throughput is low, this may indicate that the node is responsible for +too much data and growing the cluster may be necessary. Additional RAM +may also improve latency because more of the active dataset will be +cached by the operating system. + +Sometimes extreme latency spikes can be caused by [sibling explosion]({{<baseurl>}}riak/kv/2.2.6/developing/usage/conflict-resolution#siblings). This condition occurs when the client application does not resolve conflicts properly or in a timely fashion. In that scenario, the size of the value on disk grows in proportion to +the number of siblings, causing longer disk service times and slower +network responses. + +Sibling explosion can be detected by examining the `node_get_fsm_siblings` +and `node_get_fsm_objsize` statistics from the `riak-admin status` command. +To recover from sibling explosion, the application should be throttled and +the resolution policy might need to be invoked manually on offending keys. + +A Basho CSE can assist in manually finding large values, i.e. those that +potentially have a sibling explosion problem, in the storage backend. + +MapReduce requests typically involve multiple I/O operations and are +thus the most likely to time out. From the perspective of the client +application, the success of MapReduce requests can be improved by reducing the +number of inputs, supplying a longer request timeout, and reducing the usage +of secondary indexes. Heavily loaded clusters may experience more MapReduce +timeouts simply because many other requests are being serviced as well. Adding +nodes to the cluster can reduce MapReduce failure in the long term by +spreading load and increasing available CPU and IOPS. + + +## Cluster Recovery From Backups + +See [Changing Cluster Information]({{<baseurl>}}riak/kv/2.1.4/using/cluster-operations/changing-cluster-info/#clusters-from-backups) for instructions on cluster recovery. + +{{% note title="Tip" %}} +If you are a TI Tokyo Riak supprt customer and require assistance or +further advice with a cluster recovery, please file a ticket with the +<a href="https://support.tiot.jp">TI Tokyo Helpdesk</a>. +{{% /note %}} diff --git a/content/riak/kv/2.2.6/using/repair-recovery/repairs.md b/content/riak/kv/2.2.6/using/repair-recovery/repairs.md new file mode 100644 index 0000000000..5aa04d56fa --- /dev/null +++ b/content/riak/kv/2.2.6/using/repair-recovery/repairs.md @@ -0,0 +1,387 @@ +--- +title: "Repairs" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Repairs" + identifier: "repair_recover_repairs" + weight: 102 + parent: "managing_repair_recover" +toc: true +aliases: + - /riak-docs/riak/2.2.6/ops/running/recovery/repairing-indexes + - /riak-docs/riak/kv/2.2.6/ops/running/recovery/repairing-indexes + - /riak-docs/riak/2.2.6/ops/running/recovery/failed-node + - /riak-docs/riak/kv/2.2.6/ops/running/recovery/failed-node + - /riak-docs/riak/2.2.6/ops/running/recovery/repairing-leveldb + - /riak-docs/riak/kv/2.2.6/ops/running/recovery/repairing-leveldb + - /riak-docs/riak/2.2.6/ops/running/recovery/repairing-partitions + - /riak-docs/riak/kv/2.2.6/ops/running/recovery/repairing-partitions +--- + +[cluster ops aae]: {{<baseurl>}}riak/kv/2.2.6/using/cluster-operations/active-anti-entropy/ +[config ref]: {{<baseurl>}}riak/kv/2.2.6/configuring/reference/ +[Erlang shell]: http://learnyousomeerlang.com/starting-out +[glossary AAE]: {{<baseurl>}}riak/kv/2.2.6/learn/glossary/#active-anti-entropy-aae +[glossary readrep]: {{<baseurl>}}riak/kv/2.2.6/learn/glossary/#read-repair +[search config]: {{<baseurl>}}riak/kv/2.2.6/configuring/search/#search-config-settings +[tiered storage]: {{<baseurl>}}riak/kv/2.2.6/setup/planning/backend/leveldb/#tiered-storage + + + +## Repairing Search Indexes + +Riak search indexes are repaired whenever objects are corrected by [read repair][glossary readrep]. + +[Active anti-entropy (AAE)][glossary AAE] is provided for Riak search. + +Riak KV's [configuration for AAE][cluster ops aae] will be used for Riak search's AAE hashtrees by default. + +Riak search can be provided its own AAE settings in the [search config settings][search config]. + +## Repairing Secondary Indexes + +The `riak-admin repair-2i` command can be used to repair any stale or missing secondary indexes. This command scans and repairs any mismatches between the secondary index data used for querying and the secondary index data stored in the Riak objects. It can be run on all partitions of a node or on a subset of them. We recommend scheduling these repairs outside of peak load time. + +### Running a Repair + +The secondary indexes of a single partition can be repaired by executing: + +```bash +riak-admin repair-2i »Partition ID« +``` + +The secondary indexes of every partition can be repaired by executing the same command, without a partition ID: + +```bash +riak-admin repair-2i +``` + +### Monitoring a Repair + +Repairs can be monitored using the below command: + +```bash +riak-admin repair-2i status +``` + +### Killing a Repair + +In the event the secondary index repair operation needs to be halted, all repairs can be killed with: + +```bash +riak-admin repair-2i kill +``` + +## Repairing LevelDB + +In the event of major hardware or filesystem problems, LevelDB can become corrupted. These failures are uncommon, but they could happen, as heavy loads can push I/O limits. + +### Checking for Compaction Errors + +Any time there is a compaction error, it will be noted in the LevelDB logs. Those logs are located in a `LOG` file in each instance of LevelDB in a Riak node, specifically in `#(platform_data_dir)/leveldb/<vnode>/LOG`. The `platform_data_dir` can be specified in the [`riak.conf`][config ref] configuration file. The default is `./data`. + +Compaction error messages take the following form: + +``` +<timestamp> Compaction Error: Corruption: corrupted compressed block contents +``` + +To check whether your node has experienced such errors, you will need to run a script that searches for `Compaction Error` in each `LOG` file. Here is an example script: + +```bash +find . -name "LOG" -exec grep -l 'Compaction error' {} \; +``` + +If there are compaction errors in any of your vnodes, those will be listed in the console. If any vnode has experienced such errors, you would see output like this: + +``` +./442446784738847563128068650529343492278651453440/LOG +``` + + +{{% note %}} +While corruption on one vnode is not uncommon, corruption in several vnodes very likely means that there is a deeper problem that needs to be address, perhaps on the OS or hardware level. +{{% /note %}} + + +## Healing Corrupted LevelDBs + +When you have discovered corruption in your LevelDB backend, the steps you take to resolve it will depend on whether you are using [tiered storage] or not. + +Choose your setup below: + +1. [Just LevelDB](#leveldb) +2. [LevelDB with tiered storage](#leveldb-with-tiered-storage) + + +### LevelDB + +Follow the steps below to heal your corrupted LevelDB. + +1\. Stop the node: + +```bash +riak stop +``` + +2\. To repair the corrupted LevelDB through the [Erlang shell], you will run the the `riak ertspath` command to output the path to Riak's internal Erlang runtime, and the `erl` command to start the Erlang shell. You can run them in a single command: + +```bash +`riak ertspath`/erl +``` + +{{% note title="Erlang version" %}} +Note, you must start up the Erlang shell using the same version of Erlang packaged with Riak. The above command will make sure you do so. If you choose not to use the above command please pay close attention to the version and location you use with the `erl` command. +{{% /note %}} + +3\. Once in the shell, run the following command: + +```erlang +application:set_env(eleveldb, data_root, ""). +``` + +4\. Then set `Options` equal to an empty list: + +```erlang +Options = []. +``` + +5\. Set some supportive variables for the repair process. These will be custom to your environment and specific repair needs. +VNodeList should be a list of each corrupted LevelDB that you found using the [`find` command above](#checking-for-compaction-errors). + +```erlang +DataRoot = "»path to your data root«". +VNodeList = ["»vnode id you want to repair«", ...]. +``` + +6\. Run the following commands, which will parse the information you provided and run eleveldb:repair over all of the VNode IDs that you listed in VNodeList. + +```erlang +RepairPath = fun(DataRoot, VNodeNumber) -> Path = lists:flatten(DataRoot ++ "/" ++ VNodeNumber), io:format("Repairing ~s.~n",[Path]), Path end. +[eleveldb:repair(RepairPath(DataRoot, VNodeList), Options) || VNodeNumber <- VNodeList]. +``` + +7\. This process may take several minutes. When it has completed successfully, you can restart the node and continue as usual. + +```bash +riak start +``` + +### LevelDB with Tiered Storage + +Follow the steps below to heal your corrupted LevelDB. + +1\. Stop the node: + +```bash +riak stop +``` + +2\. Check your riak.conf file and make note of the following values: + +* leveldb.tiered (integer) +* leveldb.tiered.path.fast +* leveldb.tiered.path.slow + +3\. To repair the corrupted LevelDB through the [Erlang shell], you will run the the `riak ertspath` command to output the path to Riak's internal Erlang runtime, and the `erl` command to start the Erlang shell. You can run them in a single command: + +```bash +`riak ertspath`/erl +``` + +{{% note title="Erlang version" %}} +Note, you must start up the Erlang shell using the same version of Erlang packaged with Riak. The above command will make sure you do so. If you choose not to use the above command please pay close attention to the version and location you use with the `erl` command. +{{% /note %}} + +4\. Once in the shell, run the following command: + +```erlang +application:set_env(eleveldb, data_root, ""). +``` + +5\. Then supply the information you noted in Step 2: + +```erlang +Options = [ + {tiered_slow_level, »leveldb.tiered value«}, + {tiered_fast_prefix, "»leveldb.tiered.path.fast value«"}, + {tiered_slow_prefix, "»leveldb.tiered.path.slow value«"} +]. +``` + +6\. Set some supportive variables for the repair process. These will be custom to your environment and specific repair needs. +VNodeList should be a list of each corrupted LevelDB partitions that you found using the [`find` command above](#checking-for-compaction-errors) provided in double quotes. + +```erlang +DataRoot = "»path to your data root«". +VNodeList = ["»vnode id you want to repair«", ...]. +``` + +7\. Run the following commands, which will parse the information you provided and run eleveldb:repair over all of the VNode IDs that you listed in VNodeList. + +```erlang +RepairPath = fun(DataRoot, VNodeNumber) -> Path = lists:flatten(DataRoot ++ "/" ++ VNodeNumber), io:format("Repairing ~s.~n",[Path]), Path end. +[eleveldb:repair(RepairPath(DataRoot, VNodeList), Options) || VNodeNumber <- VNodeList]. +``` +8\. This process may take several minutes. When it has completed successfully, you can restart the node and continue as usual. + +```bash +riak start +``` + + +## Repairing Partitions + +If you have experienced a loss of object replicas in your cluster, you +may need to perform a repair operation on one or more of your data +[partitions]({{<baseurl>}}riak/kv/2.2.6/learn/concepts/clusters/#the-ring). Repairs of Riak KV data are typically +run in situations where partitions or whole nodes are lost due to +corruption or hardware failure. In these cases, nodes or partitions are +brought back online without any data, which means that the need to +repair data will depend mainly on your use case and on whether [active anti-entropy]({{<baseurl>}}riak/kv/2.2.6/learn/concepts/active-anti-entropy/) is enabled. + +You will need to run a repair if the following are both true: + +* Active anti-entropy is [disabled]({{<baseurl>}}riak/kv/2.2.6/learn/concepts/active-anti-entropy/#disabling-active-anti-entropy) +* You have both non-expiring data and keys that are not accessed + frequently (which means that they are not likely to be subject to + [read repair]({{<baseurl>}}riak/kv/2.2.6/learn/concepts/active-anti-entropy/#read-repair-vs-active-anti-entropy)) + +You will most likely not need to run a repair operation if _any_ of the +following is true: + +* Active anti-entropy is [enabled]({{<baseurl>}}riak/kv/2.2.6/learn/concepts/active-anti-entropy/#enabling-active-anti-entropy) +* Your entire key set is accessed frequently, allowing passive read + repair to repair the partitions +* Your data expires frequently + +In most cases, we recommend either using active anti-entropy or, if +necessary and only when necessary, running a repair operation using the +instructions below. + +### Running a Repair + +The Riak KV repair operation will repair objects from a node's adjacent +partitions on the ring, consequently fixing the index. This is done as +efficiently as possible by generating a hash range for all the buckets +and thus avoiding a preflist calculation for each key. Only a hash of +each key is done, its range determined from a bucket->range map, and +then the hash is checked against the range. + +Repairs are not allowed to occur during ownership changes. Since +ownership entails the moving of partition data it is safest to make them +mutually exclusive events. If you join or remove a node all repairs +across the entire cluster will be killed. + +### Repairing a Single Partition + +In the case of data loss in a single partition, only that partition can +be repaired. + +1. From any node in the cluster, attach to Riak's Erlang shell: + + ```bash + riak attach + ``` + + You may have to hit **Enter** again to get a console prompt. + +2. Execute the repair for a single partition using the below command: + + ```erlang + riak_kv_vnode:repair(»Partition ID«). + ``` + + where `»Partition_ID«` is replaced by the ID of the partition to + repair. For example: + + ```erlang + riak_kv_vnode:repair(251195593916248939066258330623111144003363405824). + ``` + +3. Once the command has been executed, detach from Riak using +`Control-C`. + +### Repairing All Partitions on a Node + +If a node is lost, all partitions currently owned by that node can be +repaired. + +1. From any node in the cluster, attach to Riak's Erlang shell: + + ```bash + riak attach + ``` + +2. Get a copy of the current Ring: + + ```erlang + {ok, Ring} = riak_core_ring_manager:get_my_ring(). + ``` + + You will get a lot of output with ring record information. + You can safely ignore it. + +3. Get a list of partitions owned by the node that needs to be repaired. +Replace `dev1@127.0.0.1` with the name of the node to be repaired. The +name can be found in each node's `vm.args` file, specified as the +`-name` parameter, if you are using the older configuration system; if +you are using the newer, `riak-conf`-based system, the name is given by +the `nodename` parameter. + + ```erlang + Partitions = [P || {P, 'dev1@127.0.0.1'} <- riak_core_ring:all_owners(Ring)]. + ``` + + **Note**: The above is an [Erlang list + comprehension](http://www.erlang.org/doc/programming_examples/list_comprehensions.html) + that loops over each `{Partition, Node}` tuple in the ring and + extracts only the partitions that match the given node name, as a + list. + + +4. Execute the repair on all the partitions. Executing the repairs all +at once will cause a lot of `{shutdown, max_concurrency}` messages in +the logs. These can be safely ingored, as it is just the transfers +mechanism enforcing an upper limit on the number of concurrent +transfers. + + ```erlang + [riak_kv_vnode:repair(P) || P <- Partitions]. + ``` +5. Once the command has been executed, detach from Riak using +`Control-C`. + +### Monitoring Repairs + +The above repair commands can be monitored via the `riak-admin +transfers` command. + +### Killing a Repair + +Currently there is no easy way to kill an individual repair. The only +option is to kill all repairs targeting a given node. This is done by +running `riak_core_vnode_manager:kill_repairs(Reason)` on the node +undergoing repair. This command can be executed from a `riak attach` +session like below: + +```erlang +riak_core_vnode_manager:kill_repairs(killed_by_user). +``` + +Log entries will reflect that repairs were killed manually, and will +look similar to: + +``` +2012-08-10 10:14:50.529 [warning] <0.154.0>@riak_core_vnode_manager:handle_cast:395 Killing all repairs: killed_by_user +``` + +Repairs on a node can also be killed remotely from another node in the +cluster. From a `riak attach` session the below command can be used: + +```erlang +rpc:call('dev1@127.0.0.1', riak_core_vnode_manager, kill_repairs, [killed_by_user]). +``` diff --git a/content/riak/kv/2.2.6/using/repair-recovery/rolling-replaces.md b/content/riak/kv/2.2.6/using/repair-recovery/rolling-replaces.md new file mode 100644 index 0000000000..5ea115df7c --- /dev/null +++ b/content/riak/kv/2.2.6/using/repair-recovery/rolling-replaces.md @@ -0,0 +1,71 @@ +--- +title: "Rolling Replaces" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Rolling Replaces" + identifier: "repair_recover_replace" + weight: 106 + parent: "managing_repair_recover" +toc: true +--- + +[upgrade]: {{<baseurl>}}riak/kv/2.2.6/setup/upgrading/cluster/ +[rolling restarts]: {{<baseurl>}}riak/kv/2.2.6/using/repair-recovery/rolling-restart/ +[add node]: {{<baseurl>}}riak/kv/2.2.6/using/cluster-operations/adding-removing-nodes + +Riak KV functions as a multi-node system, so cluster-level [version upgrades][upgrade] and [restarts][rolling restarts] can be performed on a node-by-node or *rolling* basis. + +The following steps should be undertaken on each Riak KV node that you wish to replace: + +1\. Create a free node: + + a\. [Create an additional node][add node] with similar specifications to the other nodes in the cluster. + + b\. Or leave a node that is currently in the cluster: + + ```bash + riak-admin cluster leave »nodename« + ``` + + After creating a node or leaving a node, wait for all transfers to complete: + + ```bash + riak-admin transfers + ``` + +2\. Join the free node to your cluster: + +```bash +riak-admin cluster join »free_node« +``` + +3\. Next, replace the free node with an existing node: + +```bash +riak-admin cluster replace »free_node« »nodename« +``` + +4\. Then review the cluster transition plan: + +```bash +riak-admin cluster plan +``` + +5\. And commit the changes: + +```bash +riak-admin cluster commit +``` + +6\. Wait for all transfers to complete: + +```bash +riak-admin transfers +``` + +7\. Repeat steps 2-6 above until each node has been replaced. + +8\. Join the replaced node back into the cluster or decommission the additional node that was created. diff --git a/content/riak/kv/2.2.6/using/repair-recovery/rolling-restart.md b/content/riak/kv/2.2.6/using/repair-recovery/rolling-restart.md new file mode 100644 index 0000000000..6f7a58f49d --- /dev/null +++ b/content/riak/kv/2.2.6/using/repair-recovery/rolling-restart.md @@ -0,0 +1,60 @@ +--- +title: "Rolling Restarts" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Rolling Restarts" + identifier: "repair_recover_restart" + weight: 103 + parent: "managing_repair_recover" +toc: true +aliases: + - /riak-docs/riak/2.2.6/ops/running/recovery/rolling-restart + - /riak-docs/riak/kv/2.2.6/ops/running/recovery/rolling-restart +--- + +Because Riak functions as a multi-node system, cluster-level [Riak version upgrades]({{<baseurl>}}riak/kv/2.2.6/setup/upgrading/cluster) and restarts can be performed on a node-by-node, "rolling" basis. + +The following steps should be undertaken on each Riak node that you wish to restart: + +1\. Stop Riak + +```bash +riak stop +``` + +2\. Perform any necessary maintenance, upgrade, or other work in your cluster. + +3\. Start Riak again + +```bash +riak start +``` + +4\. Verify that the `riak_kv` service is once again available on the target node + +```bash +riak-admin wait-for-service riak_kv <nodename> +``` + +If this responds with `riak_kv is up`, then the service is available and you can move on to the next step. Otherwise, the console will periodically return `riak_kv is not up` until the service is available. + +5\. Verify that all in-progress handoffs have been completed + +```bash +riak-admin transfers +``` + +If this responds with `No transfers active`, then all handoffs are complete. You can either run this command periodically until no more transfers are active or run the following script, which will run the `riak-admin transfers` command every 5 seconds until the transfers are complete: + +```bash +while ! riak-admin transfers | grep -iqF 'No transfers active' +do + echo 'Transfers in progress' + sleep 5 +done +``` + +6\. Repeat the above process for any other nodes that need to be restarted. diff --git a/content/riak/kv/2.2.6/using/repair-recovery/secondary-indexes.md b/content/riak/kv/2.2.6/using/repair-recovery/secondary-indexes.md new file mode 100644 index 0000000000..25cefe4e19 --- /dev/null +++ b/content/riak/kv/2.2.6/using/repair-recovery/secondary-indexes.md @@ -0,0 +1,138 @@ +--- +title: "Repairing Secondary Indexes" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Repair Secondary Indexes" + identifier: "repair_recover_2i" + weight: 105 + parent: "managing_repair_recover" +toc: true +aliases: + - /riak-docs/riak/2.2.6/ops/running/recovery/repairing-indexes + - /riak-docs/riak/kv/2.2.6/ops/running/recovery/repairing-indexes +--- + +The `riak-admin repair-2i` command can be used to repair any stale or missing secondary indexes. This command scans and repairs any mismatches between the secondary index data used for querying and the secondary index data stored in the Riak objects. It can be run on all partitions of a node or on a subset of them. We recommend scheduling these repairs outside of peak load time. + +### Running a Repair + +The secondary indexes of a single partition can be repaired by executing: + +```bash +riak-admin repair-2i <Partition_ID> +``` + +The secondary indexes of every partition can be repaired by executing the same command, without a partition ID: + +```bash +riak-admin repair-2i +``` + +### Monitoring a Repair + +Repairs can be monitored using the below command: + +```bash +riak-admin repair-2i status +``` + +### Killing a Repair + +In the event the secondary index repair operation needs to be halted, all repairs can be killed with: + +```bash +riak-admin repair-2i kill +``` + +---- + +## Repairing Search Indexes + +Riak Search indexes currently have no form of anti-entropy (such as read-repair). Furthermore, for performance and load balancing reasons, Search reads from one random node. This means that when a replica loss has occurred, inconsistent results may be returned. + +### Running a Repair + +If a replica loss has occurred, you need to run the repair command. This command repairs objects from a node's adjacent partitions on the ring, consequently fixing the search index. + +This is done as efficiently as possible by generating a hash range for all the buckets and thus avoiding a preflist calculation for each key. Only a hash of each key is done, its range determined from a bucket→range map, and then the hash is checked against the range. + +This code will force all keys in each partition on a node to be reread, thus rebuilding the search index properly. + +1. From a cluster node with Riak installed, attach to the Riak console: + + ```bash + riak attach + ``` + + You may have to hit enter again to get a console prompt. + +2. Get a list of partitions owned by the node that needs repair: + + ```erlang + {ok, Ring} = riak_core_ring_manager:get_my_ring(). + ``` + + You will get a lot of output with Ring record information. You can safely ignore it. + +3. Then run the following code to get a list of partitions. Replace 'dev1@127.0.0.1' with the name of the node you need to repair. + + ```erlang + Partitions = [P || {P, 'dev1@127.0.0.1'} <- riak_core_ring:all_owners(Ring)]. + ``` + + _Note: The above is an [Erlang list comprehension](http://www.erlang.org/doc/programming_examples/list_comprehensions.html), that loops over each `{Partition, Node}` tuple in the Ring, and extracts only the partitions that match the given node name, as a list._ + +4. Execute repair on all the partitions. Executing them all at once like this will cause a lot of `{shutdown,max_concurrency}` spam but it's not anything to worry about. That is just the transfers mechanism enforcing an upper limit on the number of concurrent transactions. + + ```erlang + [riak_search_vnode:repair(P) || P <- Partitions]. + ``` + +5. When you're done, press `Ctrl-D` to disconnect the console. DO NOT RUN q() which will cause the running Riak node to quit. Note that `Ctrl-D` merely disconnects the console from the service, it does not stop the code from running. + + +### Monitoring a Repair + +The above Repair command can be slow, so if you reattach to the console, you can run the repair_status function. You can use the `Partitions` variable defined above to get the status of every partition. + +```erlang +[{P, riak_search_vnode:repair_status(P)} || P <- Partitions]. +``` + +When you're done, press `Ctrl-D` to disconnect the console. + +### Killing a Repair + +Currently there is no easy way to kill an individual repair. The only +option is to kill all repairs targeting a given node. This is done by +running `riak_core_vnode_manager:kill_repairs(Reason)` on the node +undergoing repair. This means you'll either have to be attached to +that node's console or you can use the `rpc` module to make a remote +call. Here is an example of killing all repairs targeting partitions +on the local node. + +```erlang +riak_core_vnode_manager:kill_repairs(killed_by_user). +``` + +Log entries will reflect that repairs were killed manually, something akin to this: + +``` +2012-08-10 10:14:50.529 [warning] <0.154.0>@riak_core_vnode_manager:handle_cast:395 Killing all repairs: killed_by_user +``` + +Here is an example of executing the call remotely. + +```erlang +rpc:call('dev1@127.0.0.1', riak_core_vnode_manager, kill_repairs, [killed_by_user]). +``` + +When you're done, press `Ctrl-D` to disconnect the console. + +Repairs are not allowed to occur during ownership changes. Since +ownership entails the moving of partition data it is safest to make +them mutually exclusive events. If you join or remove a node all +repairs across the entire cluster will be killed. diff --git a/content/riak/kv/2.2.6/using/running-a-cluster.md b/content/riak/kv/2.2.6/using/running-a-cluster.md new file mode 100644 index 0000000000..1e48b317e8 --- /dev/null +++ b/content/riak/kv/2.2.6/using/running-a-cluster.md @@ -0,0 +1,335 @@ +--- +title: "Running a Cluster" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Running a Cluster" + identifier: "managing_running_a_cluster" + weight: 200 + parent: "managing" +toc: true +aliases: + - /riak-docs/riak/2.2.6/ops/building/basic-cluster-setup + - /riak-docs/riak/kv/2.2.6/ops/building/basic-cluster-setup +--- + +Configuring a Riak cluster involves instructing each node to listen on a +non-local interface, i.e. not `127.0.0.1`, and then joining all of the +nodes together to participate in the cluster. + +Most configuration changes will be applied to the [configuration file]({{<baseurl>}}riak/kv/2.2.6/configuring/reference/) located in your `rel/riak/etc` directory (if +you compiled from source) or `/etc` (if you used a binary install of +Riak). + +The commands below presume that you are running from a source install, +but if you have installed Riak with a binary install, you can substitute +the usage of `bin/riak` with `sudo /usr/sbin/riak` and `bin/riak-admin` +with `sudo /usr/sbin/riak-admin`. The `riak` and `riak-admin` scripts +are located in the `/bin` directory of your installation. + +> **Note on changing the `name` value** +> +> If possible, you should avoid starting Riak prior to editing the name of +a node. This setting corresponds to the `nodename` parameter in the +`riak.conf` file if you are using the newer configuration system, and to +the `-name` parameter in `vm.args` (as described below) if you are using +the older configuration system. If you have already started Riak with +the default settings, you cannot change the `-name` setting and then +successfully restart the node. +> +> If you cannot restart after changing the `-name` value you have two +options: +> +> * Discard the existing ring metadata by removing the contents of the +`ring` directory. This will require rejoining all nodes into a +cluster again. +> +> *Rename the node using the [`riak-admin cluster replace`]({{<baseurl>}}riak/kv/2.2.6/using/admin/riak-admin/#cluster-replace) command. This will not work if you have previously only started Riak with a single node. + +## Configure the First Node + +First, stop your Riak node if it is currently running: + +```bash +riak stop +``` + +#### Select an IP address and port + +Let's say that the IP address for your cluster is 192.168.1.10 and that +you'll be using the default port (8087). If you're using the [Protocol Buffers interface]({{<baseurl>}}riak/kv/2.2.6/developing/api/protocol-buffers/) to Riak (which we recommend over the HTTP +interface due to performance gains), you should change your +configuration file: + +```riakconf +listener.protobuf.internal = 127.0.0.1:8087 +``` + +```appconfig +%% In the pb section of riak_core: + +{"127.0.0.1", 8087 }, +``` + +becomes + +```riakconf +listener.protobuf.internal = 192.168.1.10:8087 +``` + +```appconfig +%% In the pb section of riak_core: + +{"192.168.1.10", 8087 }, +``` + +{{% note title="Note on upgrading to 2.0" %}} +If you are upgrading to Riak version 2.0 or later from an pre-2.0 +release, you can use either your old `app.config`/ `vm.args` +configuration files or the newer `riak.conf` if you wish. If you have +installed Riak 2.0 directly, you should use only `riak.conf`. + +Below, examples will be provided for both the old and new configuration +systems. Bear in mind that you need to use either the older or the newer +but never both simultaneously. + +More on configuring Riak can be found in the [Configuration documentation](../../configuring/reference). +{{% /note %}} + +If you're using the HTTP interface, you will need to alter your +configuration in an analogous way: + +```riakconf +listener.http.internal = 127.0.0.1:8098 +``` + +```appconfig +%% In the riak_core section: + +{http, [ {"127.0.0.1", 8098 } ]}, +``` + +becomes + +```riakconf +listener.http.internal = 192.168.1.10:8098 +``` + +```appconfig +{http, [ {"192.168.1.10", 8098 } ]}, +``` + +#### Name your node + +Every node in Riak has a name associated with it. The default name is +`riak@127.0.0.1`. Let's say that you want to change the name to +`riak@192.168.1.10`: + +```riakconf +nodename = riak@127.0.0.1 +``` + +```vmargs +-name riak@127.0.0.1 +``` + +becomes + +```riakconf +nodename = riak@192.168.1.10 +``` + +```vmargs +-name riak@192.168.1.10 +``` + +> **Node Names** +> +> Use fully qualified domain names ([FQDNs](http://en.wikipedia.org/wiki/Fully_qualified_domain_name)) rather than IP addresses for the cluster member node names. For example, `riak@cluster.example.com` and `riak@192.168.1.10` +are both acceptable node naming schemes, but using the FQDN style is +preferred. +> +> Once a node has been started, in order to change the name you must +either remove ring files from the `/data/ring` directory or +[`riak-admin cluster force-replace`]({{<baseurl>}}riak/kv/2.2.6/using/admin/riak-admin/#cluster-force-replace) the node. + +#### Start the node + +Now that your node is properly configured, you can start it: + +```bash +riak start +``` + +If the Riak node has been previously started, you must use the +`riak-admin cluster replace` command to change the node name and update +the node's ring file. + +```bash +riak-admin cluster replace riak@127.0.0.1 riak@192.168.1.10 +``` + +{{% note title="Note on single nodes" %}} +If a node is started singly using default settings, as you might do when you +are building your first test environment, you will need to remove the ring +files from the data directory after you edit your configuration files. +`riak-admin cluster replace` will not work since the node has not been joined +to a cluster. +{{% /note %}} + +As with all cluster changes, you need to view the planned changes by +running `riak-admin cluster plan` and then running `riak-admin cluster +commit` to finalize those changes. + +The node is now properly set up to join other nodes for cluster +participation. You can proceed to adding a second node to the cluster. + +## Add a Second Node to Your Cluster + +Repeat the above steps for a second host on the same network, providing +the second node with a host/port and node name. Once the second node has +started, use `riak-admin cluster join` to join the second node to the +first node, thereby creating an initial Riak cluster. Let's say that +we've named our second node `riak@192.168.1.11`. From the new node's +`/bin` directory: + +```bash +riak-admin cluster join riak@192.168.1.10 +``` + +Output from the above should resemble: + +``` +Success: staged join request for `riak@192.168.1.11` to `riak@192.168.1.10` +``` + +Next, plan and commit the changes: + +```bash +riak-admin cluster plan +riak-admin cluster commit +``` + +After the last command, you should see: + +``` +Cluster changes committed +``` + +If your output was similar, then the second Riak node is now part of the +cluster and has begun syncing with the first node. Riak provides several +ways to determine the cluster's ring status. Here are two ways to +examine your Riak cluster's ring: + +1. Using the `riak-admin` command: + + ```bash + bin/riak-admin status | grep ring_members + ``` + + With output resembling the following: + + ```bash + ring_members : ['riak@192.168.1.10','riak@192.168.1.11'] + ``` + +2. Running the `riak attach` command. This will open up an Erlang shell, +into which you can type the following command: + + ```erlang + 1> {ok, R} = riak_core_ring_manager:get_my_ring(). + + %% Response: + + {ok,{chstate,'riak@192.168.1.10',......... + (riak@192.168.52.129)2> riak_core_ring:all_members(R). + ['riak@192.168.1.10','riak@192.168.1.11'] + ``` + +To join additional nodes to your cluster, repeat the above steps. You +can also find more detailed instructions about [adding and removing nodes]({{<baseurl>}}riak/kv/2.2.6/using/cluster-operations/adding-removing-nodes) from a cluster. + +> **Ring Creation Size** +> +> All nodes in the cluster +must have the same initial ring size setting in order to join, and +participate in cluster activity. This setting can be adjusted in your +configuration file using the `ring_creation_size` parameter if you're +using the older configuration system or `ring_size` in the new system. +> +> Check the value of all nodes if you receive a message like this: +> `Failed: riak@10.0.1.156 has a different ring_creation_size` + +## Running Multiple Nodes on One Host + +If you built Riak from source code, or if you are using the Mac OS X +pre-built package, then you can easily run multiple Riak nodes on the +same machine. The most common scenario for doing this is to experiment +with running a Riak cluster. + +**Note**: If you have installed the `.deb` or `.rpm` package, then you +will need to download and build Riak from source to follow the +directions below. + +To run multiple nodes, make copies of the `riak` directory. + +- If you ran `make all rel`, then this can be found in `./rel/riak` + under the Riak source root directory. +- If you are running Mac OS X, then this is the directory where you + unzipped the `.tar.gz` file. + +Presuming that you copied `./rel/riak` into `./rel/riak1`, `./rel/riak2`, +`./rel/riak3`, and so on, you need to make two changes: + +1. Set your handoff port and your Protocol Buffers or HTTP port +(depending on which interface you are using) to different values on each +node. For example: + + ```riakconf + # For Protocol Buffers: + listener.protobuf.internal = 127.0.0.1:8187 + + # For HTTP: + listener.http.internal = 127.0.0.1:8198 + + # For either interface: + handoff.port = 8199 + ``` + + ```appconfig + %% In the pb section of riak_core: + {"127.0.0.1", 8187 } + + %% In the http section of riak_core: + {"127.0.0.1", 8198} + ``` + +2. Change the name of each node to a unique name. Now, start the nodes, +changing path names and nodes as appropriate: + +```bash +./rel/riak1/bin/riak start +./rel/riak2/bin/riak start +./rel/riak3/bin/riak start + +# etc +``` + +Next, join the nodes into a cluster: + +```bash +./rel/riak2/bin/riak-admin cluster join riak1@127.0.0.1 +./rel/riak3/bin/riak-admin cluster join riak1@127.0.0.1 +./rel/riak2/bin/riak-admin cluster plan +./rel/riak2/bin/riak-admin cluster commit +``` + +## Multiple Clusters on One Host + +Using the above technique, it is possible to run multiple clusters on +one computer. If a node hasn’t joined an existing cluster, it will +behave just as a cluster would. Running multiple clusters on one +computer is simply a matter of having two or more distinct nodes or +groups of clustered nodes. diff --git a/content/riak/kv/2.2.6/using/security.md b/content/riak/kv/2.2.6/using/security.md new file mode 100644 index 0000000000..0c46dbefb4 --- /dev/null +++ b/content/riak/kv/2.2.6/using/security.md @@ -0,0 +1,195 @@ +--- +title: "Security & Firewalls" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Security" + identifier: "managing_security" + weight: 205 + parent: "managing" +toc: true +aliases: + - /riak-docs/riak/2.2.6/ops/advanced/security + - /riak-docs/riak/kv/2.2.6/ops/advanced/security +--- + +[config reference search]: {{<baseurl>}}riak/kv/2.1.4/configuring/reference/#search +[config search enabling]: {{<baseurl>}}riak/kv/2.1.4/configuring/search/#enabling-riak-search +[config v3 ssl]: {{<baseurl>}}riak/kv/2.1.4/configuring/v3-multi-datacenter/ssl +[JMX]: http://www.oracle.com/technetwork/java/javase/tech/javamanagement-140525.html +[security basics]: {{<baseurl>}}riak/kv/2.1.4/using/security/basics +[security managing]: {{<baseurl>}}riak/kv/2.1.4/using/security/managing-sources/ +[Solr]: http://lucene.apache.org/solr/ +[usage search]: {{<baseurl>}}riak/kv/2.1.4/developing/usage/search + +> **Internal security** +> +> This document covers network-level security. For documentation on the +authentication and authorization features introduced in Riak 2.0, see +[Authentication and Authorization][security basics] and [Managing Security Sources][security managing] + +This article discusses standard configurations and port settings to use +when providing network security for a Riak Cluster. There are two +classes of access control for Riak: + +* Other Riak nodes participating in the cluster +* Clients making use of the Riak cluster + +The settings for both access groups are located in your cluster's +configuration settings. If you are using the newer configuration system, +you can set a host and port for each node in that node's `riak.conf` +file, setting `listener.protobuf` if you are using Riak's Protocol +Buffers interface or `listener.http` if you are using HTTP (or +`listener.https` if you are using SSL). If you are using the older +configuration system, adjust the settings of `pb`, `http`, or `https`, +depending on which client interface you are using. + +Make note of these configurations and set up your firewall to allow +incoming TCP access to those ports or IP address/port combinations. +Exceptions to this are the `handoff_ip` and `handoff_port` directives. +Those are for communication between Riak nodes only. + +## Inter-node Communication + +Riak uses the Erlang distribution mechanism for most inter-node +communication. Riak identifies other machines in the ring using Erlang +identifiers (`<hostname or IP>`, e.g. `riak@10.9.8.7`). Erlang resolves +these node identifiers to a TCP port on a given machine via the Erlang +Port Mapper daemon (epmd) running on each cluster node. + +By default, epmd binds to TCP port 4369 and listens on the wildcard +interface. For inter-node communication, Erlang uses an unpredictable +port by default; it binds to port 0, which means the first available +port. + +For ease of firewall configuration, Riak can be configured +to instruct the Erlang interpreter to use a limited range +of ports. For example, to restrict the range of ports that Erlang will +use for inter-Erlang node communication to 6000-7999, add the following +lines to the configuration file on each Riak node: + +```riakconf +erlang.distribution.port_range.minimum = 6000 +erlang.distribution.port_range.maximum = 7999 +``` + +```appconfig +{ kernel, [ + {inet_dist_listen_min, 6000}, + {inet_dist_listen_max, 7999} + ]}, +``` + +The above lines should be added into the top level list in app.config, +at the same level as all the other applications (e.g. `riak_core`). +Then configure your firewall to allow incoming access to TCP ports 6000 +through 7999 from whichever network(s) contain your Riak nodes. + +### Riak Node Ports + +Riak nodes in a cluster need to be able to communicate freely with one +another on the following ports: + +* epmd listener: TCP:4369 +* handoff_port listener: TCP:8099 +* range of ports specified in `app.config` or `riak.conf` + +### Riak Client Ports + +Riak clients must be able to contact at least one machine in a Riak +cluster on the following TCP ports: + +Protocol | Port +:--------|:---- +<a href="../../developing/api/http">HTTP</a> | TCP port 8098 +<a href="../../developing/api/protocol-buffers">Protocol Buffers</a> | TCP port 8087 + +### Riak Search Ports + +Riak's [search][usage search] feature relies on [Apache Solr][Solr], which runs +on each Riak node if security has been [enabled][config search enabling]. When +Riak's Search subsystem starts up, [JMX][JMX] opens a well-known port as well +as some ephemeral ports. The well-known port is determined by the value of the +`search.solr.jmx_port` in each node's [Search configuration][config reference search]. +The default is 8985. + +In addition to JMX ports, Solr also binds to a well-known port of its +own, as determined by each node's `search.solr.port` setting, which is +also located in each node's Search configuration. The default is 8093. + +# Riak Security Community + +## Riak + +Riak is a powerful open-source distributed database focused on scaling +predictably and easily, while remaining highly available in the face of +server crashes, network partitions or other (inevitable) disasters. + +## Commitment + +Data security is an important and sensitive issue to many of our users. +A real-world approach to security allows us to balance appropriate +levels of security and related overhead while creating a fast, scalable, +and operationally straightforward database. + +### Continuous Improvement + +Though we make every effort to thwart security vulnerabilities whenever +possible (including through independent reviews), no system is +completely secure. We will never claim that Riak is 100% secure (and you +should seriously doubt anyone who claims their solution is). What we can +promise is that we openly accept all vulnerabilities from the community. +When appropriate, we'll publish and make every attempt to quickly +address these concerns. + +### Balance + +More layers of security increase operational and administrative costs. +Sometimes those costs are warranted, sometimes they are not. Our +approach is to strike an appropriate balance between effort, cost, and +security. + +For example, Riak does not have fine-grained role-base security. Though +it can be an attractive bullet-point in a database comparison chart, +you're usually better off finely controlling data access through your +application or a service layer. + +### Notifying Basho + +If you discover a potential security issue, please email us at +**security@basho.com**, and allow us 48 hours to reply. + +We prefer to be contacted first, rather than searching for blog posts +over the Internet. This allows us to open a dialogue with the security +community on how best to handle a possible exploit without putting any +users at risk. + +## Security Best Practices + +### Authentication and Authorization + +For instructions on how to apply permissions and to require client +authentication, please see our documentation on [Riak Security][security basics]. + +### Network Configurations + +Being a distributed database means that much of Riak's security springs +from how you configure your network. We have a few recommendations for +[Security and Firewalls][security basics]. + +### Client Auth + +All of the Riak client libraries support encrypted TCP communication +as well as authentication and authorization. For instructions on how +to apply permissions and to require client authentication, please see +our documentation on [Riak Security][security basics]. + +### Multi-Datacenter Replication + +For those versions of Riak that support Multi Data Center (MDC) +Replication, you can configure Riak 1.2+ to communicate over SSL, to +seamlessly encrypt the message traffic. + +See also: [Multi Data Center Replication: SSL][config v3 ssl] diff --git a/content/riak/kv/2.2.6/using/security/basics.md b/content/riak/kv/2.2.6/using/security/basics.md new file mode 100644 index 0000000000..52d6fc4564 --- /dev/null +++ b/content/riak/kv/2.2.6/using/security/basics.md @@ -0,0 +1,847 @@ +--- +title: "Security Basics" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Security Basics" + identifier: "security_basics" + weight: 100 + parent: "managing_security" +toc: true +aliases: + - /riak-docs/riak/2.2.6/ops/running/authz + - /riak-docs/riak/kv/2.2.6/ops/running/authz +--- + +> **Note on Network security** +> +> This document covers only the 2.0 authentication and authorization +features. For a look at network security in Riak, see [Security and Firewalls]({{<baseurl>}}riak/kv/2.2.6/using/security/managing-sources/). + +As of version 2.0, Riak administrators can selectively apportion +access to a wide variety of Riak's functionality, including accessing, +modifying, and deleting objects, changing bucket properties, and +running MapReduce jobs. + +## Terminology + +* **Authentication** is the process of identifying a user. +* **Authorization** is verifying whether a user has access to perform + the requested operation. +* **Groups** can have permissions assigned to them, but cannot be + authenticated. +* **Users** can be authenticated and authorized; permissions + (authorization) may be granted directly or via group membership. +* **Sources** are used to define authentication mechanisms. A user + cannot be authenticated to Riak until a source is defined. + +## Security Checklist + +There are a few key steps that all applications will need to undertake +when turning on Riak security. Missing one of these steps will almost +certainly break your application, so make sure that you have done each +of the following **before** enabling security: + +1. Make certain that the original Riak Search (version 1) and link + walking are not required. Enabling security will break this + functionality. If you wish to use security and Search together, you + will need to use the [new Search feature]({{<baseurl>}}riak/kv/2.2.6/developing/usage/search/). +1. Because Riak security requires a secure SSL connection, you will need + to generate appropriate SSL certs, [enable SSL](#enabling-ssl) and establish a [certificate configuration](#certificate-configuration) on each node. **If you + enable security without having established a functioning SSL + connection, all requests to Riak will fail**. +1. Define [users](#user-management) + and, optionally, [groups](#add-group) +1. Define an [authentication source](#managing-sources) for each user +1. Grant the necessary [permissions](#managing-permissions) to each user (and/or group) +1. Check any Erlang MapReduce code for invocations of Riak modules other + than `riak_kv_mapreduce`. Enabling security will prevent those from + succeeding unless those modules are available via the `add_path` + mechanism documented in [Installing Custom Code]({{<baseurl>}}riak/kv/2.2.6/using/reference/custom-code). +1. Make sure that your client software will work properly: + * It must pass authentication information with each request + * It must support HTTPS or encrypted [Protocol Buffers]({{<baseurl>}}riak/kv/2.2.6/developing/api/protocol-buffers/) + traffic + * If using HTTPS, the proper port (presumably 443) is open from + client to server + * Code that uses Riak's deprecated link walking feature **will + not work** with security enabled +1. If you have applications that rely on an already existing Riak + cluster, make sure that those applications are prepared to gracefully + transition into using Riak security once security is enabled. + +Security should be enabled only after all of the above steps have been +performed and your security setup has been properly vetted. + +Clients that use [Protocol Buffers]({{<baseurl>}}riak/kv/2.2.6/developing/api/protocol-buffers/) will typically have to be +reconfigured/restarted with the proper credentials once security is +enabled. + +## Security Basics + +Riak security may be checked, enabled, or disabled by an administrator +through the command line. This allows an administrator to change +security settings for the whole cluster quickly without needing to +change settings on a node-by-node basis. + +**Note**: Currently, Riak security commands can be run only through +the command line, using the `riak-admin security` command. In future +versions of Riak, administrators may have the option of issuing +those commands through the Protocol Buffers and HTTP interfaces. + +### Enabling Security + +> **Warning: Enable security with caution** +> +> Enabling security will change the way your client libraries and +your applications interact with Riak. +> +> Once security is enabled, all client connections must be encrypted and all permissions will be denied by default. Do not enable this in production until you have worked through the [security checklist](#security-checklist) above and tested everything in a non-production environment. + +Riak security is disabled by default. To enable it: + +```bash +riak-admin security enable +``` + +**As per the warning above, do not enable security in production without +taking the appropriate precautions.** + +All users, groups, authentication sources, and permissions can be +configured while security is disabled, allowing you to create a +security configuration of any level of complexity without prematurely +impacting the service. This should be borne in mind when you are +[managing users](#user-management) and [managing sources](#managing-sources). + +### Disabling Security + +If you disable security, this means that you have disabled all of the +various permissions checks that take place when executing operations +against Riak. Users, groups, and other security attributes remain +available for configuration while security is disabled, and will be +applied if and when security is re-enabled. + +```bash +riak-admin security disable +``` + +While security is disabled, clients will need to be reconfigured to no +longer require TLS and send credentials. + +### Checking Security Status + +To check whether security is currently enabled for the cluster, use the +`status` command: + +```bash +riak-admin security status +``` + +This command will usually return `Enabled` or `Disabled`, but if +security is enabled on a mixed-mode cluster (running a combination of +Riak 2.0 and older versions) it will indicate that security is enabled +but not yet available. + +## User Management + +Riak security enables you to control _authorization_ by creating, +modifying, and deleting user characteristics and granting users +selective access to Riak functionality (and also to revoke access). +Users can be assigned one or more of the following characteristics: + +* `username` +* `groups` +* `password` + +You may also assign users characteristics beyond those listed +above---e.g., listing email addresses or other information---but those +values will carry no special significance for Riak. + +**Note**: The `username` is the one user characteristic that cannot be +changed once a user has been created. + +### Retrieve a Current User or Group List + +A list of currently existing users can be accessed at any time: + +```bash +riak-admin security print-users +``` + +The same goes for groups: + +```bash +riak-admin security print-groups +``` + +Example output, assuming user named `riakuser` with an assigned +password: + +``` ++----------+--------+----------------------+------------------------------+ +| username | groups | password | options | ++----------+--------+----------------------+------------------------------+ +| riakuser | |983e8ae1421574b8733824| [] | ++----------+--------+----------------------+------------------------------+ +``` + +**Note**: All passwords are displayed in encrypted form in console +output. + +If the user `riakuser` were assigned to the group `dev` and a `name` of +`lucius`, the output would look like this: + +```bash ++----------+----------------+----------------------+---------------------+ +| username | groups | password | options | ++----------+----------------+----------------------+---------------------+ +| riakuser | dev |983e8ae1421574b8733824| [{"name","lucius"}] | ++----------+----------------+----------------------+---------------------+ +``` + +If you'd like to see which permissions have been assigned to +`riakuser`, you would need to use the `print-grants` command, detailed +below. + +The `security print-user` or `security-print-group` (singular) commands +can be used with a name as argument to see the same information as +above, except for only that user or group. + +### Permissions Grants For a Single User or Group + +You can retrieve authorization information about a specific user or +group using the `print-grants` command, which takes the form of +`riak-admin security print-grants <username>`. + +The output will look like this if the user `riakuser` has been +explicitly granted a `riak_kv.get` permission on the bucket +`shopping_list` and inherits a set of permissions from the `admin` +group: + +```bash +Inherited permissions (user/riakuser) + ++--------+----------+----------+----------------------------------------+ +| group | type | bucket | grants | ++--------+----------+----------+----------------------------------------+ +| admin | * | * | riak_kv.get, riak_kv.delete, | +| | | | riak_kv.put | ++--------+----------+----------+----------------------------------------+ + +Dedicated permissions (user/riakuser) + ++----------+-------------+----------------------------------------+ +| type | bucket | grants | ++----------+-------------+----------------------------------------+ +| ANY |shopping_list| riak_kv.get | ++----------+-------------+----------------------------------------+ + +Cumulative permissions (user/riakuser) + ++----------+-------------+----------------------------------------+ +| type | bucket | grants | ++----------+-------------+----------------------------------------+ +| * | * | riak_kv.get, riak_kv.delete, | +| | | riak_kv.put | +| ANY |shopping_list| riak_kv.get | ++----------+-------------+----------------------------------------+ +``` + +**Note**: The term `admin` is not a reserved term in Riak security. It +is used here only for illustrative purposes. + +Because the same name can represent both a user and a group, a prefix +(`user/` or `group/`) can be used before the name (e.g., `print-grants +user/admin`). If a name collides and no prefix is supplied, grants for +both will be listed separately. + +### Add Group + +For easier management of permissions across several users, it is +possible to create groups to be assigned to those users. + +```bash +riak-admin security add-group admin +``` + +### Add User + +To create a user with the username `riakuser`, we use the `add-user` +command: + +```bash +riak-admin security add-user riakuser +``` + +Using the command this way will create the user `riakuser` without _any_ +characteristics beyond a username, which is the only attribute that you +must assign upon user creation. + +Alternatively, a password---or other attributes---can be assigned to the +user upon creation. Here, we'll assign a password: + +```bash +riak-admin security add-user riakuser password=Test1234 +``` + +### Assigning a Password and Altering Existing User Characteristics + +While passwords and other characteristics can be set upon user creation, +it often makes sense to change user characteristics after the user has +already been created. Let's say that the user `riakuser` was created +without a password (or created _with_ a password that we'd like to +change). The `alter-user` command can be used to modify our `riakuser` +user: + +```bash +riak-admin security alter-user riakuser password=opensesame +``` + +When creating or altering a user, any number of `<option>=<value>` +pairs can be appended to the end of the command. Any non-standard +options will be stored and displayed via the `riak-admin security +print-users` command. + +```bash +riak-admin security alter-user riakuser name=bill age=47 fav_color=red +``` + +Now, the `print-users` command should return this: + +``` ++----------+--------+----------+--------------------------------------------------+ +| username | groups | password | options | ++----------+--------+----------+--------------------------------------------------+ +| riakuser | | |[{"fav_color","red"},{"age","47"},{"name","bill"}]| ++----------+--------+----------+--------------------------------------------------+ +``` + +**Note**: Usernames _cannot_ be changed using the `alter-user` command. +For example, running `riak-admin security alter-user riakuser +username=other-name`, will instead add the +`{"username","other-name"}` tuple to `riakuser`'s options. + +### Managing Groups for a User + +If we have a user `riakuser` and we'd like to assign her to the +`admin` group, we assign the value `admin` to the option `groups`: + +```bash +riak-admin security alter-user riakuser groups=admin +``` + +If we'd like to make the user `riakuser` both an `admin` and an +`archoverlord`: + +```bash +riak-admin security alter-user riakuser groups=admin,archoverlord +``` + +There is no way to incrementally add groups; even if `riakuser` was +already an `admin`, it is necessary to list it again when adding the +`archoverlord` group. Thus, to remove a group from a user, use +`alter-user` and list all *other* groups. + +If the user should be removed from all groups, use `groups=` with no +list: + +```bash +riak-admin security alter-user riakuser groups= +``` + +### Managing Groups for Groups + +Groups can be added to other groups for cascading permissions. + +```bash +riak-admin security alter-group admin groups=dev +``` + +### Deleting a User or Group + +If you'd like to remove a user, use the `del-user` command: + +``` +riak-admin security del-user riakuser +``` + +For groups, use the `del-group` command: + +``` +riak-admin security del-group admin +``` + +### Adding or Deleting Multiple Users + +The `riak-admin security` command does not currently allow you to +add or delete multiple users using a single command. Instead, they must +be added or deleted one by one. + +## Managing Permissions + +Permission to perform a wide variety of operations against Riak can be +granted to---or revoked from---users via the `grant` and `revoke` +commands. + +### Basic Form + +The `grant` command takes one of the following forms: + +```bash +riak-admin security grant <permissions> on any to all|{<user>|<group>[,...]} +riak-admin security grant <permissions> on <bucket-type> to all|{<user>|<group>[,...]} +riak-admin security grant <permissions> on <bucket-type> <bucket> to all|{<user>|<group>[,...]} +``` + +The `revoke` command is essentially the same, except that `to` is +replaced with `from` of `to`: + +```bash +riak-admin security revoke <permissions> on any from all|{<user>|<group>[,...]} +riak-admin security revoke <permissions> on <bucket-type> from all|{<user>|<group>[,...]} +riak-admin security revoke <permissions> on <bucket-type> <bucket> from all|{<user>|<group>[,...]} +``` + +If you select `any`, this means that the permission (or set of +permissions) is granted/revoked for all buckets and [bucket types]({{<baseurl>}}riak/kv/2.2.6/developing/usage/bucket-types). If you specify a bucket type only, then the permission +is granted/revoked for all buckets of that type. If you specify a bucket +type _and_ a bucket, the permission is granted/revoked only for that +bucket type/bucket combination. + +**Note**: You cannot grant/revoke permissions with respect to a bucket +alone. You must specify either a bucket type by itself or a bucket type +and bucket. This limitation reflects the naming structure underlying +buckets and bucket types. + +Selecting `all` grants or revokes a permission (or set of permissions) +for all users in all groups. When specifying the user(s)/group(s) to +which you want to apply a permission (or set of permissions), you may +list any number of users or groups comma-separated with no whitespace. +Here is an example of granting multiple permissions across all buckets +and bucket types to multiple users: + +```bash +riak-admin security grant riak_kv.get,riak_search.query on any to jane,ahmed +``` + +If the same name is used for both a user and a group, the `grant` +command will ask for the name to be prefixed with `user/` or `group/` +to disambiguate. + +### Key/Value Permissions + +Permissions that can be granted for basic key/value access +functionality: + +Permission | Operation | +:----------|:----------| +`riak_kv.get` | Retrieve objects +`riak_kv.put` | Create or update objects +`riak_kv.delete` | Delete objects +`riak_kv.index` | Index objects using secondary indexes (2i) +`riak_kv.list_keys` | List all of the keys in a bucket +`riak_kv.list_buckets` | List all buckets + +{{% note title="Note on Listing Keys and Buckets" %}} +`riak_kv.list_keys` and `riak_kv.list_buckets` are both very expensive +operations that should be performed very rarely and never in production. +Access to this functionality should be granted very carefully. +{{% /note %}} + +If you'd like to create, for example, a `client` account that is +allowed only to run `GET` and `PUT` requests on all buckets: + +```bash +riak-admin security add-user client +riak-admin security grant riak_kv.get,riak_kv.put on any to client +``` + +### MapReduce Permissions + +Permission to perform [MapReduce]({{<baseurl>}}riak/kv/2.2.6/developing/usage/mapreduce/) jobs can be assigned +using `riak_kv.mapreduce`. The following example grants MapReduce +permissions to the user `mapreduce-power-user` for all buckets and +bucket types: + +```bash +riak-admin security grant riak_kv.mapreduce on any to mapreduce-power-user +``` + +### Bucket Type Permissions + +In versions 2.0 and later, Riak users can manage [bucket types]({{<baseurl>}}riak/kv/2.2.6/developing/usage/bucket-types) in addition to setting bucket properties. `riak-admin +security` allows you to manage the following bucket type-related +permissions: + +Permission | Operation +:----------|:--------- +`riak_core.get_bucket` | Retrieve the `props` associated with a bucket +`riak_core.set_bucket` | Modify the `props` associated with a bucket +`riak_core.get_bucket_type` | Retrieve the set of `props` associated with a bucket type +`riak_core.set_bucket_type` | Modify the set of `props` associated with a bucket type + +### Search Query Permission (Riak Search version 1) + +Security is incompatible with the original (and now deprecated) Riak +Search. Riak Search version 1 will stop working if security is enabled. + +### Search Query Permissions (Riak Search version 2, aka Yokozuna) + +If you are using the new Riak Search, i.e. the Solr-compatible search +capabilities included with Riak versions 2.0 and greater, the following +search-related permissions can be granted/revoked: + +Permission | Operation +:----------|:--------- +`search.admin` | The ability to perform search admin-related tasks, such as creating and deleting indexes and adding and modifying search schemas +`search.query` | The ability to query an index + +> **Note on Search Permissions** +> +> Search must be enabled in order to successfully grant/revoke Search +permissions. If you attempt to grant/revoke permissions while Search is +disabled, you will get the following error: +> +> `{error,{unknown_permission,"search.query"}}` +> +> More information on Riak Search and how to enable it can be found in the +[Riak Search Settings]({{<baseurl>}}riak/kv/2.2.6/configuring/search/) document. + +#### Usage Examples + +To grant the user `riakuser` the ability to query all indexes: + +```bash +riak-admin security grant search.query on index to riakuser + +# To revoke: +# riak-admin security revoke search.query on index from riakuser +``` + +To grant the user `riakuser` the ability to query all schemas: + +```bash +riak-admin security grant search.query on schema to riakuser + +# To revoke: +# riak-admin security revoke search.query on schema from riakuser +``` + +To grant the user `riakuser` admin privileges only on the index +`riakusers_index`: + +```bash +riak-admin security grant search.admin on index riakusers_index to riakuser + +# To revoke: +# riak-admin security revoke search.admin on index riakusers_index from riakuser +``` + +To grant `riakuser` querying and admin permissions on the index +`riakusers_index`: + +```bash +riak-admin security grant search.query,search.admin on index riakusers_index to riakuser + +# To revoke: +# riak-admin security revoke search.query,search.admin on index riakusers_index from riakuser +``` + +## Managing Sources + +While user management enables you to control _authorization_ with regard +to users, security **sources** provide you with an interface for +managing means of _authentication_. If you create users and grant them +access to some or all of Riak's functionality as described in the [User Management](#user-management) section, +you will then need to define security sources required for +authentication. + +An more in-depth tutorial can be found in [Managing Security Sources]({{<baseurl>}}riak/kv/2.2.6/using/security/managing-sources/). + +### Add Source + +Riak security sources may be applied to a specific user, multiple users, +or all users (`all`). + +#### Available Sources + +Source | Description +:--------|:----------- +`trust` | Always authenticates successfully if access has been granted to a user or all users on the specified CIDR range +`password` | Check the user's password against the [PBKFD2](http://en.wikipedia.org/wiki/PBKDF2)-hashed password stored in Riak +`pam` | Authenticate against the given pluggable authentication module (PAM) service +`certificate` | Authenticate using a client certificate + +### Example: Adding a Trusted Source + +Security sources can be added either to a specific user, multiple users, +or all users (`all`). + +In general, the `add-source` command takes the following form: + +```bash +riak-admin security add-source all|<users> <CIDR> <source> [<option>=<value>[...]] +``` + +Using `all` indicates that the authentication source can be added to +all users. A source can be added to a specific user, e.g. `add-source +superuser`, or to a list of users separated by commas, e.g. `add-source +jane,bill,admin`. + +Let's say that we want to give all users trusted access to securables +(without a password) when requests come from `localhost`: + +```bash +riak-admin security add-source all 127.0.0.1/32 trust +``` + +At that point, the `riak-admin security print-sources` command would +print the following: + +``` ++--------------------+------------+----------+----------+ +| users | cidr | source | options | ++--------------------+------------+----------+----------+ +| all |127.0.0.1/32| trust | [] | ++--------------------+------------+----------+----------+ +``` + +### Deleting Sources + +If we wish to remove the `trust` source that we granted to `all` in the +example above, we can simply use the `del-source` command and specify +the CIDR. + +```bash +riak-admin security del-source all 127.0.0.1/32 +``` + +Note that this does not require that you specify which type of source is +being deleted. You only need to specify the user(s) or `all`, because +only one source can be applied to a user or `all` at any given time. + +The following command would remove the source for `riakuser` on +`localhost`, regardless of which source is being used: + +```bash +riak-admin security del-source riakuser 127.0.0.1/32 +``` + +{{% note title="Note on Removing Sources" %}} +If you apply a security source both to `all` and to specific users and then +wish to remove that source, you will need to do so in separate steps. The +`riak-admin security del-source all ...` command by itself is not sufficient. + +For example, if you have assigned the source `password` to both `all` and to +the user `riakuser` on the network `127.0.0.1/32`, the following two-step +process would be required to fully remove the source: + +```bash +riak-admin security del-source all 127.0.0.1/32 password +riak-admin security del-source riakuser 127.0.0.1/32 password +``` +{{% /note %}} + +### More Usage Examples + +This section provides only a very brief overview of the syntax for +working with sources. For more information on using the `trust`, +`password`, `pam`, and `certificate` sources, please see our [Managing Security Sources]({{<baseurl>}}riak/kv/2.2.6/using/security/managing-sources/) document. + +## Security Ciphers + +To view a list of currently available security ciphers or change Riak's +preferences, use the `ciphers` command: + +```bash +riak-admin security ciphers +``` + +That command by itself will return a large list of available ciphers: + +``` +Configured ciphers + +ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256: ... + +Valid ciphers(35) + +ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256: ... + +Unknown/Unsupported ciphers(32) + +ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256: ... +``` + +To alter the list, i.e. to constrain it and/or to set preferred ciphers +higher in the list: + +```bash +riak-admin security ciphers DHE-RSA-AES256-SHA:AES128-GCM-SHA256 +``` + +The list of configured ciphers should now look like this: + +``` +Configured ciphers + +DHE-RSA-AES256-SHA:AES128-GCM-SHA256 + +Valid ciphers(1) + +DHE-RSA-AES256-SHA + +Unknown/Unsupported ciphers(1) + +AES128-GCM-SHA256 +``` + +A list of available ciphers on a server can be obtained using the +`openssl` command: + +```bash +openssl ciphers +``` + +That should return a list structured like this: + +``` +DHE-RSA-AES256-SHA:DHE-DSS-AES256-SHA:AES256-SHA:EDH-RSA-DES-CBC3-SHA: # and so on +``` + +Riak's cipher preferences were taken from [Mozilla's Server-Side TLS +documentation](https://wiki.mozilla.org/Security/Server_Side_TLS). + +### Client vs. Server Cipher Order + +By default, Riak prefers the cipher order that you set on the server, +i.e. the [`honor_cipher_order`]({{<baseurl>}}riak/kv/2.2.6/configuring/reference/#security) setting is set to `on`. If you prefer, however, that clients' preferred cipher +order dictate which cipher is chosen, set `honor_cipher_order` to `off`. + +> **Note on Erlang versions** +> +> Riak's default cipher order behavior has the potential to crash Erlang +VMs that do not support it. Erlang VMs that are known to support it +include Basho's patched version of Erlang R16. Instructions on +installing it can be found in [Installing Erlang]({{<baseurl>}}riak/kv/2.2.6/setup/installing/source/erlang). This issue should +not affect Erlang 17.0 and later. + +## Enabling SSL + +In order to use any authentication or authorization features, you must +enable SSL for Riak. **SSL is disabled by default**, but you will need +to enable it prior to enabling security. If you are using [Protocol Buffers]({{<baseurl>}}riak/kv/2.2.6/developing/api/protocol-buffers/) as a transport protocol for Riak (which we strongly recommend), enabling SSL on a given node requires only that you specify a [host and port]({{<baseurl>}}riak/kv/2.2.6/configuring/reference/#client-interfaces) for the node +as well as a [certification configuration](#certificate-configuration). + +If, however, you are using the [HTTP API]({{<baseurl>}}riak/kv/2.2.6/developing/api/http) for Riak and would like to +configure HTTPS, you will need to not only establish a [certificate configuration](#certificate-configuration) but also specify an HTTPS host +and port. The following configuration would establish port 8088 on +`localhost` as the HTTPS port: + +```riakconf +listener.https.$name = 127.0.0.1:8088 + +# By default, "internal" is used as the "name" setting +``` + +```appconfig +{riak_core, [ + %% Other configs + {https, [{"127.0.0.1", 8088}]}, + %% Other configs + ]} +``` + +## TLS Settings + +When using Riak security, you can choose which versions of SSL/TLS are +allowed. By default, only TLS 1.2 is allowed, but this version can be +disabled and others enabled by setting the following [configurable parameters]({{<baseurl>}}riak/kv/2.2.6/configuring/reference/#security) to `on` or `off`: + +* `tls_protocols.tlsv1` +* `tls_protocols.tlsv1.1` +* `tls_protocols.tlsv1.2` +* `tls_protocols.sslv3` + +Three things to note: + +* Among the four available options, only TLS version 1.2 is enabled by + default +* You can enable more than one protocol at a time +* We strongly recommend that you do _not_ use SSL version 3 unless + absolutely necessary + +## Certificate Configuration + +If you are using any of the available [security sources]({{<baseurl>}}riak/kv/2.2.6/using/security/managing-sources/), including [trust-based authentication]({{<baseurl>}}riak/kv/2.2.6/using/security/managing-sources/#trust-based-authentication), you will need to do so +over a secure SSL connection. In order to establish a secure connection, +you will need to ensure that each Riak node's [configuration files]({{<baseurl>}}riak/kv/2.2.6/configuring/reference/#security) point to the proper paths for your +generated certs. By default, Riak assumes that all certs are stored in +each node's `/etc` directory. + +If you are using the newer, `riak.conf`-based configuration system, you +can change the location of the `/etc` directory by modifying the +`platform_etc_dir`. More information can be found in our documentation +on [configuring directories]({{<baseurl>}}riak/kv/2.2.6/configuring/reference/#directories). + +<table class="riak-conf"> + <thead> + <tr> + <th>Type</th> + <th>Parameter</th> + <th>Default</th> + </tr> + </thead> + <tbody> + <tr> + <td><strong>Signing authority</strong></td> + <td><code>ssl.cacertfile</code></td> + <td><code>#(platform_etc_dir)/cacertfile.pem</code></td> + </tr> + <tr> + <td><strong>Cert</strong></td> + <td><code>ssl.certfile</code></td> + <td><code>#(platform_etc_dir)/cert.pem</code></td> + </tr> + <tr> + <td><strong>Key file</strong></td> + <td><code>ssl.keyfile</code></td> + <td><code>#(platform_etc_dir)/key.pem</code></td> + </tr> + </tbody> +</table> + +If you are using the older, `app.config`-based configuration system, +these paths can be set in the `ssl` subsection of the `riak_core` +section. The corresponding parameters are shown in the example below: + +```appconfig +{riak_core, [ + %% Other configs + + {ssl, [ + {certfile, "./etc/cert.pem"}, + {keyfile, "./etc/key.pem"}, + {cacertfile, "./etc/cacertfile.pem"} + ]}, + + %% Other configs +]} +``` + +## Referer Checks and Certificate Revocation Lists + +In order to provide safeguards against +[cross-site-scripting](http://en.wikipedia.org/wiki/Cross-site_scripting) +(XSS) and +[request-forgery](http://en.wikipedia.org/wiki/Cross-site_request_forgery) +attacks, Riak performs [secure referer +checks](http://en.wikipedia.org/wiki/HTTP_referer) by default. Those +checks make it impossible to serve data directly from Riak. To disable +those checks, set the `secure_referer_check` parameter to `off`. + +If you are using [certificate-based authentication]({{<baseurl>}}riak/kv/2.2.6/using/security/managing-sources/#certificate-based-authentication), Riak will check the certificate revocation list (CRL) of connecting clients' certificate by +default. To disable this behavior, set the `check_crl` parameter to +`off`. diff --git a/content/riak/kv/2.2.6/using/security/best-practices.md b/content/riak/kv/2.2.6/using/security/best-practices.md new file mode 100644 index 0000000000..68662c5030 --- /dev/null +++ b/content/riak/kv/2.2.6/using/security/best-practices.md @@ -0,0 +1,80 @@ +--- +draft: true +title: "Security Best Practices" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Best Practices" + identifier: "security_best_practices" + weight: 102 + parent: "managing_security" +toc: true +--- + +## Hanc capellae + +Lorem markdownum Byblida. Modo **etiam** litora mittat vellera infelix caeli. +Studiosius forte, potuit pectore. Puer undas dignior iam turpe sorores abesse. +Deae Saturnia levius viribus membra. + +## Iussorum ad fronti rutilasque tenuit cursu quae + +Nostros vovistis artes. **Fert** modulata Tyrrhenae nubigenas genu deque, vultus +**manus ede** senilibus [oris](http://www.youtube.com/watch?v=MghiBW3r65M) +transcurrere quem rarissima. Viderunt nutu quod, tumidaque, mihi mihi sacer pia. +Summis rediit pavidus tersere et at prosiluit natus Phaethon noxa. Singultibus +oblita **foedabis** orsa. + +- Fecere aliis postquam inviti caliginis ab inque +- Voverat dividuae et tardus huc magna non +- Sex barba ipsaque Caucason corpora sono ecce +- Non esse +- Sibi atris regna licuit Antium carituraque nubes + +## Omni levare gelidumque minanti + +Omnis adeunt ossibus gravis, Venus pinuque capit, et sereno viros ignara *plena +incaluere* percussit mellaque, vertere arte. Ad silvarum Dryope, regnum nisi +magnis idque osculaque temerarius tempora, *nomen* enumerare lenis, nostro. Ac +mutabit [arma](http://www.thesecretofinvisibility.com/) operiri saxum ratione, +crudelior feram, est usu tamen quod, hasta. Equos **sonant et deum**. Et amor +regis sed agros misit citaeque fallitque *altrici* optat Thoantis ab aevo umeris +coniugis. + +## Troiana quoque + +Equo uni Stygias trahunt, interea, in tela labores lumina, nam *Aganippe +sanctique meum*; est. [Gente inimica +premeret](http://en.wikipedia.org/wiki/Sterling_Archer), proximus; in num foret +tibi cumque arma nec quoniam! Contribuere mollis, tu dum parem viscera, tamen +ante. Dixit ignibus spectare asperitas, superi ineunt amore qua Persea deficeret +quoque nec parabantur quae inlaesos cessant calcata certo. Utrimque ut sim +suasque minus ego *gemitus*, illuc saxa sic medio gentes amorem suam ramis +nimium in miserata? + +1. `In naribus aequos aberant` +2. Naturae murmura te rimas suarum vulnus quod +3. Socios leto loquor timide +4. Ergo sub +5. Patrias mihi consumite breve + +## Ruit huic movit luminibus excubias arma + +> Loco humo tecum gurgite timui. Peragant tu regia ut umbras premit condit. Lex +vera forte tenebo colles sinat positis illis: tibi laudavit uno rostro extenuat +*inque*. Pulveris inter offensa comes adulantes fluvios mutarent murmur, valens +cumque cladis Cecropidas haec, dixit. Lucus cognomine **Achilles**: pastor nec. + +1. Hic causam et dilecte nudae nec corpus +2. Cor Si nive +3. Petis equos perosa tu perterrita exitus non +4. Per et et ire geminos parte +5. Aqua coniunx cecidisse sonum + +``` +Nominis haec lacrimis orba gloria obstipuere tu Ceyx tepebat fetus me equorum +potero! Iampridem illi; deducit [reor orbem](http://heeeeeeeey.com/), comes, et +nec rubebant pietas, ipsa. +``` diff --git a/content/riak/kv/2.2.6/using/security/managing-sources.md b/content/riak/kv/2.2.6/using/security/managing-sources.md new file mode 100644 index 0000000000..76800f238c --- /dev/null +++ b/content/riak/kv/2.2.6/using/security/managing-sources.md @@ -0,0 +1,269 @@ +--- +title: "Managing Security Sources" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Managing Security Sources" + identifier: "security_manage_sources" + weight: 101 + parent: "managing_security" +toc: true +aliases: + - /riak-docs/riak/2.2.6/ops/running/security-sources + - /riak-docs/riak/kv/2.2.6/ops/running/security-sources +--- + +If you're looking for more general information on Riak Security, it may +be best to start with our general guide to [authentication and authorization]({{<baseurl>}}riak/kv/2.2.6/using/security/basics). + +This document provides more granular information on the four available +authentication sources in Riak Security: trusted networks, password, +pluggable authentication modules (PAM), and certificates. These sources +correspond to `trust`, `password`, `pam`, and `certificate`, +respectively, in the `riak-admin security` interface. + +The examples below will assume that the network in question is +`127.0.0.1/32` and that a Riak user named `riakuser` has been +[created]({{<baseurl>}}riak/kv/2.2.6/using/security/basics/#user-management) and that +security has been [enabled]({{<baseurl>}}riak/kv/2.2.6/using/security/basics/#the-basics). + +{{% note title="Note on SSL connections" %}} +If you use _any_ of the aforementioned security sources, even `trust`, you +will need to do so via a secure SSL connection. +{{% /note %}} + +## Trust-based Authentication + +This form of authentication enables you to specify trusted +[CIDRs](http://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing) +from which all clients will be authenticated by default. + +```bash +riak-admin security add-source all 127.0.0.1/32 trust +``` + +Here, we have specified that anyone connecting to Riak from the +designated CIDR (in this case `localhost`) will be successfully +authenticated: + +```curl +curl https://localhost:8098/types/<type>/buckets/<bucket>/keys/<key> +``` + +If this request returns `not found` or a Riak object, then things have +been set up appropriately. You can specify any number of trusted +networks in the same fashion. + +You can also specify users as trusted users, as in the following +example: + +```bash +riak-admin security add-source riakuser 127.0.0.1/32 trust +``` + +Now, `riakuser` can interact with Riak without providing credentials. +Here's an example in which only the username is passed to Riak: + +```curl +curl -u riakuser: \ + https://localhost:8098/types/<type>/buckets/<bucket>/keys/<key> +``` + +## Password-based Authentication + +Authenticating via the `password` source requires that our `riakuser` be +given a password. `riakuser` can be assigned a password upon creation, +as in this example: + +```bash +riak-admin security add-user riakuser password=captheorem4life +``` + +Or a password can be assigned to an already existing user by modifying +that user's characteristics: + +```bash +riak-admin security alter-user riakuser password=captheorem4life +``` + +You can specify that _all_ users must authenticate themselves via +password when connecting to Riak from `localhost`: + +```bash +riak-admin security add-source all 127.0.0.1/32 password +``` + +Or you can specify that any number of specific users must do so: + +```bash +riak-admin security add-source riakuser 127.0.0.1/32 password +riak-admin security add-source otheruser 127.0.0.1/32 password + +# etc +``` + +Now, our `riakuser` must enter a username and password to have any +access to Riak whatsoever: + +```curl +curl -u riakuser:captheorem4life \ + https://localhost:8098/types/<type>/buckets/<bucket>/keys/<key> +``` + +## Certificate-based Authentication + +This form of authentication (`certificate`) requires that Riak and a +specified client---or clients---interacting with Riak bear certificates +signed by the same [Root Certificate +Authority](http://en.wikipedia.org/wiki/Root_certificate). + +> **Note** +> +> At this time, client certificates are not supported in Riak's HTTP +interface, and can be used only through the [protocol buffers interface]({{<baseurl>}}riak/kv/2.2.6/developing/api/protocol-buffers/). + +Let's specify that our user `riakuser` is going to be authenticated +using a certificate on `localhost`: + +```bash +riak-admin security add-source riakuser 127.0.0.1/32 certificate +``` + +When the `certificate` source is used, `riakuser` must also be entered +as the common name, aka `CN`, that you specified when you generated your +certificate, as in the following OpenSSL example: + +```bash +openssl req -new ... '/CN=riakuser' +``` + +You can add a `certificate` source to any number of clients, as long as +their `CN` and Riak username match. + +On the server side, you need to configure Riak by specifying a path to +your certificates. First, copy all relevant files to your Riak cluster. +The default directory for certificates is `/etc`, though you can specify +a different directory in your [`riak.conf`]({{<baseurl>}}riak/kv/2.2.6/configuring/reference/) by either uncommenting those lines if you choose to use the defaults or setting the paths yourself: + +```riakconf +ssl.certfile = /path/to/cert.pem +ssl.keyfile = /path/to/key.pem +ssl.cacertfile = /path/to/cacert.pem +``` + +In the client-side example above, the client's `CN` and Riak username +needed to match. On the server (i.e. Riak) side, the `CN` specified _on +each node_ must match the node's name as registered by Riak. You can +find the node's name in [`riak.conf`]({{<baseurl>}}riak/kv/2.2.6/configuring/reference/) under the parameter `nodename`. And so if the `nodename` for a cluster is +`riak-node-1`, you would need to generate your certificate with that in +mind, as in this OpenSSL example: + +```bash +openssl req -new ... '/CN=riak-node-1' +``` + +Once certificates have been properly generated and configured on all of +the nodes in your Riak cluster, you need to perform a [rolling restart]({{<baseurl>}}riak/kv/2.2.6/using/repair-recovery/rolling-restart/). Once that process is complete, you can use the client +certificate that you generated for the user `riakuser`. + +How to use Riak clients in conjunction with OpenSSL and other +certificates varies from client library to client library. We strongly +recommend checking the documentation of your client library for further +information. + +## PAM-based Authentication + +This section assumes that you have set up a PAM service bearing the name +`riak_pam`, e.g. by creating a `pam.d/riak_pam` service definition +specifying `auth` and/or other PAM services set up to authenticate a +user named `riakuser`. As in the certificate-based authentication +example above, the user's name must be the same in both your +authentication module and in Riak Security. + +If we want the user `riakuser` to use this PAM service on `localhost`, +we need to add a `pam` security source in Riak and specify the name of +the service: + +```bash +riak-admin security add-source all 127.0.0.1/32 pam service=riak_pam +``` + +**Note**: If you do not specify a name for your PAM service, Riak will +use the default, which is `riak`. + +To verify that the source has been properly specified: + +```bash +riak-admin security print-sources +``` + +That command should output the following: + +``` ++--------------------+------------+----------+------------------------+ +| users | cidr | source | options | ++--------------------+------------+----------+------------------------+ +| riakuser |127.0.0.1/32| pam |[{"service","riak_pam"}]| ++--------------------+------------+----------+------------------------+ +``` + +You can test that setup most easily by using `curl`. A normal request to +Riak without specifying a user will return an `Unauthorized` message: + +```curl +curl -u riakuser: \ + https://localhost:8098/types/<type>/buckets/<bucket>/keys/<key> +``` + +Response: + +``` +<html><head><title>401 Unauthorized

Unauthorized

Unauthorized


mochiweb+webmachine web server
+``` + +If you identify yourself as `riakuser` and are successfully +authenticated by your PAM service, you should get either `not found` or +a Riak object if one is stored in the specified bucket type/bucket/key +path: + +```curl +curl -u riakuser: \ + https://localhost:8098/types//buckets//keys/ +``` + +## How Sources Are Applied + +When managing security sources---any of the sources explained +above---you always have the option of applying a source to either a +single user, multiple users, or all users (`all`). If specific users and +`all` have no sources in common, this presents no difficulty. But what +happens if one source is applied to `all` and a different source is +applied to a specific user? + +The short answer is that the more specifically assigned source---i.e. to +the user---will be consider a user's security source. We'll illustrate +that with the following example, in which the `certificate` source is +assigned to `all`, but the `password` source is assigned to `riakuser`: + +```bash +riak-admin security add-source all 127.0.0.1/32 certificate +riak-admin security add-source riakuser 127.0.0.1/32 password +``` + +If we run `riak-admin security print-sources`, we'll get the following +output: + +``` ++--------------------+------------+-----------+----------+ +| users | cidr | source | options | ++--------------------+------------+-----------+----------+ +| riakuser |127.0.0.1/32| password | [] | +| |127.0.0.1/32|certificate| [] | +| all |127.0.0.1/32|certificate| [] | ++--------------------+------------+-----------+----------+ +``` + +As we can see, `password` is set as the security source for `riakuser`, +whereas everyone else will authenticate using `certificate`. diff --git a/content/riak/kv/2.2.6/using/security/v2-v3-ssl-ca.md b/content/riak/kv/2.2.6/using/security/v2-v3-ssl-ca.md new file mode 100644 index 0000000000..051820dd02 --- /dev/null +++ b/content/riak/kv/2.2.6/using/security/v2-v3-ssl-ca.md @@ -0,0 +1,80 @@ +--- +draft: true +title: "V2 / V3 SSL & CA Validation" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "V2/V3 SSL & CA Validation" + identifier: "security_validation" + weight: 103 + parent: "managing_security" +toc: true +--- + +## Hanc capellae + +Lorem markdownum Byblida. Modo **etiam** litora mittat vellera infelix caeli. +Studiosius forte, potuit pectore. Puer undas dignior iam turpe sorores abesse. +Deae Saturnia levius viribus membra. + +## Iussorum ad fronti rutilasque tenuit cursu quae + +Nostros vovistis artes. **Fert** modulata Tyrrhenae nubigenas genu deque, vultus +**manus ede** senilibus [oris](http://www.youtube.com/watch?v=MghiBW3r65M) +transcurrere quem rarissima. Viderunt nutu quod, tumidaque, mihi mihi sacer pia. +Summis rediit pavidus tersere et at prosiluit natus Phaethon noxa. Singultibus +oblita **foedabis** orsa. + +- Fecere aliis postquam inviti caliginis ab inque +- Voverat dividuae et tardus huc magna non +- Sex barba ipsaque Caucason corpora sono ecce +- Non esse +- Sibi atris regna licuit Antium carituraque nubes + +## Omni levare gelidumque minanti + +Omnis adeunt ossibus gravis, Venus pinuque capit, et sereno viros ignara *plena +incaluere* percussit mellaque, vertere arte. Ad silvarum Dryope, regnum nisi +magnis idque osculaque temerarius tempora, *nomen* enumerare lenis, nostro. Ac +mutabit [arma](http://www.thesecretofinvisibility.com/) operiri saxum ratione, +crudelior feram, est usu tamen quod, hasta. Equos **sonant et deum**. Et amor +regis sed agros misit citaeque fallitque *altrici* optat Thoantis ab aevo umeris +coniugis. + +## Troiana quoque + +Equo uni Stygias trahunt, interea, in tela labores lumina, nam *Aganippe +sanctique meum*; est. [Gente inimica +premeret](http://en.wikipedia.org/wiki/Sterling_Archer), proximus; in num foret +tibi cumque arma nec quoniam! Contribuere mollis, tu dum parem viscera, tamen +ante. Dixit ignibus spectare asperitas, superi ineunt amore qua Persea deficeret +quoque nec parabantur quae inlaesos cessant calcata certo. Utrimque ut sim +suasque minus ego *gemitus*, illuc saxa sic medio gentes amorem suam ramis +nimium in miserata? + +1. `In naribus aequos aberant` +2. Naturae murmura te rimas suarum vulnus quod +3. Socios leto loquor timide +4. Ergo sub +5. Patrias mihi consumite breve + +## Ruit huic movit luminibus excubias arma + +> Loco humo tecum gurgite timui. Peragant tu regia ut umbras premit condit. Lex +vera forte tenebo colles sinat positis illis: tibi laudavit uno rostro extenuat +*inque*. Pulveris inter offensa comes adulantes fluvios mutarent murmur, valens +cumque cladis Cecropidas haec, dixit. Lucus cognomine **Achilles**: pastor nec. + +1. Hic causam et dilecte nudae nec corpus +2. Cor Si nive +3. Petis equos perosa tu perterrita exitus non +4. Per et et ire geminos parte +5. Aqua coniunx cecidisse sonum + +``` +Nominis haec lacrimis orba gloria obstipuere tu Ceyx tepebat fetus me equorum +potero! Iampridem illi; deducit [reor orbem](http://heeeeeeeey.com/), comes, et +nec rubebant pietas, ipsa. +``` diff --git a/content/riak/kv/2.2.6/using/troubleshooting.md b/content/riak/kv/2.2.6/using/troubleshooting.md new file mode 100644 index 0000000000..735510a4ce --- /dev/null +++ b/content/riak/kv/2.2.6/using/troubleshooting.md @@ -0,0 +1,23 @@ +--- +title: "Troubleshooting" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "Troubleshooting" + identifier: "managing_troubleshooting" + weight: 207 + parent: "managing" +toc: true +--- + +[http 204]: ./http-204 + +## In This Section + +#### [HTTP 204][http 204] + +About the HTTP 204 response. + +[Learn More >>][http 204] diff --git a/content/riak/kv/2.2.6/using/troubleshooting/http-204.md b/content/riak/kv/2.2.6/using/troubleshooting/http-204.md new file mode 100644 index 0000000000..3b187d919e --- /dev/null +++ b/content/riak/kv/2.2.6/using/troubleshooting/http-204.md @@ -0,0 +1,17 @@ +--- +title: "HTTP 204" +description: "" +project: "riak_kv" +project_version: "2.2.6" +menu: + riak_kv-2.2.6: + name: "HTTP 204" + identifier: "troubleshooting_http_204" + weight: 101 + parent: "managing_troubleshooting" +toc: true +--- + +In the HTTP standard, a `204 No Content` is returned when the request was successful but there is nothing to return other than HTTP headers. + +If you add `returnbody=true` in the `PUT` request, you will receive a `200 OK` and the content you just stored, otherwise you will receive a `204 No Content`.