Snapshot taking too much cpu time

[chzhuo]

Filing M3 Issues

M3 Version: v1.0.0

Performance issues

flame graph

namespace config:
curl http://192.168.35.57:7201/api/v1/services/m3db/namespace

{
	"registry": {
		"namespaces": {
			"default": {
				"bootstrapEnabled": true,
				"flushEnabled": true,
				"writesToCommitLog": true,
				"cleanupEnabled": true,
				"repairEnabled": false,
				"retentionOptions": {
					"retentionPeriodNanos": "604800000000000",
					"blockSizeNanos": "7200000000000",
					"bufferFutureNanos": "120000000000",
					"bufferPastNanos": "600000000000",
					"blockDataExpiry": true,
					"blockDataExpiryAfterNotAccessPeriodNanos": "300000000000",
					"futureRetentionPeriodNanos": "0"
				},
				"snapshotEnabled": true,
				"indexOptions": {
					"enabled": true,
					"blockSizeNanos": "7200000000000"
				},
				"schemaOptions": null,
				"coldWritesEnabled": false,
				"runtimeOptions": null,
				"cacheBlocksOnRetrieve": false,
				"aggregationOptions": {
					"aggregations": [{
						"aggregated": false,
						"attributes": null
					}]
				},
				"stagingState": {
					"status": "UNKNOWN"
				},
				"extendedOptions": null
			}
		}
	}
}

[gibbscullen]

@chzhuo -- The snapshot will take a significant amount of CPU time as it is (snapshotting every few mins); however, the complexity of this is dependent on your particular workload.