From e02555ce822ec7ba7e89a21a5415d187b9377d33 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Tue, 22 Sep 2020 11:07:02 +0200 Subject: [PATCH] Cleanup Blobstore Repository Metadata Serialization (#62727) Follow ups to #62684 making use of shorter utility for corruption checks. --- .../BlobStoreIndexShardSnapshot.java | 54 ++++----- .../BlobStoreIndexShardSnapshots.java | 13 +- .../repositories/RepositoryData.java | 28 ++--- .../elasticsearch/snapshots/SnapshotInfo.java | 113 +++++++++--------- 4 files changed, 92 insertions(+), 116 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java b/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java index c53ad31080e4d..a72510b2b446a 100644 --- a/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java +++ b/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java @@ -29,6 +29,7 @@ import org.elasticsearch.common.xcontent.ToXContentFragment; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentParserUtils; import org.elasticsearch.index.store.StoreFileMetadata; import java.io.IOException; @@ -507,36 +508,33 @@ public static BlobStoreIndexShardSnapshot fromXContent(XContentParser parser) th XContentParser.Token token = parser.currentToken(); if (token == XContentParser.Token.START_OBJECT) { while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { - if (token == XContentParser.Token.FIELD_NAME) { - String currentFieldName = parser.currentName(); - token = parser.nextToken(); - if (token.isValue()) { - if (PARSE_NAME.match(currentFieldName, parser.getDeprecationHandler())) { - snapshot = parser.text(); - } else if (PARSE_INDEX_VERSION.match(currentFieldName, parser.getDeprecationHandler())) { - // The index-version is needed for backward compatibility with v 1.0 - indexVersion = parser.longValue(); - } else if (PARSE_START_TIME.match(currentFieldName, parser.getDeprecationHandler())) { - startTime = parser.longValue(); - } else if (PARSE_TIME.match(currentFieldName, parser.getDeprecationHandler())) { - time = parser.longValue(); - } else if (PARSE_INCREMENTAL_FILE_COUNT.match(currentFieldName, parser.getDeprecationHandler())) { - incrementalFileCount = parser.intValue(); - } else if (PARSE_INCREMENTAL_SIZE.match(currentFieldName, parser.getDeprecationHandler())) { - incrementalSize = parser.longValue(); - } else { - throw new ElasticsearchParseException("unknown parameter [{}]", currentFieldName); - } - } else if (token == XContentParser.Token.START_ARRAY) { - if (PARSE_FILES.match(currentFieldName, parser.getDeprecationHandler())) { - while ((parser.nextToken()) != XContentParser.Token.END_ARRAY) { - indexFiles.add(FileInfo.fromXContent(parser)); - } - } else { - throw new ElasticsearchParseException("unknown parameter [{}]", currentFieldName); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.FIELD_NAME, token, parser); + final String currentFieldName = parser.currentName(); + token = parser.nextToken(); + if (token.isValue()) { + if (PARSE_NAME.match(currentFieldName, parser.getDeprecationHandler())) { + snapshot = parser.text(); + } else if (PARSE_INDEX_VERSION.match(currentFieldName, parser.getDeprecationHandler())) { + // The index-version is needed for backward compatibility with v 1.0 + indexVersion = parser.longValue(); + } else if (PARSE_START_TIME.match(currentFieldName, parser.getDeprecationHandler())) { + startTime = parser.longValue(); + } else if (PARSE_TIME.match(currentFieldName, parser.getDeprecationHandler())) { + time = parser.longValue(); + } else if (PARSE_INCREMENTAL_FILE_COUNT.match(currentFieldName, parser.getDeprecationHandler())) { + incrementalFileCount = parser.intValue(); + } else if (PARSE_INCREMENTAL_SIZE.match(currentFieldName, parser.getDeprecationHandler())) { + incrementalSize = parser.longValue(); + } else { + throw new ElasticsearchParseException("unknown parameter [{}]", currentFieldName); + } + } else if (token == XContentParser.Token.START_ARRAY) { + if (PARSE_FILES.match(currentFieldName, parser.getDeprecationHandler())) { + while ((parser.nextToken()) != XContentParser.Token.END_ARRAY) { + indexFiles.add(FileInfo.fromXContent(parser)); } } else { - throw new ElasticsearchParseException("unexpected token [{}]", token); + throw new ElasticsearchParseException("unknown parameter [{}]", currentFieldName); } } else { throw new ElasticsearchParseException("unexpected token [{}]", token); diff --git a/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshots.java b/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshots.java index 1e2c8e0e91c22..337034e8dab0d 100644 --- a/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshots.java +++ b/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshots.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.xcontent.ToXContentFragment; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentParserUtils; import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot.FileInfo; import java.io.IOException; @@ -227,9 +228,7 @@ public static BlobStoreIndexShardSnapshots fromXContent(XContentParser parser) t Map files = new HashMap<>(); if (token == XContentParser.Token.START_OBJECT) { while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { - if (token != XContentParser.Token.FIELD_NAME) { - throw new ElasticsearchParseException("unexpected token [{}]", token); - } + XContentParserUtils.ensureExpectedToken(XContentParser.Token.FIELD_NAME, token, parser); String currentFieldName = parser.currentName(); token = parser.nextToken(); if (token == XContentParser.Token.START_ARRAY) { @@ -245,13 +244,9 @@ public static BlobStoreIndexShardSnapshots fromXContent(XContentParser parser) t throw new ElasticsearchParseException("unknown object [{}]", currentFieldName); } while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { - if (token != XContentParser.Token.FIELD_NAME) { - throw new ElasticsearchParseException("unknown object [{}]", currentFieldName); - } + XContentParserUtils.ensureExpectedToken(XContentParser.Token.FIELD_NAME, token, parser); String snapshot = parser.currentName(); - if (parser.nextToken() != XContentParser.Token.START_OBJECT) { - throw new ElasticsearchParseException("unknown object [{}]", currentFieldName); - } + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser); while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); diff --git a/server/src/main/java/org/elasticsearch/repositories/RepositoryData.java b/server/src/main/java/org/elasticsearch/repositories/RepositoryData.java index 4327b1311de2c..162c7bb3404b5 100644 --- a/server/src/main/java/org/elasticsearch/repositories/RepositoryData.java +++ b/server/src/main/java/org/elasticsearch/repositories/RepositoryData.java @@ -529,9 +529,7 @@ public IndexMetaDataGenerations indexMetaDataGenerations() { * from cached bytes that we trust to not contain broken generations. */ public static RepositoryData snapshotsFromXContent(XContentParser parser, long genId, boolean fixBrokenShardGens) throws IOException { - if (parser.nextToken() != XContentParser.Token.START_OBJECT) { - throw new ElasticsearchParseException("start object expected"); - } + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser); final Map snapshots = new HashMap<>(); final Map snapshotStates = new HashMap<>(); @@ -551,15 +549,11 @@ public static RepositoryData snapshotsFromXContent(XContentParser parser, long g parseIndices(parser, fixBrokenShardGens, snapshots, indexSnapshots, indexLookup, shardGenerations); break; case INDEX_METADATA_IDENTIFIERS: - if (parser.nextToken() != XContentParser.Token.START_OBJECT) { - throw new ElasticsearchParseException("start object expected [" + INDEX_METADATA_IDENTIFIERS + "]"); - } + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser); indexMetaIdentifiers = parser.mapStrings(); break; case MIN_VERSION: - if (parser.nextToken() != XContentParser.Token.VALUE_STRING) { - throw new ElasticsearchParseException("version string expected [min_version]"); - } + XContentParserUtils.ensureExpectedToken(XContentParser.Token.VALUE_STRING, parser.nextToken(), parser); final Version version = Version.fromString(parser.text()); assert SnapshotsService.useShardGenerations(version); break; @@ -615,9 +609,7 @@ private static IndexMetaDataGenerations buildIndexMetaGenerations(Map snapshots, Map snapshotStates, Map snapshotVersions, Map> indexMetaLookup) throws IOException { - if (parser.nextToken() != XContentParser.Token.START_ARRAY) { - throw new ElasticsearchParseException("expected array for [" + SNAPSHOTS + "]"); - } + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.nextToken(), parser); while (parser.nextToken() != XContentParser.Token.END_ARRAY) { String name = null; String uuid = null; @@ -673,18 +665,14 @@ private static void parseSnapshots(XContentParser parser, Map snapshots, Map> indexSnapshots, Map indexLookup, ShardGenerations.Builder shardGenerations) throws IOException { - if (parser.nextToken() != XContentParser.Token.START_OBJECT) { - throw new ElasticsearchParseException("start object expected [indices]"); - } + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser); while (parser.nextToken() != XContentParser.Token.END_OBJECT) { final String indexName = parser.currentName(); final List snapshotIds = new ArrayList<>(); final List gens = new ArrayList<>(); IndexId indexId = null; - if (parser.nextToken() != XContentParser.Token.START_OBJECT) { - throw new ElasticsearchParseException("start object expected index[" + indexName + "]"); - } + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser); while (parser.nextToken() != XContentParser.Token.END_OBJECT) { final String indexMetaFieldName = parser.currentName(); final XContentParser.Token currentToken = parser.nextToken(); @@ -693,9 +681,7 @@ private static void parseIndices(XContentParser parser, boolean fixBrokenShardGe indexId = new IndexId(indexName, parser.text()); break; case SNAPSHOTS: - if (currentToken != XContentParser.Token.START_ARRAY) { - throw new ElasticsearchParseException("start array expected [snapshots]"); - } + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, currentToken, parser); XContentParser.Token currToken; while ((currToken = parser.nextToken()) != XContentParser.Token.END_ARRAY) { final String uuid; diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java index 7c85f5582427b..6ddc695cd618f 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java @@ -18,7 +18,6 @@ */ package org.elasticsearch.snapshots; -import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.Version; import org.elasticsearch.action.ShardOperationFailedException; import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest; @@ -34,6 +33,7 @@ import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentParserUtils; import org.elasticsearch.repositories.IndexId; import org.elasticsearch.rest.RestStatus; @@ -625,70 +625,67 @@ public static SnapshotInfo fromXContentInternal(final XContentParser parser) thr parser.nextToken(); } XContentParser.Token token; - if ((token = parser.nextToken()) == XContentParser.Token.START_OBJECT) { - String currentFieldName = parser.currentName(); - if (SNAPSHOT.equals(currentFieldName)) { - while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { - if (token == XContentParser.Token.FIELD_NAME) { - currentFieldName = parser.currentName(); - token = parser.nextToken(); - if (token.isValue()) { - if (NAME.equals(currentFieldName)) { - name = parser.text(); - } else if (UUID.equals(currentFieldName)) { - uuid = parser.text(); - } else if (STATE.equals(currentFieldName)) { - state = SnapshotState.valueOf(parser.text()); - } else if (REASON.equals(currentFieldName)) { - reason = parser.text(); - } else if (START_TIME.equals(currentFieldName)) { - startTime = parser.longValue(); - } else if (END_TIME.equals(currentFieldName)) { - endTime = parser.longValue(); - } else if (TOTAL_SHARDS.equals(currentFieldName)) { - totalShards = parser.intValue(); - } else if (SUCCESSFUL_SHARDS.equals(currentFieldName)) { - successfulShards = parser.intValue(); - } else if (VERSION_ID.equals(currentFieldName)) { - version = Version.fromId(parser.intValue()); - } else if (INCLUDE_GLOBAL_STATE.equals(currentFieldName)) { - includeGlobalState = parser.booleanValue(); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser); + String currentFieldName = parser.currentName(); + if (SNAPSHOT.equals(currentFieldName)) { + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + token = parser.nextToken(); + if (token.isValue()) { + if (NAME.equals(currentFieldName)) { + name = parser.text(); + } else if (UUID.equals(currentFieldName)) { + uuid = parser.text(); + } else if (STATE.equals(currentFieldName)) { + state = SnapshotState.valueOf(parser.text()); + } else if (REASON.equals(currentFieldName)) { + reason = parser.text(); + } else if (START_TIME.equals(currentFieldName)) { + startTime = parser.longValue(); + } else if (END_TIME.equals(currentFieldName)) { + endTime = parser.longValue(); + } else if (TOTAL_SHARDS.equals(currentFieldName)) { + totalShards = parser.intValue(); + } else if (SUCCESSFUL_SHARDS.equals(currentFieldName)) { + successfulShards = parser.intValue(); + } else if (VERSION_ID.equals(currentFieldName)) { + version = Version.fromId(parser.intValue()); + } else if (INCLUDE_GLOBAL_STATE.equals(currentFieldName)) { + includeGlobalState = parser.booleanValue(); + } + } else if (token == XContentParser.Token.START_ARRAY) { + if (DATA_STREAMS.equals(currentFieldName)){ + dataStreams = new ArrayList<>(); + while (parser.nextToken() != XContentParser.Token.END_ARRAY) { + dataStreams.add(parser.text()); } - } else if (token == XContentParser.Token.START_ARRAY) { - if (DATA_STREAMS.equals(currentFieldName)){ - dataStreams = new ArrayList<>(); - while (parser.nextToken() != XContentParser.Token.END_ARRAY) { - dataStreams.add(parser.text()); - } - } else if (INDICES.equals(currentFieldName)) { - ArrayList indicesArray = new ArrayList<>(); - while (parser.nextToken() != XContentParser.Token.END_ARRAY) { - indicesArray.add(parser.text()); - } - indices = Collections.unmodifiableList(indicesArray); - } else if (FAILURES.equals(currentFieldName)) { - ArrayList shardFailureArrayList = new ArrayList<>(); - while (parser.nextToken() != XContentParser.Token.END_ARRAY) { - shardFailureArrayList.add(SnapshotShardFailure.fromXContent(parser)); - } - shardFailures = Collections.unmodifiableList(shardFailureArrayList); - } else { - // It was probably created by newer version - ignoring - parser.skipChildren(); + } else if (INDICES.equals(currentFieldName)) { + ArrayList indicesArray = new ArrayList<>(); + while (parser.nextToken() != XContentParser.Token.END_ARRAY) { + indicesArray.add(parser.text()); } - } else if (token == XContentParser.Token.START_OBJECT) { - if (USER_METADATA.equals(currentFieldName)) { - userMetadata = parser.map(); - } else { - // It was probably created by newer version - ignoring - parser.skipChildren(); + indices = Collections.unmodifiableList(indicesArray); + } else if (FAILURES.equals(currentFieldName)) { + ArrayList shardFailureArrayList = new ArrayList<>(); + while (parser.nextToken() != XContentParser.Token.END_ARRAY) { + shardFailureArrayList.add(SnapshotShardFailure.fromXContent(parser)); } + shardFailures = Collections.unmodifiableList(shardFailureArrayList); + } else { + // It was probably created by newer version - ignoring + parser.skipChildren(); + } + } else if (token == XContentParser.Token.START_OBJECT) { + if (USER_METADATA.equals(currentFieldName)) { + userMetadata = parser.map(); + } else { + // It was probably created by newer version - ignoring + parser.skipChildren(); } } } } - } else { - throw new ElasticsearchParseException("unexpected token [" + token + "]"); } if (uuid == null) { // the old format where there wasn't a UUID