Feature/#5390 segment indexing reload status api #5718

guruguha · 2020-07-20T00:59:34Z

Description

This PR adds an APIs on the controller and one new API on the server. The purpose of the APIs is to provide segment metadata from the Pinot Server.

Following updates need to be made in the documentation:

Need to update documentation for querying segment metadata from the server

Sample response:

{
  "baseballStats_OFFLINE_0": {
    "segmentName": "baseballStats_OFFLINE_0",
    "schemaName": null,
    "crc": 2400783875,
    "creationTimeMillis": 1599866671163,
    "creationTimeReadable": "2020-09-11T23:24:31:163 UTC",
    "timeGranularitySec": null,
    "startTimeMillis": null,
    "startTimeReadable": null,
    "endTimeMillis": null,
    "endTimeReadable": null,
    "pushTimeMillis": -9223372036854776000,
    "pushTimeReadable": null,
    "refreshTimeMillis": -9223372036854776000,
    "refreshTimeReadable": null,
    "segmentVersion": "v3",
    "creatorName": null,
    "paddingCharacter": "\u0000",
    "columns": [],
    "indexes": {
      "homeRuns": {
        "bloom-filter": "NO",
        "dictionary": "YES",
        "forward-index": "YES",
        "inverted-index": "NO",
        "null-value-vector-reader": "NO",
        "range-index": "NO"
      },
      "$hostName": {
        "bloom-filter": "NO",
        "dictionary": "YES",
        "forward-index": "YES",
        "inverted-index": "YES",
        "null-value-vector-reader": "NO",
        "range-index": "NO"
      },
      "playerStint": {
        "bloom-filter": "NO",
        "dictionary": "YES",
        "forward-index": "YES",
        "inverted-index": "NO",
        "null-value-vector-reader": "NO",
        "range-index": "NO"
      },
      "groundedIntoDoublePlays": {
        "bloom-filter": "NO",
        "dictionary": "YES",
        "forward-index": "YES",
        "inverted-index": "NO",
        "null-value-vector-reader": "NO",
        "range-index": "NO"
      },
      "numberOfGames": {
        "bloom-filter": "NO",
        "dictionary": "YES",
        "forward-index": "YES",
        "inverted-index": "NO",
        "null-value-vector-reader": "NO",
        "range-index": "NO"
      },
      "$segmentName": {
        "bloom-filter": "NO",
        "dictionary": "YES",
        "forward-index": "YES",
        "inverted-index": "YES",
        "null-value-vector-reader": "NO",
        "range-index": "NO"
      },
      "AtBatting": {
        "bloom-filter": "NO",
        "dictionary": "YES",
        "forward-index": "YES",
        "inverted-index": "NO",
        "null-value-vector-reader": "NO",
        "range-index": "NO"
      },
      "stolenBases": {
        "bloom-filter": "NO",
        "dictionary": "YES",
        "forward-index": "YES",
        "inverted-index": "NO",
        "null-value-vector-reader": "NO",
        "range-index": "NO"
      },
      "tripples": {
        "bloom-filter": "NO",
        "dictionary": "YES",
        "forward-index": "YES",
        "inverted-index": "NO",
        "null-value-vector-reader": "NO",
        "range-index": "NO"
      },
      "hitsByPitch": {
        "bloom-filter": "NO",
        "dictionary": "YES",
        "forward-index": "YES",
        "inverted-index": "NO",
        "null-value-vector-reader": "NO",
        "range-index": "NO"
      },
      "teamID": {
        "bloom-filter": "NO",
        "dictionary": "YES",
        "forward-index": "YES",
        "inverted-index": "YES",
        "null-value-vector-reader": "NO",
        "range-index": "NO"
      },
      "numberOfGamesAsBatter": {
        "bloom-filter": "NO",
        "dictionary": "YES",
        "forward-index": "YES",
        "inverted-index": "NO",
        "null-value-vector-reader": "NO",
        "range-index": "NO"
      },
      "strikeouts": {
        "bloom-filter": "NO",
        "dictionary": "YES",
        "forward-index": "YES",
        "inverted-index": "NO",
        "null-value-vector-reader": "NO",
        "range-index": "NO"
      },
      "sacrificeFlies": {
        "bloom-filter": "NO",
        "dictionary": "YES",
        "forward-index": "YES",
        "inverted-index": "NO",
        "null-value-vector-reader": "NO",
        "range-index": "NO"
      },
      "caughtStealing": {
        "bloom-filter": "NO",
        "dictionary": "YES",
        "forward-index": "YES",
        "inverted-index": "NO",
        "null-value-vector-reader": "NO",
        "range-index": "NO"
      },
      "baseOnBalls": {
        "bloom-filter": "NO",
        "dictionary": "YES",
        "forward-index": "YES",
        "inverted-index": "NO",
        "null-value-vector-reader": "NO",
        "range-index": "NO"
      },
      "playerName": {
        "bloom-filter": "NO",
        "dictionary": "YES",
        "forward-index": "YES",
        "inverted-index": "NO",
        "null-value-vector-reader": "NO",
        "range-index": "NO"
      },
      "league": {
        "bloom-filter": "NO",
        "dictionary": "YES",
        "forward-index": "YES",
        "inverted-index": "YES",
        "null-value-vector-reader": "NO",
        "range-index": "NO"
      },
      "doules": {
        "bloom-filter": "NO",
        "dictionary": "YES",
        "forward-index": "YES",
        "inverted-index": "NO",
        "null-value-vector-reader": "NO",
        "range-index": "NO"
      },
      "$docId": {
        "bloom-filter": "NO",
        "dictionary": "YES",
        "forward-index": "YES",
        "inverted-index": "YES",
        "null-value-vector-reader": "NO",
        "range-index": "NO"
      },
      "yearID": {
        "bloom-filter": "NO",
        "dictionary": "YES",
        "forward-index": "YES",
        "inverted-index": "YES",
        "null-value-vector-reader": "NO",
        "range-index": "NO"
      },
      "hits": {
        "bloom-filter": "NO",
        "dictionary": "YES",
        "forward-index": "YES",
        "inverted-index": "NO",
        "null-value-vector-reader": "NO",
        "range-index": "NO"
      },
      "runsBattedIn": {
        "bloom-filter": "NO",
        "dictionary": "YES",
        "forward-index": "YES",
        "inverted-index": "NO",
        "null-value-vector-reader": "NO",
        "range-index": "NO"
      },
      "G_old": {
        "bloom-filter": "NO",
        "dictionary": "YES",
        "forward-index": "YES",
        "inverted-index": "NO",
        "null-value-vector-reader": "NO",
        "range-index": "NO"
      },
      "sacrificeHits": {
        "bloom-filter": "NO",
        "dictionary": "YES",
        "forward-index": "YES",
        "inverted-index": "NO",
        "null-value-vector-reader": "NO",
        "range-index": "NO"
      },
      "intentionalWalks": {
        "bloom-filter": "NO",
        "dictionary": "YES",
        "forward-index": "YES",
        "inverted-index": "NO",
        "null-value-vector-reader": "NO",
        "range-index": "NO"
      },
      "runs": {
        "bloom-filter": "NO",
        "dictionary": "YES",
        "forward-index": "YES",
        "inverted-index": "NO",
        "null-value-vector-reader": "NO",
        "range-index": "NO"
      },
      "playerID": {
        "bloom-filter": "NO",
        "dictionary": "YES",
        "forward-index": "YES",
        "inverted-index": "YES",
        "null-value-vector-reader": "NO",
        "range-index": "NO"
      }
    }
  }
}

Documentation

The documentation to the PR is here.

- added a new API endpoint for users to query segment reload status API - Table metadata from Server - added a new endpoint to fetch segment metadata - added helper classes and methods to fetch metadata from the server Tests - added test to server API to fetch metadata including indexing information

- Moved status classes to logical places Logs - Added logging statements Tests - Added unit tests for Pinot Controller reload status and segment metadata API - Added unit tests for Pinot Server reload status and segment metadata API License Headers - Add license headers to files added to this feature

npawar

Please add javadocs to all new classes and to any public methods introduced

...ler/src/main/java/org/apache/pinot/controller/api/resources/PinotSegmentRestletResource.java

...ler/src/main/java/org/apache/pinot/controller/api/resources/ServerSegmentMetadataReader.java

...ler/src/main/java/org/apache/pinot/controller/api/resources/PinotSegmentRestletResource.java

...ler/src/main/java/org/apache/pinot/controller/api/resources/ServerSegmentMetadataReader.java

...-server/src/main/java/org/apache/pinot/server/api/resources/SegmentColumnIndexesFetcher.java

pinot-server/src/test/java/org/apache/pinot/server/api/TablesResourceTest.java

- Updating code as per PR review comments

Removing SegmentMetadataFetcher as it seemed redundant Refactoring code to save failed segment reload status API calls as part of response

mcvsubbu

A lot of code duplication with the existing table size reader. Please find ways to use a common base class if possible

...ler/src/main/java/org/apache/pinot/controller/api/resources/PinotSegmentRestletResource.java

mcvsubbu · 2020-07-20T15:56:08Z

...ler/src/main/java/org/apache/pinot/controller/api/resources/PinotSegmentRestletResource.java

+        tableReloadStatus = getSegmentsReloadStatus(tableNameWithType);
+      } catch (InvalidConfigException e) {
+        throw new ControllerApplicationException(LOGGER,
+                "Failed to load segment reload status for table: " + tableName, Status.NOT_FOUND);


If you are returning 404 (NOT_FOUND) then please do not use "Failed" in the exception message. Since the exception is invalid config, determine what is invalid and throw that exception, may be as 400 (BAD_REQUEST)

Updated in latest commit.

mcvsubbu · 2020-07-20T15:57:08Z

...ler/src/main/java/org/apache/pinot/controller/api/resources/PinotSegmentRestletResource.java

+                "Failed to load segment reload status for table: " + tableName, Status.NOT_FOUND);
+      }
+      if (Objects.isNull(tableReloadStatus))
+        throw new ControllerApplicationException(LOGGER,


The exception message reads as if the table is not found. That is not the case, right?

mcvsubbu · 2020-07-20T15:57:26Z

...ler/src/main/java/org/apache/pinot/controller/api/resources/PinotSegmentRestletResource.java

+          @ApiParam(value = "Name of the table", required = true) @PathParam("tableName") String tableName,
+          @ApiParam(value = "OFFLINE|REALTIME") @QueryParam("type") String tableTypeStr) {
+    List<String> tableNamesWithType = getExistingTableNamesWithType(tableName, Constants.validateTableType(tableTypeStr));
+    Map<String, TableMetadataReader.TableReloadStatus> reloadStatusMap = new HashMap<>();


If there are no tables found, then this is the place to throw 404

mcvsubbu · 2020-07-20T16:00:46Z

...ler/src/main/java/org/apache/pinot/controller/api/resources/PinotSegmentRestletResource.java

+
+  private Map<String, String> getSegmentsMetadataFromServer(String tableNameWithType)
+          throws InvalidConfigException, IOException {
+    LOGGER.trace("Inside getSegmentsMetadataFromServer() entry");


remove these trace logs please

mcvsubbu · 2020-07-29T17:09:52Z

pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/SegmentStatus.java

+
+import java.util.Objects;
+
+public class SegmentStatus {


Please document each member in this object clearly, what it contains in various situations

...ler/src/main/java/org/apache/pinot/controller/api/resources/PinotSegmentRestletResource.java

mcvsubbu · 2020-07-29T17:13:24Z

...ler/src/main/java/org/apache/pinot/controller/api/resources/PinotSegmentRestletResource.java

+  @ApiOperation(value = "Get the server metadata for all table segments", notes = "Get the server metadata for all table segments")
+  public Map<String, String> getServerMetadata(@ApiParam(value = "Name of the table", required = true) @PathParam("tableName") String tableName,
+                                               @ApiParam(value = "OFFLINE|REALTIME") @QueryParam("type") String tableTypeStr) {
+    LOGGER.info("Received a request to fetch metadata for all segments for table {}", tableName);


Seems like a debug level log.

mcvsubbu · 2020-07-29T17:14:41Z

...ler/src/main/java/org/apache/pinot/controller/api/resources/PinotSegmentRestletResource.java

+      throw new ControllerApplicationException(LOGGER, e.getMessage(), Status.BAD_REQUEST);
+    } catch (IOException ioe) {
+      throw new ControllerApplicationException(LOGGER,
+          "Error parsing Pinot server response: " + ioe.getMessage(), Status.INTERNAL_SERVER_ERROR, ioe);


Indicate the server name here that caused the error (unless that is logged elsewhere)

Yes, is logged in the helper class

mcvsubbu · 2020-07-29T17:18:11Z

...ler/src/main/java/org/apache/pinot/controller/api/resources/PinotSegmentRestletResource.java

+  @GET
+  @Path("segments/{tableName}/reload-status")
+  @Produces(MediaType.APPLICATION_JSON)
+  @ApiOperation(value = "Status of segment reload", notes = "Status of segment reload")


I strongly suggest adding a verbosity level and/or a limit here. Can be added later if you wish. Imagine a table with a million segments. Do we really want to kill the servers trying to query all the segments? Or, output them only to let the client time out?

An example could be: limit=100 by default, verbosity=5. A level of 4, 3, 2,1 will show less information for each segment. Maybe 0 will only show how many segments that are online/offline etc.?

Thanks for the suggestion. Makes sense to add limit. I did think about this, but then, the issue will be knowing the status of the remaining segments. For a table with say, 1000 segments, how do we let the user know of the status of the rest of the segments?

Add an API to get status for a range of segments, maybe? Or, add some sort of start/limit?

If you are adding one to get the status of one segment at a time, then the user can (if needed) iterate over the segments and get each segment. Let us evaluate the use case first. Are we talking about a full table reload or a segment reload? If full table reload, maybe we only want to return those segments that DID NOT reload properly?

The API definition leaves much discussion to be desired, and a PR is NOT the place to discuss API. If you have a design doc, we will discuss there.

this can be done in another PR. Lets get this in and add the optimizations as we need them. Million segments in a table is not a common use case.

guruguha · 2020-08-01T00:31:46Z

Initially, the thought was to have status for a single segment reload. But then it was observed that usually, users request for a full segment reload. So, I added the API to show reload status for all segments. I do have a design doc <https://docs.google.com/document/d/1E_J7PxF9WtaE6ido__u0O-Emtyu0IghQicC-1eLZeRo/edit#>. Sure, let's discuss there. :)

…

-Guru

On Fri, Jul 31, 2020 at 5:25 PM Subbu Subramaniam ***@***.***> wrote: ***@***.**** commented on this pull request. ------------------------------ In pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotSegmentRestletResource.java <#5718 (comment)> : > @@ -485,4 +503,80 @@ private void deleteSegmentsInternal(String tableNameWithType, List<String> segme throw new ControllerApplicationException(LOGGER, e.getMessage(), Response.Status.FORBIDDEN); } } + + @get + @path("segments/{tableName}/reload-status") + @produces(MediaType.APPLICATION_JSON) + @ApiOperation(value = "Status of segment reload", notes = "Status of segment reload") If you are adding one to get the status of one segment at a time, then the user can (if needed) iterate over the segments and get each segment. Let us evaluate the use case first. Are we talking about a full table reload or a segment reload? If full table reload, maybe we only want to return those segments that DID NOT reload properly? The API definition leaves much discussion to be desired, and a PR is NOT the place to discuss API. If you have a design doc, we will discuss there. — You are receiving this because you authored the thread. Reply to this email directly, view it on GitHub <#5718 (comment)>, or unsubscribe <https://github.com/notifications/unsubscribe-auth/ACFCTWTQNNHNVC63IBMT3DDR6NOHNANCNFSM4PBWT4GA> .

kishoreg · 2020-08-09T14:58:33Z

is this ready to go?

guruguha · 2020-08-09T15:32:26Z

There were a few concerns raised by Subbu on the number of API calls that would be made to a server in the case where there are a large number of segments. Wanted to discuss further on what to do next for this feature. What do you suggest?

…

On Sun, Aug 9, 2020 at 7:58 AM Kishore Gopalakrishna < ***@***.***> wrote: is this ready to go? — You are receiving this because you authored the thread. Reply to this email directly, view it on GitHub <#5718 (comment)>, or unsubscribe <https://github.com/notifications/unsubscribe-auth/ACFCTWVTNNAWM3UHNLGNRV3R722SLANCNFSM4PBWT4GA> .

Pinot codestyle corrections Moving ServerSegmentMetadataReader to util

npawar

LGTM as well

kishoreg · 2020-08-13T06:14:52Z

A lot of code duplication with the existing table size reader. Please find ways to use a common base class if possible

Is this addressed?

guruguha · 2020-08-13T06:20:54Z

A lot of code duplication with the existing table size reader. Please find ways to use a common base class if possible

Is this addressed?

Oh! Somehow missed this comment. I think it got lost in between. Let me make this change and commit again. Sorry about that!

…xing-reload-status-api' into feature/apache#5390-segment-indexing-reload-status-api # Conflicts: # pinot-controller/src/main/java/org/apache/pinot/controller/util/CompletionServiceHelper.java

mcvsubbu · 2020-08-20T15:57:04Z

pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/SegmentStatus.java

+
+import java.util.Objects;
+
+public class SegmentStatus {


Not all comments have been addressed. Please justify the use of the same class to return values to the user. It makes upgrades bad. Add json ignore case so that the pain is at least reduced a bit.

mcvsubbu · 2020-08-20T16:03:14Z

...ontroller/src/main/java/org/apache/pinot/controller/api/resources/ServerTableSizeReader.java

+        serverToSegmentSizeInfoListMap.put(streamResponse.getKey(), tableSizeInfo.segments);
+      } catch (IOException e) {
+        failedParses++;
+        LOGGER.error("Unable to parse server response due to an error: ", e);


Add the server name to this log

mcvsubbu · 2020-08-20T16:06:00Z

...ontroller/src/main/java/org/apache/pinot/controller/api/resources/ServerTableSizeReader.java

-    } else {
-      LOGGER.info("Finish reading segment sizes for table: {}", tableNameWithType);
+    if (failedParses != 0) {
+      LOGGER.warn("Failed to parse {} segment size info responses from server.", failedParses);


Suggested change

LOGGER.warn("Failed to parse {} segment size info responses from server.", failedParses);

LOGGER.warn("Failed to parse segment size info responses from {} servers.", failedParses);

If possible, add the total number of servers to this message as well

mcvsubbu · 2020-08-20T16:08:21Z

pinot-controller/src/main/java/org/apache/pinot/controller/util/CompletionServiceHelper.java

+    int numServersResponded = completionServiceResponse._httpResponses.size();
+    if (numServersResponded != serverURLs.size()) {
+      LOGGER.warn("Finish reading information for table: {} with {}/{} server responses", tableNameWithType,
+          numServersResponded, serverURLs);


Suggested change

numServersResponded, serverURLs);

numServersResponded, serverURLs.size());

mcvsubbu · 2020-08-20T16:09:30Z

...t-controller/src/main/java/org/apache/pinot/controller/util/ServerSegmentMetadataReader.java

+        segmentsMetadata.add(JsonUtils.objectToString(segmentMetadata));
+      } catch (IOException e) {
+        failedParses++;
+        LOGGER.error("Unable to parse server response due to an error: ", e);


Add server name in the log

mcvsubbu · 2020-08-20T16:12:15Z

...t-controller/src/main/java/org/apache/pinot/controller/util/ServerSegmentMetadataReader.java

+        segmentsStatus.add(segmentStatus);
+      } catch (IOException e) {
+        failedParses++;
+        LOGGER.error("Unable to parse server response due to an error: ", e);


add server name

mcvsubbu · 2020-08-20T16:12:32Z

...t-controller/src/main/java/org/apache/pinot/controller/util/ServerSegmentMetadataReader.java

+      }
+    }
+    if (failedParses != 0) {
+      LOGGER.warn("Failed to parse {} segment load status responses from server.", failedParses);


Suggested change

LOGGER.warn("Failed to parse {} segment load status responses from server.", failedParses);

LOGGER.warn("Failed to parse segment load status responses from {} servers.", failedParses);

...t-controller/src/main/java/org/apache/pinot/controller/util/ServerSegmentMetadataReader.java

mcvsubbu · 2020-08-20T16:14:41Z

pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/SegmentStatus.java

+  public String _segmentName;
+  // The last segment reload time in ISO date format (yyyy-MM-dd HH:mm:ss:SSS UTC)
+  // If the segment reload failed for a segment, then the value will be the previous segment reload was successful
+  public String _segmentReloadTimeUTC;


Why is this a String and not long?

The date is in string format: ISO date format (yyyy-MM-dd HH:mm:ss:SSS UTC)

I have updated the API to return long instead of String

...ler/src/main/java/org/apache/pinot/controller/api/resources/PinotSegmentRestletResource.java

guruguha · 2020-08-27T03:43:22Z

@mcvsubbu can you please review again?

Updating variable names to reflect their value type

codecov-commenter · 2020-09-02T04:51:10Z

Codecov Report

❗ No coverage uploaded for pull request base (master@7708341). Click here to learn what that means.
The diff coverage is 62.02%.

@@            Coverage Diff            @@
##             master    #5718   +/-   ##
=========================================
  Coverage          ?   67.15%           
=========================================
  Files             ?     1205           
  Lines             ?    63599           
  Branches          ?     9741           
=========================================
  Hits              ?    42708           
  Misses            ?    17747           
  Partials          ?     3144

Flag	Coverage Δ
#integration	`43.55% <0.63%> (?)`
#unittests	`58.36% <62.02%> (?)`

Flags with carried forward coverage won't be shown. Click here to find out more.

Impacted Files	Coverage Δ
...che/pinot/controller/util/TableMetadataReader.java	`0.00% <0.00%> (ø)`
...ler/api/resources/PinotSegmentRestletResource.java	`14.19% <5.88%> (ø)`
...che/pinot/server/api/resources/TablesResource.java	`72.83% <25.00%> (ø)`
...ontroller/api/resources/ServerTableSizeReader.java	`83.87% <58.33%> (ø)`
...t/server/api/resources/SegmentMetadataFetcher.java	`70.45% <70.45%> (ø)`
...t/controller/util/ServerSegmentMetadataReader.java	`84.37% <84.37%> (ø)`
...pinot/controller/util/CompletionServiceHelper.java	`93.75% <93.75%> (ø)`
...e/indexsegment/immutable/ImmutableSegmentImpl.java	`74.35% <100.00%> (ø)`
...pinot/core/util/IntDoubleIndexedPriorityQueue.java	`85.36% <0.00%> (ø)`
...java/org/apache/pinot/spi/utils/DataSizeUtils.java	`88.23% <0.00%> (ø)`
... and 1203 more

Continue to review full report at Codecov.

Legend - Click here to learn more
Δ = absolute <relative> (impact), ø = not affected, ? = missing data
Powered by Codecov. Last update 7708341...b36d4cf. Read the comment docs.

Code refactor to clean up lines that went beyond line length

kishoreg

This is amazing! Very well done

Most of the changes are addressed

guruguha added 2 commits July 19, 2020 21:49

- initial feature push

e9a1c93

guruguha force-pushed the feature/#5390-segment-indexing-reload-status-api branch from 6914c32 to 422c76a Compare July 21, 2020 01:50

guruguha force-pushed the feature/#5390-segment-indexing-reload-status-api branch from 422c76a to 7e31c11 Compare July 21, 2020 02:44

npawar reviewed Jul 23, 2020

View reviewed changes

...ler/src/main/java/org/apache/pinot/controller/api/resources/PinotSegmentRestletResource.java Outdated Show resolved Hide resolved

...ler/src/main/java/org/apache/pinot/controller/api/resources/PinotSegmentRestletResource.java Outdated Show resolved Hide resolved

npawar reviewed Jul 26, 2020

View reviewed changes

guruguha added 2 commits July 26, 2020 01:38

Code Review Changes

a2e8ac6

- Updating code as per PR review comments

Adding comments to new classes and methods added as part of this feature

1cea905

Removing SegmentMetadataFetcher as it seemed redundant Refactoring code to save failed segment reload status API calls as part of response

mcvsubbu previously requested changes Jul 29, 2020

View reviewed changes

kishoreg approved these changes Aug 12, 2020

View reviewed changes

guruguha added 2 commits August 11, 2020 22:28

Code refactor to accommodate PR comments

d2d1c84

Pinot codestyle corrections Moving ServerSegmentMetadataReader to util

Updating API definition for loadStatus

d14f143

npawar approved these changes Aug 12, 2020

View reviewed changes

guruguha added 3 commits August 13, 2020 13:26

Code refactor to remove duplicate code

b5b7d1c

Code refactor to remove duplicate code

8ee255f

Merge remote-tracking branch 'origin/feature/apache#5390-segment-inde…

2198469

…xing-reload-status-api' into feature/apache#5390-segment-indexing-reload-status-api # Conflicts: # pinot-controller/src/main/java/org/apache/pinot/controller/util/CompletionServiceHelper.java

kishoreg requested a review from mcvsubbu August 17, 2020 03:17

mcvsubbu reviewed Aug 20, 2020

View reviewed changes

Code refactor as per PR comments

6e03c9b

guruguha force-pushed the feature/#5390-segment-indexing-reload-status-api branch from f3e67e7 to 6e03c9b Compare August 22, 2020 15:33

guruguha added 3 commits August 28, 2020 13:21

Updating segment loadStatus API to return long time than readable string

2a1cc16

- Bug fix on the server API endpoint

563e56c

Adding pretty print for server metadata response

d234345

guruguha added 2 commits August 31, 2020 19:56

Reverting incorrect filename refactoring

d13c91e

Updating variable names to reflect their value type

removing unused method variable

ac98a41

guruguha force-pushed the feature/#5390-segment-indexing-reload-status-api branch from f26a5aa to ac98a41 Compare September 2, 2020 04:17

guruguha and others added 4 commits September 8, 2020 18:06

Enabling Pretty print of server response.

2cf5629

Code refactor to clean up lines that went beyond line length

Removing reload status API

b36d4cf

Pretty print result

3f98c99

Fix test

ec3603b

npawar force-pushed the feature/#5390-segment-indexing-reload-status-api branch from ae2ce79 to ec3603b Compare September 12, 2020 00:32

kishoreg approved these changes Sep 12, 2020

View reviewed changes

kishoreg merged commit dc77271 into apache:master Sep 14, 2020

This was referenced Apr 9, 2021

Add star-tree and json index to the output of /segments/{tableName}/metadata API #6766

Closed

Support REALTIME tables in reload status (segment metadata API) #6767

Closed

	LOGGER.warn("Failed to parse {} segment size info responses from server.", failedParses);
	LOGGER.warn("Failed to parse segment size info responses from {} servers.", failedParses);

	numServersResponded, serverURLs);
	numServersResponded, serverURLs.size());

	LOGGER.warn("Failed to parse {} segment load status responses from server.", failedParses);
	LOGGER.warn("Failed to parse segment load status responses from {} servers.", failedParses);

Feature/#5390 segment indexing reload status api #5718

Feature/#5390 segment indexing reload status api #5718

Conversation

guruguha commented Jul 20, 2020 • edited by npawar

Description

Documentation

npawar left a comment

Choose a reason for hiding this comment

mcvsubbu left a comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

guruguha Aug 1, 2020 • edited

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

guruguha commented Aug 1, 2020 via email

kishoreg commented Aug 9, 2020

guruguha commented Aug 9, 2020 via email

npawar left a comment

Choose a reason for hiding this comment

kishoreg commented Aug 13, 2020

guruguha commented Aug 13, 2020

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

guruguha commented Aug 27, 2020

codecov-commenter commented Sep 2, 2020 • edited

Codecov Report

kishoreg left a comment

Choose a reason for hiding this comment

guruguha commented Jul 20, 2020 •

edited by npawar

guruguha Aug 1, 2020 •

edited

codecov-commenter commented Sep 2, 2020 •

edited