Ability to update the repository description (#7376)

versioned/storage/common-tests/src/main/java/org/projectnessie/versioned/storage/commontests/AbstractRepositoryLogicTests.java

@@ -45,8 +45,10 @@
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.ValueSource;
 import org.projectnessie.versioned.storage.common.exceptions.RefNotFoundException;
+import org.projectnessie.versioned.storage.common.exceptions.RetryTimeoutException;
 import org.projectnessie.versioned.storage.common.logic.CommitLogic;
 import org.projectnessie.versioned.storage.common.logic.CreateCommit;
+import org.projectnessie.versioned.storage.common.logic.ImmutableRepositoryDescription;
 import org.projectnessie.versioned.storage.common.logic.InternalRef;
 import org.projectnessie.versioned.storage.common.logic.ReferenceLogic;
 import org.projectnessie.versioned.storage.common.logic.RepositoryDescription;
@@ -234,4 +236,33 @@ public void internalRefsAndDefaultBranch(String defaultBranchName) {
     soft.assertThat(newHashSet(referenceLogic.queryReferences(referencesQuery())))
         .isEqualTo(refQuery);
   }
+
+  @Test
+  public void updateRepositoryDescription() throws RetryTimeoutException {
+    RepositoryLogic repositoryLogic = repositoryLogic(persist);
+
+    repositoryLogic.initialize("main");
+
+    RepositoryDescription initial = requireNonNull(repositoryLogic.fetchRepositoryDescription());
+
+    RepositoryDescription updated =
+        RepositoryDescription.builder()
+            .putProperties("updated", "true")
+            .defaultBranchName("main2")
+            // the following attributes are read-only, should not be updated
+            .oldestPossibleCommitTime(Instant.ofEpochSecond(12345))
+            .repositoryCreatedTime(Instant.ofEpochSecond(456789))
+            .build();
+
+    RepositoryDescription previous = repositoryLogic.updateRepositoryDescription(updated);
+
+    soft.assertThat(previous).isEqualTo(initial);
+    soft.assertThat(repositoryLogic.fetchRepositoryDescription())
+        .isEqualTo(
+            ImmutableRepositoryDescription.builder()
+                .from(initial)
+                .putProperties("updated", "true")
+                .defaultBranchName("main2")
+                .build());
+  }
 }

versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/logic/InternalRef.java

@@ -54,8 +54,7 @@ static InternalRef internalReference(String name) {
   InternalRef REF_REFS = internalReference("refs");
 
   /**
-   * Internal reference with always exactly one commit that serves as a reference for when the
-   * repository has been created.
+   * Internal reference that points to the current {@link RepositoryDescription}.
    *
    * <p>The actual information is available via {@link #KEY_REPO_DESCRIPTION} from the HEAD commit.
    */

versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/logic/RepositoryLogic.java

@@ -18,6 +18,7 @@
 import java.util.function.Consumer;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
+import org.projectnessie.versioned.storage.common.exceptions.RetryTimeoutException;
 
 /** Logic to setup/initialize a Nessie repository. */
 public interface RepositoryLogic {
@@ -33,5 +34,18 @@ void initialize(
   @jakarta.annotation.Nullable
   RepositoryDescription fetchRepositoryDescription();
 
+  /**
+   * Updates the repository description, and returns the previous description, or {@code null} if
+   * there was no previous description.
+   *
+   * @param repositoryDescription the new description.
+   * @return the previous description, or {@code null} if there was no previous description.
+   * @throws RetryTimeoutException if the update failed after all retries.
+   */
+  @Nullable
+  @jakarta.annotation.Nullable
+  RepositoryDescription updateRepositoryDescription(RepositoryDescription repositoryDescription)
+      throws RetryTimeoutException;
+
   boolean repositoryExists();
 }

versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/logic/RepositoryLogicImpl.java

@@ -17,6 +17,7 @@
 
 import static java.util.Collections.emptyList;
 import static java.util.Objects.requireNonNull;
+import static org.projectnessie.versioned.storage.common.logic.CommitRetry.commitRetry;
 import static org.projectnessie.versioned.storage.common.logic.CreateCommit.Add.commitAdd;
 import static org.projectnessie.versioned.storage.common.logic.CreateCommit.newCommitBuilder;
 import static org.projectnessie.versioned.storage.common.logic.InternalRef.KEY_REPO_DESCRIPTION;
@@ -38,12 +39,16 @@
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 import org.projectnessie.versioned.storage.common.exceptions.CommitConflictException;
+import org.projectnessie.versioned.storage.common.exceptions.CommitWrappedException;
 import org.projectnessie.versioned.storage.common.exceptions.ObjNotFoundException;
 import org.projectnessie.versioned.storage.common.exceptions.ObjTooLargeException;
 import org.projectnessie.versioned.storage.common.exceptions.RefAlreadyExistsException;
+import org.projectnessie.versioned.storage.common.exceptions.RefConditionFailedException;
+import org.projectnessie.versioned.storage.common.exceptions.RefNotFoundException;
 import org.projectnessie.versioned.storage.common.exceptions.RetryTimeoutException;
 import org.projectnessie.versioned.storage.common.indexes.StoreIndex;
 import org.projectnessie.versioned.storage.common.indexes.StoreIndexElement;
+import org.projectnessie.versioned.storage.common.logic.CommitRetry.RetryException;
 import org.projectnessie.versioned.storage.common.logic.StringLogic.StringValue;
 import org.projectnessie.versioned.storage.common.objtypes.CommitObj;
 import org.projectnessie.versioned.storage.common.objtypes.CommitOp;
@@ -135,20 +140,28 @@ public RepositoryDescription fetchRepositoryDescription() {
                   requireNonNull(
                       op.value(), "Commit operation for repository description has no value"));
 
-      return readRepositoryDescription(value);
+      return deserialize(value);
     } catch (ObjNotFoundException e) {
       return null;
     }
   }
 
-  private RepositoryDescription readRepositoryDescription(StringValue value) {
+  private static RepositoryDescription deserialize(StringValue value) {
     try {
       return SHARED_OBJECT_MAPPER.readValue(value.completeValue(), RepositoryDescription.class);
     } catch (ObjNotFoundException | IOException e) {
       throw new RuntimeException(e);
     }
   }
 
+  private static byte[] serialize(RepositoryDescription repositoryDescription) {
+    try {
+      return SHARED_OBJECT_MAPPER.writeValueAsBytes(repositoryDescription);
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
   private void addRepositoryDescription(
       CreateCommit.Builder b,
       Consumer<RepositoryDescription.Builder> repositoryDescription,
@@ -169,14 +182,59 @@ private void addRepositoryDescription(
                   null,
                   "application/json",
                   SHARED_OBJECT_MAPPER.writeValueAsBytes(repoDesc.build()));
-      // can safely ignore the ID returned from storeObj() - it's fine, if the obj already exists
+      // can safely ignore the response from storeObj() - it's fine, if the obj already exists
       persist.storeObj(string);
       b.addAdds(commitAdd(KEY_REPO_DESCRIPTION, 0, requireNonNull(string.id()), null, null));
     } catch (ObjTooLargeException | ObjNotFoundException | IOException e) {
       throw new RuntimeException(e);
     }
   }
 
+  @Override
+  public RepositoryDescription updateRepositoryDescription(RepositoryDescription newDescription)
+      throws RetryTimeoutException {
+    // prevent modification of read-only attributes
+    RepositoryDescription existingDescription = requireNonNull(fetchRepositoryDescription());
+    RepositoryDescription sanitizedDescription =
+        ImmutableRepositoryDescription.builder()
+            .from(newDescription)
+            .oldestPossibleCommitTime(existingDescription.oldestPossibleCommitTime())
+            .repositoryCreatedTime(existingDescription.repositoryCreatedTime())
+            .build();
+    byte[] serialized = serialize(sanitizedDescription);
+    try {
+      StringValue existing =
+          commitRetry(
+              persist,
+              (p, retryState) -> {
+                try {
+                  Reference reference = requireNonNull(persist.fetchReference(REF_REPO.name()));
+                  return stringLogic(persist)
+                      .updateStringOnRef(
+                          reference,
+                          KEY_REPO_DESCRIPTION,
+                          b ->
+                              b.message("Update repository description")
+                                  .commitType(CommitType.INTERNAL),
+                          "application/json",
+                          serialized);
+                } catch (RefConditionFailedException | CommitConflictException e) {
+                  throw new RetryException();
+                } catch (ObjNotFoundException | RefNotFoundException e) {
+                  throw new CommitWrappedException(e);
+                }
+              });
+      return existing != null ? deserialize(existing) : null;
+    } catch (CommitConflictException e) {
+      throw new RuntimeException(
+          "An unexpected internal error happened while committing a repository description update");
+    } catch (CommitWrappedException e) {
+      throw new RuntimeException(
+          "An unexpected internal error happened while committing a repository description update",
+          e.getCause());
+    }
+  }
+
   @SuppressWarnings({"JavaTimeDefaultTimeZone"})
   private void initializeInternalRef(
       InternalRef internalRef, Consumer<CreateCommit.Builder> commitEnhancer) {

versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/logic/StringLogic.java

@@ -15,9 +15,17 @@
  */
 package org.projectnessie.versioned.storage.common.logic;
 
+import java.util.function.Consumer;
+import javax.annotation.Nullable;
+import org.projectnessie.versioned.storage.common.exceptions.CommitConflictException;
 import org.projectnessie.versioned.storage.common.exceptions.ObjNotFoundException;
+import org.projectnessie.versioned.storage.common.exceptions.RefConditionFailedException;
+import org.projectnessie.versioned.storage.common.exceptions.RefNotFoundException;
+import org.projectnessie.versioned.storage.common.indexes.StoreKey;
+import org.projectnessie.versioned.storage.common.logic.CreateCommit.Builder;
 import org.projectnessie.versioned.storage.common.objtypes.StringObj;
 import org.projectnessie.versioned.storage.common.persist.ObjId;
+import org.projectnessie.versioned.storage.common.persist.Reference;
 
 /**
  * Provides and encapsulates all logic around string data compression and diff creation and
@@ -34,6 +42,30 @@ StringObj updateString(StringValue previousValue, String contentType, byte[] str
   StringObj updateString(StringValue previousValue, String contentType, String stringValue)
       throws ObjNotFoundException;
 
+  /**
+   * Updates a string value stored as a content on a reference.
+   *
+   * @param reference The reference to update.
+   * @param storeKey The store key to use for storing the string value.
+   * @param commitEnhancer A consumer that can be used to enhance the commit that will be created.
+   * @param contentType The content type of the string value.
+   * @param stringValueUtf8 The string value as UTF-8 encoded byte array.
+   * @return The previously stored string value, if any, or {@code null} if no string value was
+   *     stored on the reference.
+   */
+  @Nullable
+  @jakarta.annotation.Nullable
+  StringValue updateStringOnRef(
+      Reference reference,
+      StoreKey storeKey,
+      Consumer<Builder> commitEnhancer,
+      String contentType,
+      byte[] stringValueUtf8)
+      throws ObjNotFoundException,
+          CommitConflictException,
+          RefNotFoundException,
+          RefConditionFailedException;
+
   /**
    * Describes a string value and allows retrieval of complete string values. Implementations can
    * lazily decompress data and/or re-assemble values that consist of diffs.

versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/logic/StringLogicImpl.java

@@ -17,17 +17,35 @@
 
 import static com.google.common.base.Preconditions.checkState;
 import static java.util.Collections.emptyList;
+import static java.util.Collections.singletonList;
+import static java.util.Objects.requireNonNull;
 import static org.projectnessie.nessie.relocated.protobuf.UnsafeByteOperations.unsafeWrap;
+import static org.projectnessie.versioned.storage.common.logic.CreateCommit.Add.commitAdd;
+import static org.projectnessie.versioned.storage.common.logic.Logics.commitLogic;
+import static org.projectnessie.versioned.storage.common.logic.Logics.indexesLogic;
+import static org.projectnessie.versioned.storage.common.objtypes.CommitHeaders.EMPTY_COMMIT_HEADERS;
 import static org.projectnessie.versioned.storage.common.objtypes.StringObj.stringData;
 import static org.projectnessie.versioned.storage.common.persist.ObjType.STRING;
 
 import java.nio.charset.StandardCharsets;
+import java.util.UUID;
+import java.util.function.Consumer;
 import org.projectnessie.nessie.relocated.protobuf.ByteString;
+import org.projectnessie.versioned.storage.common.exceptions.CommitConflictException;
 import org.projectnessie.versioned.storage.common.exceptions.ObjNotFoundException;
+import org.projectnessie.versioned.storage.common.exceptions.RefConditionFailedException;
+import org.projectnessie.versioned.storage.common.exceptions.RefNotFoundException;
+import org.projectnessie.versioned.storage.common.indexes.StoreIndex;
+import org.projectnessie.versioned.storage.common.indexes.StoreIndexElement;
+import org.projectnessie.versioned.storage.common.indexes.StoreKey;
+import org.projectnessie.versioned.storage.common.logic.CreateCommit.Builder;
+import org.projectnessie.versioned.storage.common.objtypes.CommitObj;
+import org.projectnessie.versioned.storage.common.objtypes.CommitOp;
 import org.projectnessie.versioned.storage.common.objtypes.Compression;
 import org.projectnessie.versioned.storage.common.objtypes.StringObj;
 import org.projectnessie.versioned.storage.common.persist.ObjId;
 import org.projectnessie.versioned.storage.common.persist.Persist;
+import org.projectnessie.versioned.storage.common.persist.Reference;
 
 final class StringLogicImpl implements StringLogic {
   private final Persist persist;
@@ -68,6 +86,54 @@ public StringObj updateString(StringValue previousValue, String contentType, Str
     return updateString(previousValue, contentType, stringValue.getBytes(StandardCharsets.UTF_8));
   }
 
+  @Override
+  public StringValue updateStringOnRef(
+      Reference reference,
+      StoreKey storeKey,
+      Consumer<Builder> commitEnhancer,
+      String contentType,
+      byte[] stringValueUtf8)
+      throws ObjNotFoundException,
+          CommitConflictException,
+          RefNotFoundException,
+          RefConditionFailedException {
+    CommitLogic commitLogic = commitLogic(persist);
+    IndexesLogic indexesLogic = indexesLogic(persist);
+    CommitObj head = commitLogic.headCommit(reference);
+    StoreIndex<CommitOp> index = indexesLogic.buildCompleteIndexOrEmpty(head);
+    StoreIndexElement<CommitOp> existingElement = index.get(storeKey);
+
+    // If we are updating an existing string, reuse its content-id (which may be null,
+    // e.g. for repo descriptions). Otherwise, generate a new content-id.
+    ObjId existingValueId = null;
+    UUID contentId = null;
+    if (existingElement != null) {
+      CommitOp op = existingElement.content();
+      if (op.action().exists()) {
+        existingValueId = op.value();
+        contentId = op.contentId();
+      }
+    } else {
+      contentId = UUID.randomUUID();
+    }
+
+    StringValue existing = existingValueId != null ? fetchString(existingValueId) : null;
+    StringObj newValue = updateString(existing, contentType, stringValueUtf8);
+
+    ObjId newValueId = requireNonNull(newValue.id());
+    if (!newValueId.equals(existingValueId)) {
+      CreateCommit.Builder builder =
+          CreateCommit.newCommitBuilder()
+              .parentCommitId(reference.pointer())
+              .headers(EMPTY_COMMIT_HEADERS)
+              .addAdds(commitAdd(storeKey, 0, newValueId, existingValueId, contentId));
+      commitEnhancer.accept(builder);
+      CommitObj committed = commitLogic.doCommit(builder.build(), singletonList(newValue));
+      persist.updateReferencePointer(reference, requireNonNull(committed).id());
+    }
+    return existing;
+  }
+
   static final class StringValueHolder implements StringValue {
     final StringObj obj;