Repo history rewrite (bluesky-social#1479)

* logical changes in repo * tests * tweak commit data * building pds * patching up some tests * tidy + more tests * patch up bsky * clean up db * db migration * small patches * fix up another test * reinclude prevs * api & lex updates * add back in deprecated routes * backward compatibility for commit v2 * add upgrade repo version root * move namespace * migration test * patch up a few more tests * remove deprecated rebase routes * tweak api * sprinkle rev around a few more places * getCurrent -> getLatestCommit * fix test * pr feedback & tidy * fix up block pagination * tidy again * add in a tets * add cursor to listBlobs * getRepo rev -> since * clean up proofs test * dont change getHead * tweak migrate route * build branch * hit index in block range query * check for dupe record refs * tidy * set prev to null * dont build branch
mloar · Sep 25, 2023 · 490dcf2 · 490dcf2
1 parent e5d275d
commit 490dcf2
Show file tree

Hide file tree

Showing 102 changed files with 1,853 additions and 3,132 deletions.
diff --git a/lexicons/com/atproto/admin/rebaseRepo.json b/lexicons/com/atproto/admin/rebaseRepo.json
diff --git a/lexicons/com/atproto/repo/rebaseRepo.json b/lexicons/com/atproto/repo/rebaseRepo.json
diff --git a/lexicons/com/atproto/sync/getCheckout.json b/lexicons/com/atproto/sync/getCheckout.json
@@ -4,7 +4,7 @@
   "defs": {
     "main": {
       "type": "query",
-      "description": "Gets the repo state.",
+      "description": "DEPRECATED - please use com.atproto.sync.getRepo instead",
       "parameters": {
         "type": "params",
         "required": ["did"],
@@ -13,11 +13,6 @@
             "type": "string",
             "format": "did",
             "description": "The DID of the repo."
-          },
-          "commit": {
-            "type": "string",
-            "format": "cid",
-            "description": "The commit to get the checkout from. Defaults to current HEAD."
           }
         }
       },

diff --git a/lexicons/com/atproto/sync/getCommitPath.json b/lexicons/com/atproto/sync/getCommitPath.json
diff --git a/lexicons/com/atproto/sync/getHead.json b/lexicons/com/atproto/sync/getHead.json
@@ -4,7 +4,7 @@
   "defs": {
     "main": {
       "type": "query",
-      "description": "Gets the current HEAD CID of a repo.",
+      "description": "DEPRECATED - please use com.atproto.sync.getLatestCommit instead",
       "parameters": {
         "type": "params",
         "required": ["did"],

diff --git a/lexicons/com/atproto/sync/getLatestCommit.json b/lexicons/com/atproto/sync/getLatestCommit.json
@@ -0,0 +1,35 @@
+{
+  "lexicon": 1,
+  "id": "com.atproto.sync.getLatestCommit",
+  "defs": {
+    "main": {
+      "type": "query",
+      "description": "Gets the current commit CID & revision of the repo.",
+      "parameters": {
+        "type": "params",
+        "required": ["did"],
+        "properties": {
+          "did": {
+            "type": "string",
+            "format": "did",
+            "description": "The DID of the repo."
+          }
+        }
+      },
+      "output": {
+        "encoding": "application/json",
+        "schema": {
+          "type": "object",
+          "required": ["cid", "rev"],
+          "properties": {
+            "cid": {"type": "string", "format": "cid"},
+            "rev": {"type": "string"}
+          }
+        }
+      },
+      "errors": [
+        {"name": "RepoNotFound"}
+      ]
+    }
+  }
+}
diff --git a/lexicons/com/atproto/sync/getRepo.json b/lexicons/com/atproto/sync/getRepo.json
@@ -4,7 +4,7 @@
   "defs": {
     "main": {
       "type": "query",
-      "description": "Gets the repo state.",
+      "description": "Gets the did's repo, optionally catching up from a specific revision.",
       "parameters": {
         "type": "params",
         "required": ["did"],
@@ -14,15 +14,10 @@
             "format": "did",
             "description": "The DID of the repo."
           },
-          "earliest": {
+          "since": {
             "type": "string",
             "format": "cid",
-            "description": "The earliest commit in the commit range (not inclusive)"
-          },
-          "latest": {
-            "type": "string",
-            "format": "cid",
-            "description": "The latest commit in the commit range (inclusive)"
+            "description": "The revision of the repo to catch up from."
           }
         }
       },

diff --git a/lexicons/com/atproto/sync/listBlobs.json b/lexicons/com/atproto/sync/listBlobs.json
@@ -4,14 +4,15 @@
   "defs": {
     "main": {
       "type": "query",
-      "description": "List blob cids for some range of commits",
+      "description": "List blob cids since some revision",
       "parameters": {
         "type": "params",
         "required": ["did"],
         "properties": {
           "did": {"type": "string", "format": "did", "description": "The DID of the repo."},
-          "latest": { "type": "string", "format": "cid", "description": "The most recent commit"},
-          "earliest": { "type": "string", "format": "cid", "description": "The earliest commit to start from"}
+          "since": { "type": "string", "format": "cid", "description": "Optional revision of the repo to list blobs since"},
+          "limit": {"type": "integer", "minimum": 1, "maximum": 1000, "default": 500},
+          "cursor": {"type": "string"}
         }
       },
       "output": {
@@ -20,6 +21,7 @@
           "type": "object",
           "required": ["cids"],
           "properties": {
+            "cursor": {"type": "string"},
             "cids": {
               "type": "array",
               "items": { "type": "string", "format": "cid" }

diff --git a/lexicons/com/atproto/sync/subscribeRepos.json b/lexicons/com/atproto/sync/subscribeRepos.json
@@ -33,15 +33,23 @@
     },
     "commit": {
       "type": "object",
-      "required": ["seq", "rebase", "tooBig", "repo", "commit", "prev", "blocks", "ops", "blobs", "time"],
-      "nullable": ["prev"],
+      "required": ["seq", "rebase", "tooBig", "repo", "commit", "rev", "since", "blocks", "ops", "blobs", "time"],
+      "nullable": ["prev", "since"],
       "properties": {
         "seq": {"type": "integer"},
         "rebase": {"type": "boolean"},
         "tooBig": {"type": "boolean"},
         "repo": {"type": "string", "format": "did"},
         "commit": {"type": "cid-link"},
         "prev": {"type": "cid-link"},
+        "rev": {
+          "type": "string",
+          "description": "The rev of the emitted commit"
+        },
+        "since": {
+          "type": "string",
+          "description": "The rev of the last emitted commit from this repo"
+        },
         "blocks": {
           "type": "bytes",
           "description": "CAR file containing relevant blocks",

diff --git a/lexicons/com/atproto/temp/upgradeRepoVersion.json b/lexicons/com/atproto/temp/upgradeRepoVersion.json
@@ -0,0 +1,20 @@
+{
+  "lexicon": 1,
+  "id": "com.atproto.temp.upgradeRepoVersion",
+  "defs": {
+    "main": {
+      "type": "procedure",
+      "description": "Upgrade a repo to v3",
+      "input": {
+        "encoding": "application/json",
+        "schema": {
+          "type": "object",
+          "required": ["did"],
+          "properties": {
+            "did": { "type": "string", "format": "did" }
+          }
+        }
+      }
+    }
+  }
+}