From fba52f55fadc69593fe63759d827441a5364fdd7 Mon Sep 17 00:00:00 2001 From: "Artem V. Navrotskiy" Date: Fri, 8 Jan 2016 22:51:26 +0300 Subject: [PATCH 1/2] Modify object caching for support glob path as arguments --- .../java/git/lfs/migrate/GitConverter.java | 51 +++++++++++-------- src/main/java/git/lfs/migrate/Main.java | 4 +- src/main/java/git/lfs/migrate/TaskKey.java | 25 +++++++-- 3 files changed, 53 insertions(+), 27 deletions(-) diff --git a/src/main/java/git/lfs/migrate/GitConverter.java b/src/main/java/git/lfs/migrate/GitConverter.java index e279a1e..3c21cce 100644 --- a/src/main/java/git/lfs/migrate/GitConverter.java +++ b/src/main/java/git/lfs/migrate/GitConverter.java @@ -73,7 +73,7 @@ public ConvertTask convertTask(@NotNull ObjectReader reader, @NotNull TaskKey ke return convertCommitTask((RevCommit) revObject); } if (revObject instanceof RevTree) { - return convertTreeTask(reader, revObject, false); + return convertTreeTask(reader, revObject, Objects.requireNonNull(key.getPath())); } if (revObject instanceof RevBlob) { return copyTask(reader, revObject); @@ -83,13 +83,6 @@ public ConvertTask convertTask(@NotNull ObjectReader reader, @NotNull TaskKey ke } throw new IllegalStateException("Unsupported object type: " + key + " (" + revObject.getClass().getName() + ")"); } - case Root: { - final RevObject revObject = new RevWalk(reader).parseAny(key.getObjectId()); - if (revObject instanceof RevTree) { - return convertTreeTask(reader, revObject, true); - } - throw new IllegalStateException("Unsupported object type: " + key + " (" + revObject.getClass().getName() + ")"); - } case Attribute: return createAttributesTask(reader, key.getObjectId()); case UploadLfs: @@ -122,14 +115,14 @@ private ConvertTask convertTagTask(@NotNull RevTag revObject) throws IOException @Override public Iterable depends() { return Collections.singletonList( - new TaskKey(TaskType.Simple, revObject.getObject()) + new TaskKey(TaskType.Simple, "", revObject.getObject()) ); } @NotNull @Override public ObjectId convert(@NotNull ObjectInserter inserter, @NotNull ConvertResolver resolver, @Nullable Uploader uploader) throws IOException { - final ObjectId id = resolver.resolve(TaskType.Simple, revObject.getObject()); + final ObjectId id = resolver.resolve(TaskType.Simple, "", revObject.getObject()); final TagBuilder builder = new TagBuilder(); builder.setMessage(revObject.getFullMessage()); builder.setTag(revObject.getTagName()); @@ -148,9 +141,9 @@ private ConvertTask convertCommitTask(@NotNull RevCommit revObject) throws IOExc public Iterable depends() { List result = new ArrayList<>(); for (RevCommit parent : revObject.getParents()) { - result.add(new TaskKey(TaskType.Simple, parent)); + result.add(new TaskKey(TaskType.Simple, "", parent)); } - result.add(new TaskKey(TaskType.Root, revObject.getTree())); + result.add(new TaskKey(TaskType.Simple, "", revObject.getTree())); return result; } @@ -164,39 +157,43 @@ public ObjectId convert(@NotNull ObjectInserter inserter, @NotNull ConvertResolv builder.setMessage(revObject.getFullMessage()); // Set parents for (RevCommit oldParent : revObject.getParents()) { - builder.addParentId(resolver.resolve(TaskType.Simple, oldParent)); + builder.addParentId(resolver.resolve(TaskType.Simple, "", oldParent)); } // Set tree - builder.setTreeId(resolver.resolve(TaskType.Root, revObject.getTree())); + builder.setTreeId(resolver.resolve(TaskType.Simple, "", revObject.getTree())); return inserter.insert(builder); } }; } @NotNull - private ConvertTask convertTreeTask(@NotNull ObjectReader reader, @NotNull ObjectId id, boolean rootTree) { + private ConvertTask convertTreeTask(@NotNull ObjectReader reader, @NotNull ObjectId id, @NotNull String path) { return new ConvertTask() { @NotNull private List getEntries() throws IOException { final List entries = new ArrayList<>(); final CanonicalTreeParser treeParser = new CanonicalTreeParser(null, reader, id); - boolean needAttributes = rootTree; + boolean needAttributes = path.isEmpty(); while (!treeParser.eof()) { final FileMode fileMode = treeParser.getEntryFileMode(); final TaskType blobTask; + final String pathTask; if (needAttributes && treeParser.getEntryPathString().equals(GIT_ATTRIBUTES)) { blobTask = TaskType.Attribute; + pathTask = null; needAttributes = false; } else if (isFile(fileMode) && matchFilename(treeParser.getEntryPathString())) { blobTask = TaskType.UploadLfs; + pathTask = null; } else { blobTask = TaskType.Simple; + pathTask = path + "/" + treeParser.getEntryPathString(); } - entries.add(new GitTreeEntry(fileMode, new TaskKey(blobTask, treeParser.getEntryObjectId()), treeParser.getEntryPathString())); + entries.add(new GitTreeEntry(fileMode, new TaskKey(blobTask, pathTask, treeParser.getEntryObjectId()), treeParser.getEntryPathString())); treeParser.next(); } if (needAttributes && suffixes.length > 0) { - entries.add(new GitTreeEntry(FileMode.REGULAR_FILE, new TaskKey(TaskType.Attribute, ObjectId.zeroId()), GIT_ATTRIBUTES)); + entries.add(new GitTreeEntry(FileMode.REGULAR_FILE, new TaskKey(TaskType.Attribute, null, ObjectId.zeroId()), GIT_ATTRIBUTES)); } return entries; } @@ -410,7 +407,19 @@ private InputStream openAttributes(@NotNull ObjectReader reader, @Nullable Objec } public enum TaskType { - Simple, Root, Attribute, UploadLfs, + Simple(true), + Attribute(false), + UploadLfs(false); + + TaskType(boolean needPath) { + this.needPath = needPath; + } + + private final boolean needPath; + + public boolean needPath() { + return needPath; + } } public interface ConvertResolver { @@ -418,8 +427,8 @@ public interface ConvertResolver { ObjectId resolve(@NotNull TaskKey key); @NotNull - default ObjectId resolve(@NotNull TaskType type, @NotNull ObjectId objectId) { - return resolve(new TaskKey(type, objectId)); + default ObjectId resolve(@NotNull TaskType type, @Nullable String path, @NotNull ObjectId objectId) { + return resolve(new TaskKey(type, path, objectId)); } } diff --git a/src/main/java/git/lfs/migrate/Main.java b/src/main/java/git/lfs/migrate/Main.java index 8a4a627..72c07c9 100644 --- a/src/main/java/git/lfs/migrate/Main.java +++ b/src/main/java/git/lfs/migrate/Main.java @@ -150,7 +150,7 @@ public static void processRepository(@NotNull File srcPath, @NotNull File dstPat for (Map.Entry ref : srcRepo.getAllRefs().entrySet()) { RefUpdate refUpdate = dstRepo.updateRef(ref.getKey()); final ObjectId oldId = ref.getValue().getObjectId(); - final ObjectId newId = converted.get(new TaskKey(GitConverter.TaskType.Simple, oldId)); + final ObjectId newId = converted.get(new TaskKey(GitConverter.TaskType.Simple, "", oldId)); refUpdate.setNewObjectId(newId); refUpdate.update(); log.info(" convert ref: {} -> {} ({})", oldId.getName(), newId.getName(), ref.getKey()); @@ -269,7 +269,7 @@ private static SimpleDirectedGraph loadTaskGraph(@NotNull final Deque queue = new ArrayDeque<>(); // Heads for (Ref ref : refs.values()) { - final TaskKey taskKey = new TaskKey(GitConverter.TaskType.Simple, ref.getObjectId()); + final TaskKey taskKey = new TaskKey(GitConverter.TaskType.Simple, "", ref.getObjectId()); if (graph.addVertex(taskKey)) { queue.add(taskKey); reporter.increment(); diff --git a/src/main/java/git/lfs/migrate/TaskKey.java b/src/main/java/git/lfs/migrate/TaskKey.java index 15bba5d..ba2da6b 100644 --- a/src/main/java/git/lfs/migrate/TaskKey.java +++ b/src/main/java/git/lfs/migrate/TaskKey.java @@ -2,6 +2,9 @@ import org.eclipse.jgit.lib.ObjectId; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.util.Objects; /** * Key of converter task. @@ -14,10 +17,16 @@ public final class TaskKey { private final GitConverter.TaskType type; @NotNull private final ObjectId objectId; + @Nullable + private final String path; - public TaskKey(@NotNull GitConverter.TaskType type, @NotNull ObjectId objectId) { + public TaskKey(@NotNull GitConverter.TaskType type, @Nullable String path, @NotNull ObjectId objectId) { this.type = type; + this.path = path; this.objectId = objectId.copy(); + if (type.needPath() == (path == null)) { + throw new IllegalStateException(); + } } @NotNull @@ -30,6 +39,11 @@ public ObjectId getObjectId() { return objectId; } + @Nullable + public String getPath() { + return path; + } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -38,19 +52,22 @@ public boolean equals(Object o) { TaskKey taskKey = (TaskKey) o; return (type == taskKey.type) - && objectId.equals(taskKey.objectId); - + && objectId.equals(taskKey.objectId) + && Objects.equals(path, taskKey.path); } @Override public int hashCode() { int result = type.hashCode(); result = 31 * result + objectId.hashCode(); + if (path != null) { + result = 31 * result + path.hashCode(); + } return result; } @Override public String toString() { - return type + ":" + objectId.name(); + return type + ":" + objectId.name() + (path == null ? "" : " (" + path + ")"); } } From 6c3cd1def3c4877022e29d0a0de4a6ec0485874e Mon Sep 17 00:00:00 2001 From: "Artem V. Navrotskiy" Date: Fri, 8 Jan 2016 23:21:08 +0300 Subject: [PATCH 2/2] Support an arbitrary glob, not just a suffix --- README.md | 10 ++--- .../java/git/lfs/migrate/GitConverter.java | 38 +++++++++++++------ src/main/java/git/lfs/migrate/Main.java | 13 ++++--- 3 files changed, 39 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index afec2b7..f3aadd8 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ For quick run you need: * After unpacking archive you can run server executing: ```bash - java -jar build/deploy/git-lfs-migrate.jar -s source-repo.git -d target-repo.git -l http://test:test@lfs-server/ .psd .zip .bin + java -jar build/deploy/git-lfs-migrate.jar -s source-repo.git -d target-repo.git -l http://test:test@lfs-server/ "*.psd" "*.zip" "*.bin" ``` For example, you can convert bozaro/git-lfs-migrate to bozaro/git-lfs-migrate-converted by commands: @@ -27,7 +27,7 @@ git clone --mirror git@github.com:bozaro/git-lfs-migrate.git # Convert repository with moving .md and .jar file to LFS # -# Usage:
[options] LFS file suffixes +# Usage:
[options] LFS file glob patterns # Options: # -c, --cache # Source repository @@ -56,8 +56,8 @@ java -jar git-lfs-migrate.jar \ -s git-lfs-migrate.git \ -d git-lfs-migrate-converted.git \ -g git@github.com:bozaro/git-lfs-migrate-converted.git \ - .md \ - .jar + "*.md" \ + "*.jar" # Push coverted repository to new repository cd git-lfs-migrate-converted.git @@ -101,5 +101,5 @@ call gradlew.bat deployZip When build completes you can convert repository executing: ```bash -java -jar build/deploy/git-lfs-migrate.jar -s source-repo.git -d target-repo.git -l http://test:test@lfs-server/ .psd .zip .bin +java -jar build/deploy/git-lfs-migrate.jar -s source-repo.git -d target-repo.git -l http://test:test@lfs-server/ "*.psd" "*.zip" "*.bin" ``` diff --git a/src/main/java/git/lfs/migrate/GitConverter.java b/src/main/java/git/lfs/migrate/GitConverter.java index 3c21cce..917f1ba 100644 --- a/src/main/java/git/lfs/migrate/GitConverter.java +++ b/src/main/java/git/lfs/migrate/GitConverter.java @@ -1,6 +1,8 @@ package git.lfs.migrate; import org.apache.commons.codec.binary.Hex; +import org.eclipse.jgit.errors.InvalidPatternException; +import org.eclipse.jgit.fnmatch.FileNameMatcher; import org.eclipse.jgit.lib.*; import org.eclipse.jgit.revwalk.*; import org.eclipse.jgit.treewalk.CanonicalTreeParser; @@ -31,7 +33,7 @@ public class GitConverter implements AutoCloseable { @NotNull private static final String GIT_ATTRIBUTES = ".gitattributes"; @NotNull - private final String[] suffixes; + private final String[] globs; @NotNull private final File basePath; @NotNull @@ -41,9 +43,14 @@ public class GitConverter implements AutoCloseable { @NotNull private final HTreeMap cacheSha256; - public GitConverter(@NotNull File cachePath, @NotNull File basePath, @NotNull String[] suffixes) throws IOException { + public GitConverter(@NotNull File cachePath, @NotNull File basePath, @NotNull String[] globs) throws IOException, InvalidPatternException { this.basePath = basePath; - this.suffixes = suffixes.clone(); + this.globs = globs.clone(); + Arrays.sort(globs); + + for (String glob : globs) { + new FileNameMatcher(glob, '/'); + } tempPath = new File(basePath, "lfs/tmp"); makeParentDirs(tempPath); @@ -182,7 +189,7 @@ private List getEntries() throws IOException { blobTask = TaskType.Attribute; pathTask = null; needAttributes = false; - } else if (isFile(fileMode) && matchFilename(treeParser.getEntryPathString())) { + } else if (isFile(fileMode) && matchFilename(path + "/" + treeParser.getEntryPathString())) { blobTask = TaskType.UploadLfs; pathTask = null; } else { @@ -192,7 +199,7 @@ private List getEntries() throws IOException { entries.add(new GitTreeEntry(fileMode, new TaskKey(blobTask, pathTask, treeParser.getEntryObjectId()), treeParser.getEntryPathString())); treeParser.next(); } - if (needAttributes && suffixes.length > 0) { + if (needAttributes && globs.length > 0) { entries.add(new GitTreeEntry(FileMode.REGULAR_FILE, new TaskKey(TaskType.Attribute, null, ObjectId.zeroId()), GIT_ATTRIBUTES)); } return entries; @@ -225,12 +232,21 @@ public ObjectId convert(@NotNull ObjectInserter inserter, @NotNull ConvertResolv } private boolean matchFilename(@NotNull String fileName) { - for (String suffix : suffixes) { - if (fileName.endsWith(suffix)) { - return true; + if (!fileName.startsWith("/")) { + throw new IllegalStateException("Unexpected file name: " + fileName); + } + try { + for (String glob : globs) { + final FileNameMatcher matcher = new FileNameMatcher(glob, null); + matcher.append(fileName.substring(1)); + if (matcher.isMatch()) { + return true; + } } + return false; + } catch (InvalidPatternException e) { + throw new IllegalArgumentException(e); } - return false; } @NotNull @@ -355,8 +371,8 @@ public Iterable depends() throws IOException { @Override public ObjectId convert(@NotNull ObjectInserter inserter, @NotNull ConvertResolver resolver, @Nullable Uploader uploader) throws IOException { final Set attributes = new TreeSet<>(); - for (String suffix : suffixes) { - attributes.add("*" + suffix + "\tfilter=lfs diff=lfs merge=lfs -crlf"); + for (String glob : globs) { + attributes.add(glob + "\tfilter=lfs diff=lfs merge=lfs -crlf"); } final ByteArrayOutputStream blob = new ByteArrayOutputStream(); try (BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(openAttributes(reader, id), StandardCharsets.UTF_8))) { diff --git a/src/main/java/git/lfs/migrate/Main.java b/src/main/java/git/lfs/migrate/Main.java index 72c07c9..ed6e224 100644 --- a/src/main/java/git/lfs/migrate/Main.java +++ b/src/main/java/git/lfs/migrate/Main.java @@ -3,6 +3,7 @@ import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; import org.apache.commons.httpclient.HttpStatus; +import org.eclipse.jgit.errors.InvalidPatternException; import org.eclipse.jgit.lib.*; import org.eclipse.jgit.storage.file.FileRepositoryBuilder; import org.jetbrains.annotations.NotNull; @@ -43,7 +44,7 @@ public class Main { @NotNull private static final Logger log = LoggerFactory.getLogger(Main.class); - public static void main(@NotNull String[] args) throws IOException, InterruptedException, ExecutionException { + public static void main(@NotNull String[] args) throws IOException, InterruptedException, ExecutionException, InvalidPatternException { final CmdArgs cmd = new CmdArgs(); final JCommander jc = new JCommander(cmd); jc.parse(args); @@ -69,7 +70,7 @@ public static void main(@NotNull String[] args) throws IOException, InterruptedE } return; } - processRepository(cmd.src, cmd.dst, cmd.cache, auth, cmd.writeThreads, cmd.uploadThreads, cmd.suffixes.toArray(new String[cmd.suffixes.size()])); + processRepository(cmd.src, cmd.dst, cmd.cache, auth, cmd.writeThreads, cmd.uploadThreads, cmd.globs.toArray(new String[cmd.globs.size()])); log.info("Convert time: {}", System.currentTimeMillis() - time); } @@ -112,7 +113,7 @@ private static boolean checkLfsAuthenticate(@Nullable AuthProvider auth) throws return false; } - public static void processRepository(@NotNull File srcPath, @NotNull File dstPath, @NotNull File cachePath, @Nullable AuthProvider auth, int writeThreads, int uploadThreads, @NotNull String... suffixes) throws IOException, InterruptedException, ExecutionException { + public static void processRepository(@NotNull File srcPath, @NotNull File dstPath, @NotNull File cachePath, @Nullable AuthProvider auth, int writeThreads, int uploadThreads, @NotNull String... globs) throws IOException, InterruptedException, ExecutionException, InvalidPatternException { removeDirectory(dstPath); dstPath.mkdirs(); @@ -123,7 +124,7 @@ public static void processRepository(@NotNull File srcPath, @NotNull File dstPat .setMustExist(false) .setGitDir(dstPath).build(); - final GitConverter converter = new GitConverter(cachePath, dstPath, suffixes); + final GitConverter converter = new GitConverter(cachePath, dstPath, globs); try { dstRepo.create(true); // Load all revision list. @@ -420,9 +421,9 @@ public static class CmdArgs { @Parameter(names = {"--check-lfs"}, description = "Check LFS server settings and exit") private boolean checkLfs = false; - @Parameter(description = "LFS file suffixes") + @Parameter(description = "LFS file glob patterns") @NotNull - private List suffixes = new ArrayList<>(); + private List globs = new ArrayList<>(); @Parameter(names = {"-h", "--help"}, description = "Show help", help = true) private boolean help = false; }