Skip to content

Commit

Permalink
Generate resource graph on bundle (defold#7802)
Browse files Browse the repository at this point in the history 
* Initial commit

* Update ResourceNodeGraph.java

* Update ResourceNodeGraph.java

* Fix imports for tests

* Improved tostring

* Forgot to call visitor.leave() on early out

* Split resource and graph builds

* Merged set and graph creation

* Use resources instead of resource paths

* Cache hex digests

* Added hex digest to resource nodes

* Added hashCode and equals to resource nodes

* Set hex digests on all nodes in a resource graph

* Changed how the archive builder is created

* Fix keep-unused

* Set excluded resources

* Merge dev + fixes (defold#7989)

* merge

* remove import

* Change from absolute to relative paths

* Use relative paths in resource nodes

* Added timers

* Added option to write graph directly to stream

* Reworked the resource graph

* Fixed tests

* Moved test

* take into account only unique children

* fix issue with ManifestBuilder

* fix issue with resurce exclusion

* fix an issue in tests

* semantics refactoring

* remove old comment

* do not include information about the main bundle if user doesn't use LU

---------

Co-authored-by: Alexey Gulev <me@agulev.com>
  • Loading branch information
@britzl @AGulev @ultimanidev
2 people authored and ultimanidev committed Nov 8, 2023
1 parent 465bd08 commit 1d9045e
Show file tree
Hide file tree
Showing 14 changed files with 1,006 additions and 618 deletions.
126 changes: 60 additions & 66 deletions com.dynamo.cr/com.dynamo.cr.bob.test/src/com/dynamo/bob/archive/test/ArchiveTest.java
View file
Expand Up @@ -40,7 +40,11 @@
import com.dynamo.bob.archive.ArchiveBuilder;
import com.dynamo.bob.archive.ArchiveReader;
import com.dynamo.bob.archive.ManifestBuilder;
import com.dynamo.bob.pipeline.ResourceNode;
import com.dynamo.bob.Project;
import com.dynamo.bob.fs.DefaultFileSystem;
import com.dynamo.bob.pipeline.graph.ResourceNode;
import com.dynamo.bob.pipeline.graph.ResourceGraph;

import com.dynamo.liveupdate.proto.Manifest.HashAlgorithm;
import com.dynamo.liveupdate.proto.Manifest.ResourceEntryFlag;

Expand All @@ -54,6 +58,8 @@ public class ArchiveTest {

private ManifestBuilder manifestBuilder;

private ResourceGraph resourceGraph;

private String createDummyFile(String dir, String filepath, byte[] data) throws IOException {
File tmp = new File(Paths.get(FilenameUtils.concat(dir, filepath)).toString());
tmp.getParentFile().mkdirs();
Expand Down Expand Up @@ -86,20 +92,21 @@ else if(b1Shift < b2Shift)
return 0;
}

private ResourceNode addEntryToManifest(String filename, ResourceNode parent) throws IOException {
ResourceNode current = new ResourceNode(filename, FilenameUtils.concat(contentRoot, filename));
parent.addChild(current);
return current;
}

private ResourceNode addEntry(String filename, String content, ArchiveBuilder archiveBuilder, ResourceNode parent) throws IOException {
String filepath = createDummyFile(contentRoot, filename, filename.getBytes());
archiveBuilder.add(filepath, true, false);
return addEntryToManifest(filename, parent);
return resourceGraph.add(filename, parent);
}
private ResourceNode addExcludedEntry(String filename, String content, ArchiveBuilder archiveBuilder, ResourceNode parent) throws IOException {
ResourceNode current = addEntry(filename, content, archiveBuilder, parent);
current.setType(ResourceNode.Type.ExcludedCollectionProxy);
return current;
}

@Before
public void setUp() throws Exception {
resourceGraph = new ResourceGraph(new Project(new DefaultFileSystem()));

contentRoot = Files.createTempDirectory(null).toFile().getAbsolutePath();
outputDarc = Files.createTempFile("tmp.darc", "").toFile();

Expand Down Expand Up @@ -265,18 +272,16 @@ public void testShouldUseCompressedResourceData() throws Exception {
public void testWriteArchive() throws Exception {
ManifestBuilder manifestBuilder = new ManifestBuilder();
manifestBuilder.setResourceHashAlgorithm(HashAlgorithm.HASH_MD5);
manifestBuilder.setResourceGraph(resourceGraph);

ArchiveBuilder instance = new ArchiveBuilder(FilenameUtils.separatorsToSystem(contentRoot), manifestBuilder, 4);

ResourceNode root = new ResourceNode("<Anonymous Root>", "<Anonymous Root>");
ResourceNode root = resourceGraph.getRootNode();
ResourceNode collection1 = addEntry("main.collectionc", "alpha", instance, root);
ResourceNode collectionproxy1 = addEntry("main.collectionproxyc", "beta", instance, collection1);
ResourceNode collectionproxy1 = addExcludedEntry("main.collectionproxyc", "beta", instance, collection1);
ResourceNode gameobject1 = addEntry("main.goc", "delta", instance, collectionproxy1);

manifestBuilder.setRoot(root);

List<String> excludedResources = new ArrayList<String>();
excludedResources.add("/main.collectionproxyc");
List<String> excludedResources = resourceGraph.createExcludedResourcesList();

// Test
RandomAccessFile outFileIndex = new RandomAccessFile(outputIndex, "rw");
Expand All @@ -293,20 +298,18 @@ public void testWriteArchive() throws Exception {
public void testWriteArchive_SiblingProxies() throws Exception {
ManifestBuilder manifestBuilder = new ManifestBuilder();
manifestBuilder.setResourceHashAlgorithm(HashAlgorithm.HASH_MD5);
manifestBuilder.setResourceGraph(resourceGraph);

ArchiveBuilder instance = new ArchiveBuilder(FilenameUtils.separatorsToSystem(contentRoot), manifestBuilder, 4);

ResourceNode root = new ResourceNode("<Anonymous Root>", "<Anonymous Root>");
ResourceNode root = resourceGraph.getRootNode();
ResourceNode collection1 = addEntry("main.collectionc", "alpha", instance, root);
ResourceNode collectionproxy1 = addEntry("level1.collectionproxyc", "beta", instance, collection1);
ResourceNode collectionproxy2 = addEntry("level2.collectionproxyc", "delta", instance, collection1);
ResourceNode collectionproxy2 = addExcludedEntry("level2.collectionproxyc", "delta", instance, collection1);
ResourceNode gameobject1 = addEntry("level1.goc", "gamma", instance, collectionproxy1);
ResourceNode gameobject2 = addEntry("level2.goc", "epsilon", instance, collectionproxy2);

manifestBuilder.setRoot(root);

List<String> excludedResources = new ArrayList<String>();
excludedResources.add("/level2.collectionproxyc");
List<String> excludedResources = resourceGraph.createExcludedResourcesList();

// Test
RandomAccessFile outFileIndex = new RandomAccessFile(outputIndex, "rw");
Expand All @@ -325,20 +328,18 @@ public void testWriteArchive_SiblingProxies() throws Exception {
public void testWriteArchive_SharedResourceExcludedProxy() throws Exception {
ManifestBuilder manifestBuilder = new ManifestBuilder();
manifestBuilder.setResourceHashAlgorithm(HashAlgorithm.HASH_MD5);
manifestBuilder.setResourceGraph(resourceGraph);

ArchiveBuilder instance = new ArchiveBuilder(FilenameUtils.separatorsToSystem(contentRoot), manifestBuilder, 4);

ResourceNode root = new ResourceNode("<Anonymous Root>", "<Anonymous Root>");
ResourceNode root = resourceGraph.getRootNode();
ResourceNode collection1 = addEntry("main.collectionc", "alpha", instance, root);
ResourceNode collectionproxy1 = addEntry("level1.collectionproxyc", "beta", instance, collection1);
ResourceNode collectionproxy1 = addExcludedEntry("level1.collectionproxyc", "beta", instance, collection1);
ResourceNode collectionproxy2 = addEntry("level2.collectionproxyc", "delta", instance, collection1);
ResourceNode gameobject1 = addEntry("shared.goc", "gamma", instance, collectionproxy1);
ResourceNode gameobject2 = addEntry("shared.goc", "gamma", instance, collectionproxy2);

manifestBuilder.setRoot(root);

List<String> excludedResources = new ArrayList<String>();
excludedResources.add("/level1.collectionproxyc");
List<String> excludedResources = resourceGraph.createExcludedResourcesList();

// Test
RandomAccessFile outFileIndex = new RandomAccessFile(outputIndex, "rw");
Expand All @@ -357,20 +358,18 @@ public void testWriteArchive_SharedResourceExcludedProxy() throws Exception {
public void testWriteArchive_DeepProxies() throws Exception {
ManifestBuilder manifestBuilder = new ManifestBuilder();
manifestBuilder.setResourceHashAlgorithm(HashAlgorithm.HASH_MD5);
manifestBuilder.setResourceGraph(resourceGraph);

ArchiveBuilder instance = new ArchiveBuilder(FilenameUtils.separatorsToSystem(contentRoot), manifestBuilder, 4);

ResourceNode root = new ResourceNode("<Anonymous Root>", "<Anonymous Root>");
ResourceNode root = resourceGraph.getRootNode();
ResourceNode collection1 = addEntry("main.collectionc", "alpha", instance, root);
ResourceNode collectionproxy1 = addEntry("level1.collectionproxyc", "beta", instance, collection1);
ResourceNode collectionproxy2 = addEntry("level2.collectionproxyc", "delta", instance, collectionproxy1);
ResourceNode collectionproxy2 = addExcludedEntry("level2.collectionproxyc", "delta", instance, collectionproxy1);
ResourceNode gameobject1 = addEntry("level1.goc", "gamma", instance, collectionproxy1);
ResourceNode gameobject2 = addEntry("level2.goc", "epsilon", instance, collectionproxy2);

manifestBuilder.setRoot(root);

List<String> excludedResources = new ArrayList<String>();
excludedResources.add("/level2.collectionproxyc");
List<String> excludedResources = resourceGraph.createExcludedResourcesList();

// Test
RandomAccessFile outFileIndex = new RandomAccessFile(outputIndex, "rw");
Expand All @@ -386,55 +385,52 @@ public void testWriteArchive_DeepProxies() throws Exception {

@SuppressWarnings("unused")
@Test
public void testExcludeResource() throws Exception {
public void testWriteArchive_ExcludedProxy() throws Exception {
ManifestBuilder manifestBuilder = new ManifestBuilder();
manifestBuilder.setResourceHashAlgorithm(HashAlgorithm.HASH_MD5);
manifestBuilder.setResourceGraph(resourceGraph);

ArchiveBuilder instance = new ArchiveBuilder(FilenameUtils.separatorsToSystem(contentRoot), manifestBuilder, 4);
ResourceNode root = new ResourceNode("<Anonymous Root>", "<Anonymous Root>");
ResourceNode collection1 = addEntryToManifest("main.collectionc", root);
ResourceNode collectionproxy1 = addEntryToManifest("level1.collectionproxyc", collection1);
ResourceNode collectionproxy2 = addEntryToManifest("level2.collectionproxyc", collection1);
ResourceNode gameobject2 = addEntryToManifest("level2.goc", collectionproxy2); // should be excluded
ResourceNode gameobject11 = addEntryToManifest("level1.goc", collectionproxy1); // should be bundled
ResourceNode gameobject12 = addEntryToManifest("level1.goc", collectionproxy2);

manifestBuilder.setRoot(root);

List<String> excludedResources = new ArrayList<String>();
excludedResources.add("/level2.collectionproxyc");

byte[] buffer = "level 2 content".getBytes();
String normalisedPath = FilenameUtils.separatorsToUnix("/level2.goc");
manifestBuilder.addResourceEntry(normalisedPath, buffer, buffer.length, buffer.length, ResourceEntryFlag.EXCLUDED.getNumber());
buffer = "level 1 content".getBytes();
normalisedPath = FilenameUtils.separatorsToUnix("/level1.goc");
manifestBuilder.addResourceEntry(normalisedPath, buffer, buffer.length, buffer.length, ResourceEntryFlag.BUNDLED.getNumber()); // TODO
ResourceNode root = resourceGraph.getRootNode();
ResourceNode collection1 = addEntry("main.collectionc", "alpha", instance, root);
ResourceNode collectionproxy1 = addEntry("level1.collectionproxyc", "beta", instance, collection1);
ResourceNode collectionproxy2 = addExcludedEntry("level2.collectionproxyc", "delta", instance, collection1);
ResourceNode gameobject11 = addEntry("level1.goc", "gamma", instance, collectionproxy1); // should be bundled
ResourceNode gameobject2 = addEntry("level2.goc", "epsilon", instance, collectionproxy2); // should be excluded

List<String> excludedResources = resourceGraph.createExcludedResourcesList();

// Test
assertFalse(instance.excludeResource("/level1.goc", excludedResources));
assertTrue(instance.excludeResource("/level2.goc", excludedResources));
RandomAccessFile outFileIndex = new RandomAccessFile(outputIndex, "rw");
RandomAccessFile outFileData = new RandomAccessFile(outputData, "rw");
instance.write(outFileIndex, outFileData, resourcePackDir, excludedResources);

assertEquals(4, instance.getArchiveEntrySize());
assertEquals("/level1.collectionproxyc", instance.getArchiveEntry(0).getRelativeFilename()); // 617905b1d0e858ca35230357710cf5f2
assertEquals("/main.collectionc", instance.getArchiveEntry(1).getRelativeFilename()); // b32b3904944e63ed5a269caa47904645
assertEquals("/level2.collectionproxyc", instance.getArchiveEntry(2).getRelativeFilename()); // bc05302047f95ca60709254556402710
assertEquals("/level1.goc", instance.getArchiveEntry(3).getRelativeFilename()); // d25298c59a872b5bfd5473de7b36a4a4

}

@SuppressWarnings("unused")
@Test
public void testWriteArchive_ResourceInBundledAndExcludedProxies() throws Exception {
ManifestBuilder manifestBuilder = new ManifestBuilder();
manifestBuilder.setResourceHashAlgorithm(HashAlgorithm.HASH_MD5);
manifestBuilder.setResourceGraph(resourceGraph);

ArchiveBuilder instance = new ArchiveBuilder(FilenameUtils.separatorsToSystem(contentRoot), manifestBuilder, 4);
ResourceNode root = new ResourceNode("<Anonymous Root>", "<Anonymous Root>");
ResourceNode root = resourceGraph.getRootNode();
ResourceNode collection1 = addEntry("main.collectionc", "alpha", instance, root);
ResourceNode collectionproxy1 = addEntry("level1.collectionproxyc", "beta", instance, collection1);
ResourceNode collectionproxy2 = addEntry("level2.collectionproxyc", "delta", instance, collection1);
ResourceNode collectionproxy2 = addExcludedEntry("level2.collectionproxyc", "delta", instance, collection1);
ResourceNode gameobject11 = addEntry("level1.goc", "gamma", instance, collectionproxy1); // should be bundled
ResourceNode gameobject12 = addEntry("level1.goc", "gamma", instance, collectionproxy2);
ResourceNode gameobject2 = addEntry("level2.goc", "epsilon", instance, collectionproxy2); // should be excluded
ResourceNode gameobject3 = addEntry("level3.goc", "eta", instance, collectionproxy2); // should be excluded

manifestBuilder.setRoot(root);

List<String> excludedResources = new ArrayList<String>();
excludedResources.add("/level2.collectionproxyc");
List<String> excludedResources = resourceGraph.createExcludedResourcesList();

// Test
RandomAccessFile outFileIndex = new RandomAccessFile(outputIndex, "rw");
Expand All @@ -453,20 +449,18 @@ public void testWriteArchive_ResourceInBundledAndExcludedProxies() throws Except
public void testWriteArchive_DeepProxiesExcludeGrandparent() throws Exception {
ManifestBuilder manifestBuilder = new ManifestBuilder();
manifestBuilder.setResourceHashAlgorithm(HashAlgorithm.HASH_MD5);
manifestBuilder.setResourceGraph(resourceGraph);

ArchiveBuilder instance = new ArchiveBuilder(FilenameUtils.separatorsToSystem(contentRoot), manifestBuilder, 4);

ResourceNode root = new ResourceNode("<Anonymous Root>", "<Anonymous Root>");
ResourceNode root = resourceGraph.getRootNode();
ResourceNode collection1 = addEntry("main.collectionc", "alpha", instance, root);
ResourceNode collectionproxy1 = addEntry("level1.collectionproxyc", "beta", instance, collection1);
ResourceNode collectionproxy1 = addExcludedEntry("level1.collectionproxyc", "beta", instance, collection1);
ResourceNode collectionproxy2 = addEntry("level2.collectionproxyc", "delta", instance, collectionproxy1);
ResourceNode gameobject1 = addEntry("level1.goc", "gamma", instance, collectionproxy1);
ResourceNode gameobject2 = addEntry("level2.goc", "epsilon", instance, collectionproxy2);

manifestBuilder.setRoot(root);

List<String> excludedResources = new ArrayList<String>();
excludedResources.add("/level1.collectionproxyc");
List<String> excludedResources = resourceGraph.createExcludedResourcesList();

// Test
RandomAccessFile outFileIndex = new RandomAccessFile(outputIndex, "rw");
Expand Down

0 comments on commit 1d9045e

Please sign in to comment.