Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Generate resource graph on bundle #7802

Merged
merged 39 commits into from Oct 25, 2023
Merged

Generate resource graph on bundle #7802

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
39 commits
Select commit Hold shift + click to select a range
845b323
Initial commit
britzl Jul 14, 2023
e3abf4e
Update ResourceNodeGraph.java
britzl Jul 14, 2023
1effd98
Update ResourceNodeGraph.java
britzl Jul 14, 2023
11434a1
Fix imports for tests
britzl Jul 15, 2023
742b215
Improved tostring
britzl Jul 17, 2023
5c5127d
Forgot to call visitor.leave() on early out
britzl Jul 17, 2023
a582cc5
Split resource and graph builds
britzl Jul 17, 2023
eba23b9
Merged set and graph creation
britzl Jul 18, 2023
060869b
Use resources instead of resource paths
britzl Jul 26, 2023
d51454d
Merge branch 'dev' into Issue-7753-generate-resource-graph-on-bundle
britzl Jul 26, 2023
2253d9f
Merge branch 'dev' into Issue-7753-generate-resource-graph-on-bundle
britzl Jul 31, 2023
edac1c8
Cache hex digests
britzl Jul 31, 2023
bd4dd18
Added hex digest to resource nodes
britzl Jul 31, 2023
37c1541
Added hashCode and equals to resource nodes
britzl Jul 31, 2023
f16c33f
Set hex digests on all nodes in a resource graph
britzl Jul 31, 2023
beeb830
Changed how the archive builder is created
britzl Jul 31, 2023
d5cead0
Fix keep-unused
britzl Jul 31, 2023
2db66b5
Set excluded resources
britzl Jul 31, 2023
b3ab0c0
Merge branch 'dev' into Issue-7753-generate-resource-graph-on-bundle
britzl Jul 31, 2023
d33411c
Merge dev + fixes (#7989)
AGulev Sep 12, 2023
35e2b79
Merge branch 'dev' into Issue-7753-generate-resource-graph-on-bundle
AGulev Sep 12, 2023
7a168a0
Change from absolute to relative paths
britzl Sep 19, 2023
440a46e
Use relative paths in resource nodes
britzl Sep 19, 2023
8ac9e29
Merge branch 'dev' into Issue-7753-generate-resource-graph-on-bundle
britzl Sep 25, 2023
e1f5918
Added timers
britzl Oct 2, 2023
8c860ca
Added option to write graph directly to stream
britzl Oct 2, 2023
62f246d
Reworked the resource graph
britzl Oct 5, 2023
5ca699c
Fixed tests
britzl Oct 8, 2023
0fda199
Moved test
britzl Oct 9, 2023
3c642a0
take into account only unique children
AGulev Oct 10, 2023
1c89c65
Merge branch 'dev' into Issue-7753-generate-resource-graph-on-bundle
AGulev Oct 10, 2023
6ac4acf
Merge branch 'dev' into Issue-7753-generate-resource-graph-on-bundle
AGulev Oct 12, 2023
2883b05
fix issue with ManifestBuilder
AGulev Oct 13, 2023
cd88f07
fix issue with resurce exclusion
AGulev Oct 16, 2023
2f29916
Merge branch 'dev' into Issue-7753-generate-resource-graph-on-bundle
AGulev Oct 17, 2023
a1dc374
fix an issue in tests
AGulev Oct 17, 2023
b5e4459
semantics refactoring
AGulev Oct 17, 2023
dfc5c19
remove old comment
AGulev Oct 17, 2023
765ffb0
do not include information about the main bundle if user doesn't use LU
AGulev Oct 18, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
126 changes: 60 additions & 66 deletions com.dynamo.cr/com.dynamo.cr.bob.test/src/com/dynamo/bob/archive/test/ArchiveTest.java
View file
Open in desktop
Expand Up @@ -40,7 +40,11 @@
import com.dynamo.bob.archive.ArchiveBuilder;
import com.dynamo.bob.archive.ArchiveReader;
import com.dynamo.bob.archive.ManifestBuilder;
import com.dynamo.bob.pipeline.ResourceNode;
import com.dynamo.bob.Project;
import com.dynamo.bob.fs.DefaultFileSystem;
import com.dynamo.bob.pipeline.graph.ResourceNode;
import com.dynamo.bob.pipeline.graph.ResourceGraph;

import com.dynamo.liveupdate.proto.Manifest.HashAlgorithm;
import com.dynamo.liveupdate.proto.Manifest.ResourceEntryFlag;

Expand All @@ -54,6 +58,8 @@ public class ArchiveTest {

private ManifestBuilder manifestBuilder;

private ResourceGraph resourceGraph;

private String createDummyFile(String dir, String filepath, byte[] data) throws IOException {
File tmp = new File(Paths.get(FilenameUtils.concat(dir, filepath)).toString());
tmp.getParentFile().mkdirs();
Expand Down Expand Up @@ -86,20 +92,21 @@ else if(b1Shift < b2Shift)
return 0;
}

private ResourceNode addEntryToManifest(String filename, ResourceNode parent) throws IOException {
ResourceNode current = new ResourceNode(filename, FilenameUtils.concat(contentRoot, filename));
parent.addChild(current);
return current;
}

private ResourceNode addEntry(String filename, String content, ArchiveBuilder archiveBuilder, ResourceNode parent) throws IOException {
String filepath = createDummyFile(contentRoot, filename, filename.getBytes());
archiveBuilder.add(filepath, true, false);
return addEntryToManifest(filename, parent);
return resourceGraph.add(filename, parent);
}
private ResourceNode addExcludedEntry(String filename, String content, ArchiveBuilder archiveBuilder, ResourceNode parent) throws IOException {
ResourceNode current = addEntry(filename, content, archiveBuilder, parent);
current.setType(ResourceNode.Type.ExcludedCollectionProxy);
return current;
}

@Before
public void setUp() throws Exception {
resourceGraph = new ResourceGraph(new Project(new DefaultFileSystem()));

contentRoot = Files.createTempDirectory(null).toFile().getAbsolutePath();
outputDarc = Files.createTempFile("tmp.darc", "").toFile();

Expand Down Expand Up @@ -265,18 +272,16 @@ public void testShouldUseCompressedResourceData() throws Exception {
public void testWriteArchive() throws Exception {
ManifestBuilder manifestBuilder = new ManifestBuilder();
manifestBuilder.setResourceHashAlgorithm(HashAlgorithm.HASH_MD5);
manifestBuilder.setResourceGraph(resourceGraph);

ArchiveBuilder instance = new ArchiveBuilder(FilenameUtils.separatorsToSystem(contentRoot), manifestBuilder, 4);

ResourceNode root = new ResourceNode("<Anonymous Root>", "<Anonymous Root>");
ResourceNode root = resourceGraph.getRootNode();
ResourceNode collection1 = addEntry("main.collectionc", "alpha", instance, root);
ResourceNode collectionproxy1 = addEntry("main.collectionproxyc", "beta", instance, collection1);
ResourceNode collectionproxy1 = addExcludedEntry("main.collectionproxyc", "beta", instance, collection1);
ResourceNode gameobject1 = addEntry("main.goc", "delta", instance, collectionproxy1);

manifestBuilder.setRoot(root);

List<String> excludedResources = new ArrayList<String>();
excludedResources.add("/main.collectionproxyc");
List<String> excludedResources = resourceGraph.createExcludedResourcesList();

// Test
RandomAccessFile outFileIndex = new RandomAccessFile(outputIndex, "rw");
Expand All @@ -293,20 +298,18 @@ public void testWriteArchive() throws Exception {
public void testWriteArchive_SiblingProxies() throws Exception {
ManifestBuilder manifestBuilder = new ManifestBuilder();
manifestBuilder.setResourceHashAlgorithm(HashAlgorithm.HASH_MD5);
manifestBuilder.setResourceGraph(resourceGraph);

ArchiveBuilder instance = new ArchiveBuilder(FilenameUtils.separatorsToSystem(contentRoot), manifestBuilder, 4);

ResourceNode root = new ResourceNode("<Anonymous Root>", "<Anonymous Root>");
ResourceNode root = resourceGraph.getRootNode();
ResourceNode collection1 = addEntry("main.collectionc", "alpha", instance, root);
ResourceNode collectionproxy1 = addEntry("level1.collectionproxyc", "beta", instance, collection1);
ResourceNode collectionproxy2 = addEntry("level2.collectionproxyc", "delta", instance, collection1);
ResourceNode collectionproxy2 = addExcludedEntry("level2.collectionproxyc", "delta", instance, collection1);
ResourceNode gameobject1 = addEntry("level1.goc", "gamma", instance, collectionproxy1);
ResourceNode gameobject2 = addEntry("level2.goc", "epsilon", instance, collectionproxy2);

manifestBuilder.setRoot(root);

List<String> excludedResources = new ArrayList<String>();
excludedResources.add("/level2.collectionproxyc");
List<String> excludedResources = resourceGraph.createExcludedResourcesList();

// Test
RandomAccessFile outFileIndex = new RandomAccessFile(outputIndex, "rw");
Expand All @@ -325,20 +328,18 @@ public void testWriteArchive_SiblingProxies() throws Exception {
public void testWriteArchive_SharedResourceExcludedProxy() throws Exception {
ManifestBuilder manifestBuilder = new ManifestBuilder();
manifestBuilder.setResourceHashAlgorithm(HashAlgorithm.HASH_MD5);
manifestBuilder.setResourceGraph(resourceGraph);

ArchiveBuilder instance = new ArchiveBuilder(FilenameUtils.separatorsToSystem(contentRoot), manifestBuilder, 4);

ResourceNode root = new ResourceNode("<Anonymous Root>", "<Anonymous Root>");
ResourceNode root = resourceGraph.getRootNode();
ResourceNode collection1 = addEntry("main.collectionc", "alpha", instance, root);
ResourceNode collectionproxy1 = addEntry("level1.collectionproxyc", "beta", instance, collection1);
ResourceNode collectionproxy1 = addExcludedEntry("level1.collectionproxyc", "beta", instance, collection1);
ResourceNode collectionproxy2 = addEntry("level2.collectionproxyc", "delta", instance, collection1);
ResourceNode gameobject1 = addEntry("shared.goc", "gamma", instance, collectionproxy1);
ResourceNode gameobject2 = addEntry("shared.goc", "gamma", instance, collectionproxy2);

manifestBuilder.setRoot(root);

List<String> excludedResources = new ArrayList<String>();
excludedResources.add("/level1.collectionproxyc");
List<String> excludedResources = resourceGraph.createExcludedResourcesList();

// Test
RandomAccessFile outFileIndex = new RandomAccessFile(outputIndex, "rw");
Expand All @@ -357,20 +358,18 @@ public void testWriteArchive_SharedResourceExcludedProxy() throws Exception {
public void testWriteArchive_DeepProxies() throws Exception {
ManifestBuilder manifestBuilder = new ManifestBuilder();
manifestBuilder.setResourceHashAlgorithm(HashAlgorithm.HASH_MD5);
manifestBuilder.setResourceGraph(resourceGraph);

ArchiveBuilder instance = new ArchiveBuilder(FilenameUtils.separatorsToSystem(contentRoot), manifestBuilder, 4);

ResourceNode root = new ResourceNode("<Anonymous Root>", "<Anonymous Root>");
ResourceNode root = resourceGraph.getRootNode();
ResourceNode collection1 = addEntry("main.collectionc", "alpha", instance, root);
ResourceNode collectionproxy1 = addEntry("level1.collectionproxyc", "beta", instance, collection1);
ResourceNode collectionproxy2 = addEntry("level2.collectionproxyc", "delta", instance, collectionproxy1);
ResourceNode collectionproxy2 = addExcludedEntry("level2.collectionproxyc", "delta", instance, collectionproxy1);
ResourceNode gameobject1 = addEntry("level1.goc", "gamma", instance, collectionproxy1);
ResourceNode gameobject2 = addEntry("level2.goc", "epsilon", instance, collectionproxy2);

manifestBuilder.setRoot(root);

List<String> excludedResources = new ArrayList<String>();
excludedResources.add("/level2.collectionproxyc");
List<String> excludedResources = resourceGraph.createExcludedResourcesList();

// Test
RandomAccessFile outFileIndex = new RandomAccessFile(outputIndex, "rw");
Expand All @@ -386,55 +385,52 @@ public void testWriteArchive_DeepProxies() throws Exception {

@SuppressWarnings("unused")
@Test
public void testExcludeResource() throws Exception {
public void testWriteArchive_ExcludedProxy() throws Exception {
ManifestBuilder manifestBuilder = new ManifestBuilder();
manifestBuilder.setResourceHashAlgorithm(HashAlgorithm.HASH_MD5);
manifestBuilder.setResourceGraph(resourceGraph);

ArchiveBuilder instance = new ArchiveBuilder(FilenameUtils.separatorsToSystem(contentRoot), manifestBuilder, 4);
ResourceNode root = new ResourceNode("<Anonymous Root>", "<Anonymous Root>");
ResourceNode collection1 = addEntryToManifest("main.collectionc", root);
ResourceNode collectionproxy1 = addEntryToManifest("level1.collectionproxyc", collection1);
ResourceNode collectionproxy2 = addEntryToManifest("level2.collectionproxyc", collection1);
ResourceNode gameobject2 = addEntryToManifest("level2.goc", collectionproxy2); // should be excluded
ResourceNode gameobject11 = addEntryToManifest("level1.goc", collectionproxy1); // should be bundled
ResourceNode gameobject12 = addEntryToManifest("level1.goc", collectionproxy2);

manifestBuilder.setRoot(root);

List<String> excludedResources = new ArrayList<String>();
excludedResources.add("/level2.collectionproxyc");

byte[] buffer = "level 2 content".getBytes();
String normalisedPath = FilenameUtils.separatorsToUnix("/level2.goc");
manifestBuilder.addResourceEntry(normalisedPath, buffer, buffer.length, buffer.length, ResourceEntryFlag.EXCLUDED.getNumber());
buffer = "level 1 content".getBytes();
normalisedPath = FilenameUtils.separatorsToUnix("/level1.goc");
manifestBuilder.addResourceEntry(normalisedPath, buffer, buffer.length, buffer.length, ResourceEntryFlag.BUNDLED.getNumber()); // TODO
ResourceNode root = resourceGraph.getRootNode();
ResourceNode collection1 = addEntry("main.collectionc", "alpha", instance, root);
ResourceNode collectionproxy1 = addEntry("level1.collectionproxyc", "beta", instance, collection1);
ResourceNode collectionproxy2 = addExcludedEntry("level2.collectionproxyc", "delta", instance, collection1);
ResourceNode gameobject11 = addEntry("level1.goc", "gamma", instance, collectionproxy1); // should be bundled
ResourceNode gameobject2 = addEntry("level2.goc", "epsilon", instance, collectionproxy2); // should be excluded

List<String> excludedResources = resourceGraph.createExcludedResourcesList();

// Test
assertFalse(instance.excludeResource("/level1.goc", excludedResources));
assertTrue(instance.excludeResource("/level2.goc", excludedResources));
RandomAccessFile outFileIndex = new RandomAccessFile(outputIndex, "rw");
RandomAccessFile outFileData = new RandomAccessFile(outputData, "rw");
instance.write(outFileIndex, outFileData, resourcePackDir, excludedResources);

assertEquals(4, instance.getArchiveEntrySize());
assertEquals("/level1.collectionproxyc", instance.getArchiveEntry(0).getRelativeFilename()); // 617905b1d0e858ca35230357710cf5f2
assertEquals("/main.collectionc", instance.getArchiveEntry(1).getRelativeFilename()); // b32b3904944e63ed5a269caa47904645
assertEquals("/level2.collectionproxyc", instance.getArchiveEntry(2).getRelativeFilename()); // bc05302047f95ca60709254556402710
assertEquals("/level1.goc", instance.getArchiveEntry(3).getRelativeFilename()); // d25298c59a872b5bfd5473de7b36a4a4

}

@SuppressWarnings("unused")
@Test
public void testWriteArchive_ResourceInBundledAndExcludedProxies() throws Exception {
ManifestBuilder manifestBuilder = new ManifestBuilder();
manifestBuilder.setResourceHashAlgorithm(HashAlgorithm.HASH_MD5);
manifestBuilder.setResourceGraph(resourceGraph);

ArchiveBuilder instance = new ArchiveBuilder(FilenameUtils.separatorsToSystem(contentRoot), manifestBuilder, 4);
ResourceNode root = new ResourceNode("<Anonymous Root>", "<Anonymous Root>");
ResourceNode root = resourceGraph.getRootNode();
ResourceNode collection1 = addEntry("main.collectionc", "alpha", instance, root);
ResourceNode collectionproxy1 = addEntry("level1.collectionproxyc", "beta", instance, collection1);
ResourceNode collectionproxy2 = addEntry("level2.collectionproxyc", "delta", instance, collection1);
ResourceNode collectionproxy2 = addExcludedEntry("level2.collectionproxyc", "delta", instance, collection1);
ResourceNode gameobject11 = addEntry("level1.goc", "gamma", instance, collectionproxy1); // should be bundled
ResourceNode gameobject12 = addEntry("level1.goc", "gamma", instance, collectionproxy2);
ResourceNode gameobject2 = addEntry("level2.goc", "epsilon", instance, collectionproxy2); // should be excluded
ResourceNode gameobject3 = addEntry("level3.goc", "eta", instance, collectionproxy2); // should be excluded

manifestBuilder.setRoot(root);

List<String> excludedResources = new ArrayList<String>();
excludedResources.add("/level2.collectionproxyc");
List<String> excludedResources = resourceGraph.createExcludedResourcesList();

// Test
RandomAccessFile outFileIndex = new RandomAccessFile(outputIndex, "rw");
Expand All @@ -453,20 +449,18 @@ public void testWriteArchive_ResourceInBundledAndExcludedProxies() throws Except
public void testWriteArchive_DeepProxiesExcludeGrandparent() throws Exception {
ManifestBuilder manifestBuilder = new ManifestBuilder();
manifestBuilder.setResourceHashAlgorithm(HashAlgorithm.HASH_MD5);
manifestBuilder.setResourceGraph(resourceGraph);

ArchiveBuilder instance = new ArchiveBuilder(FilenameUtils.separatorsToSystem(contentRoot), manifestBuilder, 4);

ResourceNode root = new ResourceNode("<Anonymous Root>", "<Anonymous Root>");
ResourceNode root = resourceGraph.getRootNode();
ResourceNode collection1 = addEntry("main.collectionc", "alpha", instance, root);
ResourceNode collectionproxy1 = addEntry("level1.collectionproxyc", "beta", instance, collection1);
ResourceNode collectionproxy1 = addExcludedEntry("level1.collectionproxyc", "beta", instance, collection1);
ResourceNode collectionproxy2 = addEntry("level2.collectionproxyc", "delta", instance, collectionproxy1);
ResourceNode gameobject1 = addEntry("level1.goc", "gamma", instance, collectionproxy1);
ResourceNode gameobject2 = addEntry("level2.goc", "epsilon", instance, collectionproxy2);

manifestBuilder.setRoot(root);

List<String> excludedResources = new ArrayList<String>();
excludedResources.add("/level1.collectionproxyc");
List<String> excludedResources = resourceGraph.createExcludedResourcesList();

// Test
RandomAccessFile outFileIndex = new RandomAccessFile(outputIndex, "rw");
Expand Down