From d6c7321bdbf3f6bc4862e3aff637ac7a9fadfd2e Mon Sep 17 00:00:00 2001 From: Dan Pesce Date: Wed, 24 May 2023 13:37:23 -0400 Subject: [PATCH 01/38] Fix existing highliting and add orange highlight for previously modified --- frontend/public/global.css | 3 ++- frontend/src/BinaryModifyView.svelte | 5 +++-- frontend/src/ComponentsView.svelte | 2 +- frontend/src/FindReplaceView.svelte | 2 +- frontend/src/ResourceTreeNode.svelte | 4 ++-- 5 files changed, 9 insertions(+), 7 deletions(-) diff --git a/frontend/public/global.css b/frontend/public/global.css index 8d81b7f7e..1dccec025 100644 --- a/frontend/public/global.css +++ b/frontend/public/global.css @@ -3,7 +3,8 @@ --main-fg-color: white; --selected-bg-color: #86e3ed; --comment-color: #eb8e5b; - --modified-color: #dc4e47; + --last-modified-color: #dc4e47; + --all-modified-color: #fdb44e; --font: "cartograph"; --highlight-color: #fff259; /* Snake color */ --accent-text-color: #a9ed96; /* Caterpillar color */ diff --git a/frontend/src/BinaryModifyView.svelte b/frontend/src/BinaryModifyView.svelte index 97ce6082b..54d2f901f 100644 --- a/frontend/src/BinaryModifyView.svelte +++ b/frontend/src/BinaryModifyView.svelte @@ -180,8 +180,9 @@ if (selectedResource) { await selectedResource.queue_patch(patchData, startOffset, endOffset); } - - resourceNodeDataMap[$selected].modified = true; + resourceNodeDataMap[$selected] = { + lastModified: true, + }; modifierView = undefined; refreshResource(); } catch (err) { diff --git a/frontend/src/ComponentsView.svelte b/frontend/src/ComponentsView.svelte index 56e0b0623..9ccb2b66a 100644 --- a/frontend/src/ComponentsView.svelte +++ b/frontend/src/ComponentsView.svelte @@ -216,7 +216,7 @@ if (result === "modified") { for (const resource of results[result]) { resourceNodeDataMap[resource["id"]] = { - modified: true, + lastModified: true, }; } } diff --git a/frontend/src/FindReplaceView.svelte b/frontend/src/FindReplaceView.svelte index 5e4185aa6..c47a2768a 100644 --- a/frontend/src/FindReplaceView.svelte +++ b/frontend/src/FindReplaceView.svelte @@ -118,7 +118,7 @@ if (result === "modified") { for (const resource of results[result]) { resourceNodeDataMap[resource["id"]] = { - modified: true, + lastModified: true, }; } } diff --git a/frontend/src/ResourceTreeNode.svelte b/frontend/src/ResourceTreeNode.svelte index d924dcc90..b04aad1ba 100644 --- a/frontend/src/ResourceTreeNode.svelte +++ b/frontend/src/ResourceTreeNode.svelte @@ -78,13 +78,13 @@ .lastModified { text-decoration-line: underline; - text-decoration-color: var(--modified-color); + text-decoration-color: var(--last-modified-color); text-decoration-thickness: 2px; } .allModified { text-decoration-line: underline; - text-decoration-color: var(--main-fg-color); + text-decoration-color: var(--all-modified-color); text-decoration-thickness: 2px; } From e8606b5495e3bc9835edc7ee8938a8d9a3a74aa3 Mon Sep 17 00:00:00 2001 From: Dan Pesce Date: Thu, 25 May 2023 18:04:55 -0400 Subject: [PATCH 02/38] add range into get_data request and test --- frontend/src/ofrak/remote_resource.js | 6 +++++- ofrak_core/ofrak/gui/server.py | 7 +++++++ ofrak_core/test_ofrak/unit/test_ofrak_server.py | 15 +++++++++++++++ 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/frontend/src/ofrak/remote_resource.js b/frontend/src/ofrak/remote_resource.js index 13a13a9d2..df1ac2760 100644 --- a/frontend/src/ofrak/remote_resource.js +++ b/frontend/src/ofrak/remote_resource.js @@ -147,7 +147,11 @@ export class RemoteResource extends Resource { if (this.cache["get_data"]) { return this.cache["get_data"]; } - let result = await fetch(`${this.uri}/get_data`) + let range_query = "" + if (range){ + range_query = `?range=${range}` + } + let result = await fetch(`${this.uri}/get_data${range_query}`) .then((r) => r.blob()) .then((b) => b.arrayBuffer()); this.cache["get_data"] = result; diff --git a/ofrak_core/ofrak/gui/server.py b/ofrak_core/ofrak/gui/server.py index 971245cb4..911205d67 100644 --- a/ofrak_core/ofrak/gui/server.py +++ b/ofrak_core/ofrak/gui/server.py @@ -146,6 +146,7 @@ def __init__( web.get("/get_root_resources", self.get_root_resources), web.get("/{resource_id}/", self.get_resource), web.get("/{resource_id}/get_data", self.get_data), + web.get("/{resource_id}/get_data_len", self.get_data_len), web.post( "/batch/get_data_range_within_parent", self.batch_get_range, @@ -294,6 +295,12 @@ async def get_data(self, request: Request) -> Response: data = await resource.get_data(_range) return Response(body=data) + @exceptions_to_http(SerializedError) + async def get_data_len(self, request: Request) -> Response: + resource = await self._get_resource_for_request(request) + data_len = await resource.get_data_length() + return json_response(data_len) + @exceptions_to_http(SerializedError) async def get_child_data_ranges(self, request: Request) -> Response: resource = await self._get_resource_for_request(request) diff --git a/ofrak_core/test_ofrak/unit/test_ofrak_server.py b/ofrak_core/test_ofrak/unit/test_ofrak_server.py index 7579ada42..0cfc1f896 100644 --- a/ofrak_core/test_ofrak/unit/test_ofrak_server.py +++ b/ofrak_core/test_ofrak/unit/test_ofrak_server.py @@ -157,6 +157,21 @@ async def test_get_data(ofrak_client: TestClient, hello_world_elf): assert resp.status == 200 resp_body = await resp.read() assert resp_body == hello_world_elf + resp = await ofrak_client.get(f"/{create_body['id']}/get_data", params={"range": "[16,80]"}) + assert resp.status == 200 + resp_body = await resp.read() + assert resp_body == hello_world_elf[0x10:0x50] + + +async def test_get_data_len(ofrak_client: TestClient, hello_world_elf): + create_resp = await ofrak_client.post( + "/create_root_resource", params={"name": "hello_world_elf"}, data=hello_world_elf + ) + create_body = await create_resp.json() + resp = await ofrak_client.get(f"/{create_body['id']}/get_data_len") + assert resp.status == 200 + resp_body = await resp.json() + assert resp_body == len(hello_world_elf) async def test_unpack(ofrak_client: TestClient, hello_world_elf): From beaf8bd867e8a216da8a67197081e14926c10877 Mon Sep 17 00:00:00 2001 From: Dan Pesce Date: Thu, 25 May 2023 18:05:14 -0400 Subject: [PATCH 03/38] lint --- frontend/src/ofrak/remote_resource.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/frontend/src/ofrak/remote_resource.js b/frontend/src/ofrak/remote_resource.js index df1ac2760..afb76f2a5 100644 --- a/frontend/src/ofrak/remote_resource.js +++ b/frontend/src/ofrak/remote_resource.js @@ -147,9 +147,9 @@ export class RemoteResource extends Resource { if (this.cache["get_data"]) { return this.cache["get_data"]; } - let range_query = "" - if (range){ - range_query = `?range=${range}` + let range_query = ""; + if (range) { + range_query = `?range=${range}`; } let result = await fetch(`${this.uri}/get_data${range_query}`) .then((r) => r.blob()) From 711fbb7c82ee2f739b926959f651c7563694506a Mon Sep 17 00:00:00 2001 From: Dan Pesce Date: Tue, 30 May 2023 05:49:12 -0400 Subject: [PATCH 04/38] replace all data.byteLength with dataLenPromise --- frontend/src/App.svelte | 6 +- frontend/src/BinaryModifyView.svelte | 6 +- frontend/src/CarveView.svelte | 6 +- frontend/src/CommentView.svelte | 6 +- frontend/src/HexView.svelte | 212 ++++++++++-------- frontend/src/JumpToOffset.svelte | 7 +- frontend/src/ofrak/remote_resource.js | 8 + ofrak_core/ofrak/gui/server.py | 4 +- .../test_ofrak/unit/test_ofrak_server.py | 4 +- 9 files changed, 150 insertions(+), 109 deletions(-) diff --git a/frontend/src/App.svelte b/frontend/src/App.svelte index 21a0f77f9..783865e2a 100644 --- a/frontend/src/App.svelte +++ b/frontend/src/App.svelte @@ -58,6 +58,7 @@ let showRootResource = false, displayDataPromise = Promise.resolve([]), + dataLenPromise = Promise.resolve([]), hexScrollY = writable({}), useAssemblyView = false, useTextView = false, @@ -83,6 +84,7 @@ } else { $selectedResource = currentResource; displayDataPromise = currentResource.get_data(); + dataLenPromise = currentResource.get_data_len(); useAssemblyView = [ "ofrak.core.complex_block.ComplexBlock", "ofrak.core.basic_block.BasicBlock", @@ -178,6 +180,7 @@ Answer by running riddle.answer('your answer here') from the console.`); @@ -213,6 +216,7 @@ Answer by running riddle.answer('your answer here') from the console.`); {:else} {#if carouselSelection === "Entropy"} diff --git a/frontend/src/BinaryModifyView.svelte b/frontend/src/BinaryModifyView.svelte index 54d2f901f..0d6cdde74 100644 --- a/frontend/src/BinaryModifyView.svelte +++ b/frontend/src/BinaryModifyView.svelte @@ -99,7 +99,7 @@ import { selected, selectedResource as _selectedResource } from "./stores.js"; const selectedResource = $_selectedResource; - export let modifierView, dataPromise, resourceNodeDataMap; + export let modifierView, dataPromise, dataLenPromise, resourceNodeDataMap; let startInput, endInput, startOffset, @@ -108,7 +108,9 @@ errorMessage, userData; - $: dataPromise.then((data) => (dataLength = data.byteLength)); + $: dataLenPromise.then((r) => { + dataLength = r; + }); function refreshResource() { // Force hex view refresh with colors diff --git a/frontend/src/CarveView.svelte b/frontend/src/CarveView.svelte index b63175760..481d7de2e 100644 --- a/frontend/src/CarveView.svelte +++ b/frontend/src/CarveView.svelte @@ -84,10 +84,12 @@ import { calculator } from "./helpers.js"; import { selected, selectedResource } from "./stores.js"; - export let modifierView, resourceNodeDataMap, dataPromise; + export let modifierView, resourceNodeDataMap, dataLenPromise; let startInput, endInput, dataLength, errorMessage; - $: dataPromise.then((data) => (dataLength = data.byteLength)); + $: dataLenPromise.then((r) => { + dataLength = r; + }); function refreshResource() { // Force tree view children refresh diff --git a/frontend/src/CommentView.svelte b/frontend/src/CommentView.svelte index d2c61f3a3..844758f2a 100644 --- a/frontend/src/CommentView.svelte +++ b/frontend/src/CommentView.svelte @@ -82,10 +82,12 @@ import { calculator } from "./helpers.js"; import { selected, selectedResource } from "./stores.js"; - export let modifierView, resourceNodeDataMap, dataPromise; + export let modifierView, resourceNodeDataMap, dataLenPromise; let comment, startInput, endInput, dataLength, errorMessage; - $: dataPromise.then((data) => (dataLength = data.byteLength)); + $: dataLenPromise.then((r) => { + dataLength = r; + }); function refreshResource() { resourceNodeDataMap[$selected].commentsPromise = diff --git a/frontend/src/HexView.svelte b/frontend/src/HexView.svelte index 0277355cb..51f5c39f2 100644 --- a/frontend/src/HexView.svelte +++ b/frontend/src/HexView.svelte @@ -45,26 +45,37 @@ import { chunkList, buf2hex, hexToChar } from "./helpers.js"; import { selectedResource, selected, settings } from "./stores.js"; - export let dataPromise, scrollY, resourceNodeDataMap, resources; + export let dataPromise, + dataLenPromise, + scrollY, + resourceNodeDataMap, + resources; let childRangesPromise = Promise.resolve(undefined); let childRanges, - data = []; + data = [], + len = null; + $: dataPromise.then((r) => { data = r; }); + $: dataLenPromise.then((r) => { + len = r; + }); $: childRangesPromise.then((r) => { childRanges = r; }); - $: Promise.any([dataPromise, childRangesPromise]).then((_) => { - // Hacky solution to minimap view box rectangle only updating on scroll - // after data has loaded -- force a scroll to reload the rectangle after a - // timeout - setTimeout(() => { - if (scrollY !== undefined) { - $scrollY.top = 0; - } - }, 500); - }); + $: Promise.any([dataPromise, dataLenPromise, childRangesPromise]).then( + (_) => { + // Hacky solution to minimap view box rectangle only updating on scroll + // after data has loaded -- force a scroll to reload the rectangle after a + // timeout + setTimeout(() => { + if (scrollY !== undefined) { + $scrollY.top = 0; + } + }, 500); + } + ); const alignment = 16, chunkSize = 4096; @@ -86,24 +97,30 @@ end = 64; $: if (scrollY !== undefined && $scrollY !== undefined) { start = Math.max( - Math.floor((data.byteLength * $scrollY.top) / alignment) * alignment, + Math.floor((len * $scrollY.top) / alignment) * alignment, 0 ); end = Math.min( start + Math.floor($scrollY.viewHeightPixels / lineHeight) * alignment, - data.byteLength + len ); chunks = chunkList(new Uint8Array(data.slice(start, end)), alignment).map( (chunk) => chunkList(buf2hex(chunk), 2) ); } - async function calculateRanges(resource, dataPromise, colors) { + async function calculateRanges( + resource, + dataPromise, + dataLenPromise, + colors + ) { const children = await resource.get_children(); if (children === []) { return []; } const data = await dataPromise; + const len = await dataLenPromise; const childRanges = Object.entries(await resource.get_child_data_ranges()) .filter( ([_, rangeInParent]) => @@ -146,12 +163,12 @@ ranges = ranges; } else if (childRanges.length == 0) { ranges = []; - for (let i = 0; i < data.byteLength; i += chunkSize) { + for (let i = 0; i < len; i += chunkSize) { ranges.push({ color: null, resource_id: null, start: i, - end: Math.min(i + chunkSize, data.byteLength), + end: Math.min(i + chunkSize, len), }); } } @@ -160,6 +177,7 @@ $: childRangesPromise = calculateRanges( $selectedResource, dataPromise, + dataLenPromise, $settings.colors ); @@ -200,96 +218,100 @@ {#await dataPromise} {:then dataResult} - {#if dataResult !== undefined && dataResult.byteLength > 0} - -
-
-
- -
-
-
- {#each chunks as _, chunkIndex} -
- {(chunkIndex * alignment + start) - .toString(16) - .padStart(8, "0") + ": "} -
- {/each} + {#await dataLenPromise} + + {:then dataLen} + {#if dataResult !== undefined && dataLen > 0} + +
+
+
+
- - - - {#await childRangesPromise} +
- {#each chunks as hexes} + {#each chunks as _, chunkIndex}
- {#each hexes as byte} - {byte} - {/each} + {(chunkIndex * alignment + start) + .toString(16) + .padStart(8, "0") + ": "}
{/each}
- {:then childRangesResult} -
- {#each chunks as hexes, chunkIndex} -
- {#each hexes as byte, byteIndex} - {@const rangeInfo = getRangeInfo( - chunkIndex * alignment + byteIndex + start, - childRangesResult, - byte - )} - {#if rangeInfo?.resource_id === null || rangeInfo?.resource_id === undefined} - {byte} - {:else} - {byte} - {/if} - {/each} -
- {/each} -
- {/await} - + -
- {#each chunks as hexes} + {#await childRangesPromise}
- {hexes.map(hexToChar).join("") + " "} + {#each chunks as hexes} +
+ {#each hexes as byte} + {byte} + {/each} +
+ {/each} +
+ {:then childRangesResult} +
+ {#each chunks as hexes, chunkIndex} +
+ {#each hexes as byte, byteIndex} + {@const rangeInfo = getRangeInfo( + chunkIndex * alignment + byteIndex + start, + childRangesResult, + byte + )} + {#if rangeInfo?.resource_id === null || rangeInfo?.resource_id === undefined} + {byte} + {:else} + {byte} + {/if} + {/each} +
+ {/each}
- {/each} + {/await} + + + +
+ {#each chunks as hexes} +
+ {hexes.map(hexToChar).join("") + " "} +
+ {/each} +
-
- {:else} -
- -
+ {:else} +
+ +
- Resource has no data! - {/if} + Resource has no data! + {/if} + {/await} {/await} diff --git a/frontend/src/JumpToOffset.svelte b/frontend/src/JumpToOffset.svelte index 12e470ea4..31a6e6774 100644 --- a/frontend/src/JumpToOffset.svelte +++ b/frontend/src/JumpToOffset.svelte @@ -19,15 +19,16 @@ import { onMount, tick } from "svelte"; import { shortcuts } from "./keyboard"; - export let dataPromise, scrollY; + export let dataLenPromise, scrollY; let startOffset, input, mounted = false; const alignment = 16; let dataLength = 0; - $: dataPromise.then((data) => { - dataLength = data.byteLength; + + $: dataLenPromise.then((r) => { + dataLength = r; }); onMount(() => { diff --git a/frontend/src/ofrak/remote_resource.js b/frontend/src/ofrak/remote_resource.js index afb76f2a5..08a0cab36 100644 --- a/frontend/src/ofrak/remote_resource.js +++ b/frontend/src/ofrak/remote_resource.js @@ -158,6 +158,14 @@ export class RemoteResource extends Resource { return result; } + async get_data_len() { + if (this.data_id === null) { + return []; + } + let result = await fetch(`${this.uri}/get_data_len`).then((r) => r.json()); + return result; + } + async get_data_range_within_parent() { if (this.data_id === null) { return null; diff --git a/ofrak_core/ofrak/gui/server.py b/ofrak_core/ofrak/gui/server.py index 911205d67..35784981d 100644 --- a/ofrak_core/ofrak/gui/server.py +++ b/ofrak_core/ofrak/gui/server.py @@ -146,7 +146,7 @@ def __init__( web.get("/get_root_resources", self.get_root_resources), web.get("/{resource_id}/", self.get_resource), web.get("/{resource_id}/get_data", self.get_data), - web.get("/{resource_id}/get_data_len", self.get_data_len), + web.get("/{resource_id}/get_data_length", self.get_data_length), web.post( "/batch/get_data_range_within_parent", self.batch_get_range, @@ -296,7 +296,7 @@ async def get_data(self, request: Request) -> Response: return Response(body=data) @exceptions_to_http(SerializedError) - async def get_data_len(self, request: Request) -> Response: + async def get_data_length(self, request: Request) -> Response: resource = await self._get_resource_for_request(request) data_len = await resource.get_data_length() return json_response(data_len) diff --git a/ofrak_core/test_ofrak/unit/test_ofrak_server.py b/ofrak_core/test_ofrak/unit/test_ofrak_server.py index 0cfc1f896..36ff7371f 100644 --- a/ofrak_core/test_ofrak/unit/test_ofrak_server.py +++ b/ofrak_core/test_ofrak/unit/test_ofrak_server.py @@ -163,12 +163,12 @@ async def test_get_data(ofrak_client: TestClient, hello_world_elf): assert resp_body == hello_world_elf[0x10:0x50] -async def test_get_data_len(ofrak_client: TestClient, hello_world_elf): +async def test_get_data_length(ofrak_client: TestClient, hello_world_elf): create_resp = await ofrak_client.post( "/create_root_resource", params={"name": "hello_world_elf"}, data=hello_world_elf ) create_body = await create_resp.json() - resp = await ofrak_client.get(f"/{create_body['id']}/get_data_len") + resp = await ofrak_client.get(f"/{create_body['id']}/get_data_length") assert resp.status == 200 resp_body = await resp.json() assert resp_body == len(hello_world_elf) From cbb9908e7bf7085133bafc7f5165a8e8c33d57cb Mon Sep 17 00:00:00 2001 From: Dan Pesce Date: Tue, 30 May 2023 09:29:14 -0400 Subject: [PATCH 05/38] Lazily grabs data but resets the address to 0 --- frontend/src/App.svelte | 3 +- frontend/src/HexView.svelte | 125 ++++++++++++------ frontend/src/ofrak/remote_resource.js | 17 +-- .../test_ofrak/unit/test_ofrak_server.py | 2 +- 4 files changed, 92 insertions(+), 55 deletions(-) diff --git a/frontend/src/App.svelte b/frontend/src/App.svelte index 783865e2a..b45d09211 100644 --- a/frontend/src/App.svelte +++ b/frontend/src/App.svelte @@ -84,7 +84,7 @@ } else { $selectedResource = currentResource; displayDataPromise = currentResource.get_data(); - dataLenPromise = currentResource.get_data_len(); + dataLenPromise = currentResource.get_data_length(); useAssemblyView = [ "ofrak.core.complex_block.ComplexBlock", "ofrak.core.basic_block.BasicBlock", @@ -215,7 +215,6 @@ Answer by running riddle.answer('your answer here') from the console.`); {:else} { - data = r; - }); $: dataLenPromise.then((r) => { len = r; }); $: childRangesPromise.then((r) => { childRanges = r; }); - $: Promise.any([dataPromise, dataLenPromise, childRangesPromise]).then( + $: chunkDataPromise.then((r) => { + chunks = r; + }); + $: Promise.any([dataLenPromise, childRangesPromise]).then( (_) => { // Hacky solution to minimap view box rectangle only updating on scroll // after data has loaded -- force a scroll to reload the rectangle after a @@ -78,7 +78,8 @@ ); const alignment = 16, - chunkSize = 4096; + chunkSize = 4096, + loadSize = chunkSize * 10; // Sadly, this is the most flexible, most reliable way to get the line height // from arbitrary CSS units in pixels. It is definitely a little nasty :( const lineHeight = (() => { @@ -94,8 +95,25 @@ let chunks = [], start = 0, - end = 64; - $: if (scrollY !== undefined && $scrollY !== undefined) { + end = 64, + lastLoadedAddress = 0; + // $: if (scrollY !== undefined && $scrollY !== undefined) { + // start = Math.max( + // Math.floor((len * $scrollY.top) / alignment) * alignment, + // 0 + // ); + // end = Math.min( + // start + Math.floor($scrollY.viewHeightPixels / lineHeight) * alignment, + // len + // ); + // chunks = chunkList(new Uint8Array(data.slice(start, end)), alignment).map( + // (chunk) => chunkList(buf2hex(chunk), 2) + // ); + // } + + async function getNewData(){ + const len = await dataLenPromise; + console.log("scrolling") start = Math.max( Math.floor((len * $scrollY.top) / alignment) * alignment, 0 @@ -104,14 +122,34 @@ start + Math.floor($scrollY.viewHeightPixels / lineHeight) * alignment, len ); - chunks = chunkList(new Uint8Array(data.slice(start, end)), alignment).map( + // console.log("Start: " + start); + // console.log("End: " + end); + // console.log("Last Addr: " + lastLoadedAddress); + // console.log("Len: " + len); + // console.log("possible start:" + Math.floor((len * $scrollY.top) / alignment) * alignment) + + if (end >= lastLoadedAddress) { + console.log("Updating data") + lastLoadedAddress = start + loadSize; + if (lastLoadedAddress > len){ + console.log("end of file") + lastLoadedAddress = len; + } + chunkData = await $selectedResource.get_data([start, lastLoadedAddress]); + console.log("data updated") + + } + chunks = chunkList(new Uint8Array(chunkData.slice(start, end)), alignment).map( (chunk) => chunkList(buf2hex(chunk), 2) ); + return chunks; + } + $: if (scrollY !== undefined && $scrollY !== undefined) { + chunkDataPromise = getNewData($scrollY); } async function calculateRanges( resource, - dataPromise, dataLenPromise, colors ) { @@ -119,7 +157,6 @@ if (children === []) { return []; } - const data = await dataPromise; const len = await dataLenPromise; const childRanges = Object.entries(await resource.get_child_data_ranges()) .filter( @@ -176,7 +213,6 @@ } $: childRangesPromise = calculateRanges( $selectedResource, - dataPromise, dataLenPromise, $settings.colors ); @@ -215,28 +251,29 @@ } -{#await dataPromise} + +{#await dataLenPromise} -{:then dataResult} - {#await dataLenPromise} - - {:then dataLen} - {#if dataResult !== undefined && dataLen > 0} - -
-
-
- -
-
+{:then dataLen} + {#if dataLen > 0} + +
+
+
+ +
+
+ {#await chunkDataPromise} + + {:then chunks}
{#each chunks as _, chunkIndex}
@@ -303,15 +340,15 @@
{/each}
-
+ {/await}
- {:else} -
- -
+
+ {:else} +
+ +
- Resource has no data! - {/if} - {/await} + Resource has no data! + {/if} {/await} diff --git a/frontend/src/ofrak/remote_resource.js b/frontend/src/ofrak/remote_resource.js index 08a0cab36..b7f838cf3 100644 --- a/frontend/src/ofrak/remote_resource.js +++ b/frontend/src/ofrak/remote_resource.js @@ -144,25 +144,26 @@ export class RemoteResource extends Resource { return []; } - if (this.cache["get_data"]) { - return this.cache["get_data"]; - } + // if (this.cache["get_data"]) { + // return this.cache["get_data"]; + // } let range_query = ""; if (range) { - range_query = `?range=${range}`; + range_query = `?range=[${range}]`; } let result = await fetch(`${this.uri}/get_data${range_query}`) .then((r) => r.blob()) .then((b) => b.arrayBuffer()); - this.cache["get_data"] = result; + // this.cache["get_data"] = result; return result; } - async get_data_len() { + async get_data_length() { if (this.data_id === null) { - return []; + return null; } - let result = await fetch(`${this.uri}/get_data_len`).then((r) => r.json()); + let result = await fetch(`${this.uri}/get_data_length`) + .then((r) => r.json()); return result; } diff --git a/ofrak_core/test_ofrak/unit/test_ofrak_server.py b/ofrak_core/test_ofrak/unit/test_ofrak_server.py index 36ff7371f..25aac7c36 100644 --- a/ofrak_core/test_ofrak/unit/test_ofrak_server.py +++ b/ofrak_core/test_ofrak/unit/test_ofrak_server.py @@ -157,7 +157,7 @@ async def test_get_data(ofrak_client: TestClient, hello_world_elf): assert resp.status == 200 resp_body = await resp.read() assert resp_body == hello_world_elf - resp = await ofrak_client.get(f"/{create_body['id']}/get_data", params={"range": "[16,80]"}) + resp = await ofrak_client.get(f"/{create_body['id']}/get_data", params={"range": "16,80"}) assert resp.status == 200 resp_body = await resp.read() assert resp_body == hello_world_elf[0x10:0x50] From 6541625bfe21dac17aa9cf46619f64d7954cd7a7 Mon Sep 17 00:00:00 2001 From: Dan Pesce Date: Tue, 30 May 2023 10:09:22 -0400 Subject: [PATCH 06/38] Append new data to existing data --- frontend/src/HexView.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/HexView.svelte b/frontend/src/HexView.svelte index 3cf0a6b19..640d3cc2b 100644 --- a/frontend/src/HexView.svelte +++ b/frontend/src/HexView.svelte @@ -135,7 +135,7 @@ console.log("end of file") lastLoadedAddress = len; } - chunkData = await $selectedResource.get_data([start, lastLoadedAddress]); + chunkData = await $selectedResource.get_data([start, lastLoadedAddress]).then((r) => new Blob([chunkData, r]).arrayBuffer()); console.log("data updated") } From a5d96e1f68a393df9a22bb4ddcb399669f566c5a Mon Sep 17 00:00:00 2001 From: Dan Pesce Date: Tue, 30 May 2023 10:36:51 -0400 Subject: [PATCH 07/38] Use window for data, rapid requests at edges --- frontend/src/HexView.svelte | 55 ++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 29 deletions(-) diff --git a/frontend/src/HexView.svelte b/frontend/src/HexView.svelte index 640d3cc2b..f44191051 100644 --- a/frontend/src/HexView.svelte +++ b/frontend/src/HexView.svelte @@ -96,24 +96,11 @@ let chunks = [], start = 0, end = 64, - lastLoadedAddress = 0; - // $: if (scrollY !== undefined && $scrollY !== undefined) { - // start = Math.max( - // Math.floor((len * $scrollY.top) / alignment) * alignment, - // 0 - // ); - // end = Math.min( - // start + Math.floor($scrollY.viewHeightPixels / lineHeight) * alignment, - // len - // ); - // chunks = chunkList(new Uint8Array(data.slice(start, end)), alignment).map( - // (chunk) => chunkList(buf2hex(chunk), 2) - // ); - // } + endWindow = 0, + startWindow = 0; async function getNewData(){ const len = await dataLenPromise; - console.log("scrolling") start = Math.max( Math.floor((len * $scrollY.top) / alignment) * alignment, 0 @@ -122,24 +109,34 @@ start + Math.floor($scrollY.viewHeightPixels / lineHeight) * alignment, len ); - // console.log("Start: " + start); - // console.log("End: " + end); - // console.log("Last Addr: " + lastLoadedAddress); - // console.log("Len: " + len); - // console.log("possible start:" + Math.floor((len * $scrollY.top) / alignment) * alignment) - if (end >= lastLoadedAddress) { - console.log("Updating data") - lastLoadedAddress = start + loadSize; - if (lastLoadedAddress > len){ - console.log("end of file") - lastLoadedAddress = len; + if (end >= endWindow) { + startWindow = start; + if (startWindow < 0){ + startWindow = 0; } - chunkData = await $selectedResource.get_data([start, lastLoadedAddress]).then((r) => new Blob([chunkData, r]).arrayBuffer()); - console.log("data updated") + endWindow = startWindow + loadSize; + if (endWindow > len){ + endWindow = len; + } + + chunkData = await $selectedResource.get_data([startWindow, endWindow]); + } else if (start < startWindow) { + endWindow = end; + if (endWindow > len){ + endWindow = len; + } + + startWindow = endWindow - loadSize; + if (startWindow < 0){ + startWindow = 0; + } + + chunkData = await $selectedResource.get_data([startWindow, endWindow]); } - chunks = chunkList(new Uint8Array(chunkData.slice(start, end)), alignment).map( + + chunks = chunkList(new Uint8Array(chunkData.slice(start - startWindow, end - startWindow)), alignment).map( (chunk) => chunkList(buf2hex(chunk), 2) ); return chunks; From d5718d305948cc44408b6c0f99a580bc84c74585 Mon Sep 17 00:00:00 2001 From: Dan Pesce Date: Tue, 30 May 2023 10:37:53 -0400 Subject: [PATCH 08/38] lint --- frontend/src/HexView.svelte | 53 +++++++++++++++---------------------- 1 file changed, 22 insertions(+), 31 deletions(-) diff --git a/frontend/src/HexView.svelte b/frontend/src/HexView.svelte index f44191051..ba3ca2411 100644 --- a/frontend/src/HexView.svelte +++ b/frontend/src/HexView.svelte @@ -45,10 +45,7 @@ import { chunkList, buf2hex, hexToChar } from "./helpers.js"; import { selectedResource, selected, settings } from "./stores.js"; - export let dataLenPromise, - scrollY, - resourceNodeDataMap, - resources; + export let dataLenPromise, scrollY, resourceNodeDataMap, resources; let childRangesPromise = Promise.resolve(undefined); let chunkDataPromise = Promise.resolve(undefined); let childRanges, @@ -64,18 +61,16 @@ $: chunkDataPromise.then((r) => { chunks = r; }); - $: Promise.any([dataLenPromise, childRangesPromise]).then( - (_) => { - // Hacky solution to minimap view box rectangle only updating on scroll - // after data has loaded -- force a scroll to reload the rectangle after a - // timeout - setTimeout(() => { - if (scrollY !== undefined) { - $scrollY.top = 0; - } - }, 500); - } - ); + $: Promise.any([dataLenPromise, childRangesPromise]).then((_) => { + // Hacky solution to minimap view box rectangle only updating on scroll + // after data has loaded -- force a scroll to reload the rectangle after a + // timeout + setTimeout(() => { + if (scrollY !== undefined) { + $scrollY.top = 0; + } + }, 500); + }); const alignment = 16, chunkSize = 4096, @@ -99,7 +94,7 @@ endWindow = 0, startWindow = 0; - async function getNewData(){ + async function getNewData() { const len = await dataLenPromise; start = Math.max( Math.floor((len * $scrollY.top) / alignment) * alignment, @@ -112,44 +107,41 @@ if (end >= endWindow) { startWindow = start; - if (startWindow < 0){ + if (startWindow < 0) { startWindow = 0; } endWindow = startWindow + loadSize; - if (endWindow > len){ + if (endWindow > len) { endWindow = len; } chunkData = await $selectedResource.get_data([startWindow, endWindow]); } else if (start < startWindow) { endWindow = end; - if (endWindow > len){ + if (endWindow > len) { endWindow = len; } startWindow = endWindow - loadSize; - if (startWindow < 0){ + if (startWindow < 0) { startWindow = 0; } chunkData = await $selectedResource.get_data([startWindow, endWindow]); } - chunks = chunkList(new Uint8Array(chunkData.slice(start - startWindow, end - startWindow)), alignment).map( - (chunk) => chunkList(buf2hex(chunk), 2) - ); + chunks = chunkList( + new Uint8Array(chunkData.slice(start - startWindow, end - startWindow)), + alignment + ).map((chunk) => chunkList(buf2hex(chunk), 2)); return chunks; } $: if (scrollY !== undefined && $scrollY !== undefined) { chunkDataPromise = getNewData($scrollY); } - async function calculateRanges( - resource, - dataLenPromise, - colors - ) { + async function calculateRanges(resource, dataLenPromise, colors) { const children = await resource.get_children(); if (children === []) { return []; @@ -248,7 +240,6 @@ } - {#await dataLenPromise} {:then dataLen} @@ -269,7 +260,7 @@
{#await chunkDataPromise} - + {:then chunks}
{#each chunks as _, chunkIndex} From cf7d3cdc05e980d7a9bc56d6a13a31194bee4ee0 Mon Sep 17 00:00:00 2001 From: Dan Pesce Date: Tue, 30 May 2023 12:30:17 -0400 Subject: [PATCH 09/38] Remove dataPromise from App.svelte --- frontend/src/App.svelte | 5 +---- frontend/src/BinaryModifyView.svelte | 19 +++++++------------ frontend/src/HexView.svelte | 8 ++++---- frontend/src/TextView.svelte | 4 +++- frontend/src/ofrak/remote_resource.js | 5 +++-- 5 files changed, 18 insertions(+), 23 deletions(-) diff --git a/frontend/src/App.svelte b/frontend/src/App.svelte index b45d09211..51b84f097 100644 --- a/frontend/src/App.svelte +++ b/frontend/src/App.svelte @@ -57,7 +57,6 @@ printConsoleArt(); let showRootResource = false, - displayDataPromise = Promise.resolve([]), dataLenPromise = Promise.resolve([]), hexScrollY = writable({}), useAssemblyView = false, @@ -83,7 +82,6 @@ console.error("Couldn't get the resource for ID " + $selected); } else { $selectedResource = currentResource; - displayDataPromise = currentResource.get_data(); dataLenPromise = currentResource.get_data_length(); useAssemblyView = [ "ofrak.core.complex_block.ComplexBlock", @@ -179,7 +177,6 @@ Answer by running riddle.answer('your answer here') from the console.`); {#if modifierView} {:else if useTextView} - + {:else} - (userData = chunkList( - new Uint8Array(data.slice(startOffset, endOffset)), - 16 - ) - .map((r) => buf2hex(r, " ")) - .join("\n")) - ); + let data = await selectedResource.get_data([startOffset, endOffset]); + userData = chunkList(new Uint8Array(data), 16) + .map((r) => buf2hex(r, " ")) + .join("\n"); } } catch (err) { try { diff --git a/frontend/src/HexView.svelte b/frontend/src/HexView.svelte index ba3ca2411..aab47ee1b 100644 --- a/frontend/src/HexView.svelte +++ b/frontend/src/HexView.svelte @@ -74,7 +74,7 @@ const alignment = 16, chunkSize = 4096, - loadSize = chunkSize * 10; + windowSize = chunkSize * 10; // Sadly, this is the most flexible, most reliable way to get the line height // from arbitrary CSS units in pixels. It is definitely a little nasty :( const lineHeight = (() => { @@ -105,13 +105,13 @@ len ); - if (end >= endWindow) { + if (end > endWindow) { startWindow = start; if (startWindow < 0) { startWindow = 0; } - endWindow = startWindow + loadSize; + endWindow = startWindow + windowSize; if (endWindow > len) { endWindow = len; } @@ -123,7 +123,7 @@ endWindow = len; } - startWindow = endWindow - loadSize; + startWindow = endWindow - windowSize; if (startWindow < 0) { startWindow = 0; } diff --git a/frontend/src/TextView.svelte b/frontend/src/TextView.svelte index d83ec3f6d..22637d89e 100644 --- a/frontend/src/TextView.svelte +++ b/frontend/src/TextView.svelte @@ -33,8 +33,10 @@ { if (e.key === 'Enter') { input.blur(); try { + let dataLength = await $selectedResource.get_data_length(); let result = calculator.calculate(input.value) + 1; $scrollY.top = result / dataLength; } catch (_) { From 8935d978a5e525359b77869a26b18d72914c55b9 Mon Sep 17 00:00:00 2001 From: Dan Pesce Date: Mon, 5 Jun 2023 17:09:37 -0400 Subject: [PATCH 17/38] Add window padding --- frontend/src/HexView.svelte | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/frontend/src/HexView.svelte b/frontend/src/HexView.svelte index 922471916..c692c34eb 100644 --- a/frontend/src/HexView.svelte +++ b/frontend/src/HexView.svelte @@ -74,7 +74,7 @@ const alignment = 16, chunkSize = 4096, - windowSize = chunkSize * 5; + windowSize = chunkSize * 10; // Sadly, this is the most flexible, most reliable way to get the line height // from arbitrary CSS units in pixels. It is definitely a little nasty :( const lineHeight = (() => { @@ -92,7 +92,8 @@ start = 0, end = 64, endWindow = 0, - startWindow = 0; + startWindow = 0, + windowPadding = 1024; async function getNewData() { const len = await dataLenPromise; @@ -105,29 +106,25 @@ len ); - if (end > endWindow) { - startWindow = start; + if (end > endWindow - windowPadding) { + startWindow = start - windowPadding; if (startWindow < 0) { startWindow = 0; } - endWindow = startWindow + windowSize; if (endWindow > len) { endWindow = len; } - chunkData = await $selectedResource.get_data([startWindow, endWindow]); - } else if (start < startWindow) { - endWindow = end; + } else if (start < startWindow + windowPadding) { + endWindow = end + windowPadding; if (endWindow > len) { endWindow = len; } - startWindow = endWindow - windowSize; if (startWindow < 0) { startWindow = 0; } - chunkData = await $selectedResource.get_data([startWindow, endWindow]); } From 63d45391172e9454e1d383772f0bf9588cf4ded5 Mon Sep 17 00:00:00 2001 From: Dan Pesce Date: Mon, 5 Jun 2023 17:44:10 -0400 Subject: [PATCH 18/38] Undo JumpToOffset change to push to seperate PR --- frontend/src/App.svelte | 5 ++++- frontend/src/JumpToOffset.svelte | 19 +++++++++---------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/frontend/src/App.svelte b/frontend/src/App.svelte index a7bcb7ace..51b84f097 100644 --- a/frontend/src/App.svelte +++ b/frontend/src/App.svelte @@ -223,7 +223,10 @@ Answer by running riddle.answer('your answer here') from the console.`); https://github.com/sveltejs/svelte/issues/5604 --> - + {#if carouselSelection === "Entropy"} {:else if carouselSelection === "Byteclass"} diff --git a/frontend/src/JumpToOffset.svelte b/frontend/src/JumpToOffset.svelte index 66da1bd05..31a6e6774 100644 --- a/frontend/src/JumpToOffset.svelte +++ b/frontend/src/JumpToOffset.svelte @@ -18,14 +18,19 @@ import { calculator } from "./helpers"; import { onMount, tick } from "svelte"; import { shortcuts } from "./keyboard"; - import { selectedResource } from "./stores.js"; - export let scrollY; + export let dataLenPromise, scrollY; let startOffset, input, mounted = false; const alignment = 16; + let dataLength = 0; + + $: dataLenPromise.then((r) => { + dataLength = r; + }); + onMount(() => { mounted = true; }); @@ -36,27 +41,21 @@ } }; - async function getStartOffset() { - let dataLength = await $selectedResource.get_data_length(); + $: if (mounted) { startOffset = Math.max( Math.floor((dataLength * $scrollY.top) / alignment) * alignment, 0 ); input.value = `0x${startOffset.toString(16)}`; } - - $: if (mounted) { - getStartOffset(); - } { if (e.key === 'Enter') { input.blur(); try { - let dataLength = await $selectedResource.get_data_length(); let result = calculator.calculate(input.value) + 1; $scrollY.top = result / dataLength; } catch (_) { From e9bc52af2bad41518eac036ccd0df6567e6fbeb5 Mon Sep 17 00:00:00 2001 From: Dan Pesce Date: Wed, 7 Jun 2023 12:17:57 -0400 Subject: [PATCH 19/38] Chunk data to upload large files --- frontend/src/StartView.svelte | 13 ++++++++++++- ofrak_core/ofrak/gui/server.py | 16 +++++++++++++++- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/frontend/src/StartView.svelte b/frontend/src/StartView.svelte index 916adea59..f2d276092 100644 --- a/frontend/src/StartView.svelte +++ b/frontend/src/StartView.svelte @@ -109,6 +109,7 @@ tryHash = !!window.location.hash; let mouseX, selectedAnimal; const warnFileSize = 250 * 1024 * 1024; + const fileChunkSize = warnFileSize; async function createRootResource(f) { if ( @@ -123,11 +124,21 @@ return; } + for (var start = 0; start < f.size; start += fileChunkSize) { + let end = Math.min(start + fileChunkSize, f.size); + const result = await fetch( + `${$settings.backendUrl}/send_root_resource_chunk?name=${f.name}`, + { + method: "POST", + body: await f.slice(start, end), + } + ); + } + const rootModel = await fetch( `${$settings.backendUrl}/create_root_resource?name=${f.name}`, { method: "POST", - body: await f.arrayBuffer(), } ).then((r) => r.json()); diff --git a/ofrak_core/ofrak/gui/server.py b/ofrak_core/ofrak/gui/server.py index 6b8184975..66cd69933 100644 --- a/ofrak_core/ofrak/gui/server.py +++ b/ofrak_core/ofrak/gui/server.py @@ -144,8 +144,10 @@ def __init__( self.resource_view_context: ResourceViewContext = ResourceViewContext() self.component_context: ComponentContext = ClientComponentContext() self.script_builder: ScriptBuilder = ScriptBuilder() + self.resource_builder: Dict[str:bytes] = {} self._app.add_routes( [ + web.post("/send_root_resource_chunk", self.send_root_resource_chunk), web.post("/create_root_resource", self.create_root_resource), web.get("/get_root_resources", self.get_root_resources), web.get("/{resource_id}/", self.get_resource), @@ -260,12 +262,24 @@ async def run_until_cancelled(self): # pragma: no cover finally: await self.runner.cleanup() + @exceptions_to_http(SerializedError) + async def send_root_resource_chunk(self, request: Request) -> Response: + name = request.query.get("name") + print(name) + print("Chunk") + if name not in self.resource_builder.keys(): + self.resource_builder[name] = b"" + chunk_data = await request.read() + self.resource_builder[name] += chunk_data + return json_response([]) + @exceptions_to_http(SerializedError) async def create_root_resource(self, request: Request) -> Response: name = request.query.get("name") if name is None: return HTTPBadRequest(reason="Missing root resource `name` from request") - resource_data = await request.read() + + resource_data = self.resource_builder[name] script_str = rf""" root_resource = await ofrak_context.create_root_resource_from_file("{name}")""" try: From e0541d142f130140bfdaf3dcfed0aa658776edad Mon Sep 17 00:00:00 2001 From: Dan Pesce Date: Wed, 7 Jun 2023 12:23:07 -0400 Subject: [PATCH 20/38] remove data from dict after load --- ofrak_core/ofrak/gui/server.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ofrak_core/ofrak/gui/server.py b/ofrak_core/ofrak/gui/server.py index 66cd69933..8f0b14a7a 100644 --- a/ofrak_core/ofrak/gui/server.py +++ b/ofrak_core/ofrak/gui/server.py @@ -293,6 +293,7 @@ async def create_root_resource(self, request: Request) -> Response: except Exception as e: await self.script_builder.clear_script_queue(root_resource) raise e + self.resource_builder.pop(name) return json_response(self._serialize_resource(root_resource)) @exceptions_to_http(SerializedError) From d73155cb969f1fab0060680a3edec3cf98abc5bf Mon Sep 17 00:00:00 2001 From: Dan Pesce Date: Wed, 7 Jun 2023 13:22:47 -0400 Subject: [PATCH 21/38] add addr to query to avoid out of order errors --- frontend/src/StartView.svelte | 2 +- ofrak_core/ofrak/gui/server.py | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/frontend/src/StartView.svelte b/frontend/src/StartView.svelte index f2d276092..3559165d6 100644 --- a/frontend/src/StartView.svelte +++ b/frontend/src/StartView.svelte @@ -127,7 +127,7 @@ for (var start = 0; start < f.size; start += fileChunkSize) { let end = Math.min(start + fileChunkSize, f.size); const result = await fetch( - `${$settings.backendUrl}/send_root_resource_chunk?name=${f.name}`, + `${$settings.backendUrl}/send_root_resource_chunk?name=${f.name}&addr=${start}`, { method: "POST", body: await f.slice(start, end), diff --git a/ofrak_core/ofrak/gui/server.py b/ofrak_core/ofrak/gui/server.py index 8f0b14a7a..b34884aab 100644 --- a/ofrak_core/ofrak/gui/server.py +++ b/ofrak_core/ofrak/gui/server.py @@ -144,7 +144,7 @@ def __init__( self.resource_view_context: ResourceViewContext = ResourceViewContext() self.component_context: ComponentContext = ClientComponentContext() self.script_builder: ScriptBuilder = ScriptBuilder() - self.resource_builder: Dict[str:bytes] = {} + self.resource_builder: Dict[str : Dict[int:bytes]] = {} self._app.add_routes( [ web.post("/send_root_resource_chunk", self.send_root_resource_chunk), @@ -265,12 +265,13 @@ async def run_until_cancelled(self): # pragma: no cover @exceptions_to_http(SerializedError) async def send_root_resource_chunk(self, request: Request) -> Response: name = request.query.get("name") + addr = int(request.query.get("addr")) print(name) print("Chunk") if name not in self.resource_builder.keys(): - self.resource_builder[name] = b"" + self.resource_builder[name] = {} chunk_data = await request.read() - self.resource_builder[name] += chunk_data + self.resource_builder[name][addr] = chunk_data return json_response([]) @exceptions_to_http(SerializedError) @@ -279,7 +280,7 @@ async def create_root_resource(self, request: Request) -> Response: if name is None: return HTTPBadRequest(reason="Missing root resource `name` from request") - resource_data = self.resource_builder[name] + resource_data = b"".join([v for k, v in sorted(self.resource_builder[name].items())]) script_str = rf""" root_resource = await ofrak_context.create_root_resource_from_file("{name}")""" try: From 9042fbba7c6846f26d61a5b0858a4a149a8c98bf Mon Sep 17 00:00:00 2001 From: Dan Pesce Date: Wed, 7 Jun 2023 14:48:12 -0400 Subject: [PATCH 22/38] Fix typing issue --- ofrak_core/ofrak/gui/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ofrak_core/ofrak/gui/server.py b/ofrak_core/ofrak/gui/server.py index ad12069a5..fca7b4906 100644 --- a/ofrak_core/ofrak/gui/server.py +++ b/ofrak_core/ofrak/gui/server.py @@ -144,7 +144,7 @@ def __init__( self.resource_view_context: ResourceViewContext = ResourceViewContext() self.component_context: ComponentContext = ClientComponentContext() self.script_builder: ScriptBuilder = ScriptBuilder() - self.resource_builder: Dict[str : Dict[int:bytes]] = {} + self.resource_builder: Dict[str, Dict[int, bytes]] = {} self._app.add_routes( [ web.post("/send_root_resource_chunk", self.send_root_resource_chunk), From 34ec277100a0c642ba83b4a7e3915d0219ed697a Mon Sep 17 00:00:00 2001 From: Dan Pesce Date: Wed, 7 Jun 2023 14:54:09 -0400 Subject: [PATCH 23/38] query type for addr, check None --- ofrak_core/ofrak/gui/server.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ofrak_core/ofrak/gui/server.py b/ofrak_core/ofrak/gui/server.py index fca7b4906..3669d31e2 100644 --- a/ofrak_core/ofrak/gui/server.py +++ b/ofrak_core/ofrak/gui/server.py @@ -265,7 +265,11 @@ async def run_until_cancelled(self): # pragma: no cover @exceptions_to_http(SerializedError) async def send_root_resource_chunk(self, request: Request) -> Response: name = request.query.get("name") - addr = int(request.query.get("addr")) + addr = request.query.get("addr") + if addr is not None: + addr = int(addr) + else: + raise HTTPBadRequest(reason="Missing chunk address from request") print(name) print("Chunk") if name not in self.resource_builder.keys(): From 304fd9df06dc88447580134d0b0b0c1a65611abc Mon Sep 17 00:00:00 2001 From: Dan Pesce Date: Wed, 7 Jun 2023 15:20:12 -0400 Subject: [PATCH 24/38] Fix mypy --- ofrak_core/ofrak/gui/server.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ofrak_core/ofrak/gui/server.py b/ofrak_core/ofrak/gui/server.py index 3669d31e2..b842a4b38 100644 --- a/ofrak_core/ofrak/gui/server.py +++ b/ofrak_core/ofrak/gui/server.py @@ -266,16 +266,16 @@ async def run_until_cancelled(self): # pragma: no cover async def send_root_resource_chunk(self, request: Request) -> Response: name = request.query.get("name") addr = request.query.get("addr") - if addr is not None: - addr = int(addr) - else: + if name is None: + raise HTTPBadRequest(reason="Missing resource name from request") + if addr is None: raise HTTPBadRequest(reason="Missing chunk address from request") print(name) print("Chunk") if name not in self.resource_builder.keys(): self.resource_builder[name] = {} chunk_data = await request.read() - self.resource_builder[name][addr] = chunk_data + self.resource_builder[name][int(addr)] = chunk_data return json_response([]) @exceptions_to_http(SerializedError) From c244475a5aa4615e6a936c84cd6ecdbdeb1063e0 Mon Sep 17 00:00:00 2001 From: Dan Pesce Date: Wed, 7 Jun 2023 15:35:01 -0400 Subject: [PATCH 25/38] Remove debug prints --- ofrak_core/ofrak/gui/server.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/ofrak_core/ofrak/gui/server.py b/ofrak_core/ofrak/gui/server.py index b842a4b38..acf0cc5a1 100644 --- a/ofrak_core/ofrak/gui/server.py +++ b/ofrak_core/ofrak/gui/server.py @@ -270,8 +270,6 @@ async def send_root_resource_chunk(self, request: Request) -> Response: raise HTTPBadRequest(reason="Missing resource name from request") if addr is None: raise HTTPBadRequest(reason="Missing chunk address from request") - print(name) - print("Chunk") if name not in self.resource_builder.keys(): self.resource_builder[name] = {} chunk_data = await request.read() From 5011fe20a6ba59f04b6024d74bd0210cb819b4f8 Mon Sep 17 00:00:00 2001 From: Dan Pesce Date: Wed, 7 Jun 2023 16:17:02 -0400 Subject: [PATCH 26/38] Add back original create_root_resource_method and check size in frontend --- ofrak_core/ofrak/gui/server.py | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/ofrak_core/ofrak/gui/server.py b/ofrak_core/ofrak/gui/server.py index acf0cc5a1..8ae8ca1a0 100644 --- a/ofrak_core/ofrak/gui/server.py +++ b/ofrak_core/ofrak/gui/server.py @@ -147,8 +147,9 @@ def __init__( self.resource_builder: Dict[str, Dict[int, bytes]] = {} self._app.add_routes( [ - web.post("/send_root_resource_chunk", self.send_root_resource_chunk), web.post("/create_root_resource", self.create_root_resource), + web.post("/root_resource_chunk", self.root_resource_chunk), + web.post("/create_chunked_root_resource", self.create_chunked_root_resource), web.get("/get_root_resources", self.get_root_resources), web.get("/{resource_id}/", self.get_resource), web.get("/{resource_id}/get_data", self.get_data), @@ -263,7 +264,7 @@ async def run_until_cancelled(self): # pragma: no cover await self.runner.cleanup() @exceptions_to_http(SerializedError) - async def send_root_resource_chunk(self, request: Request) -> Response: + async def root_resource_chunk(self, request: Request) -> Response: name = request.query.get("name") addr = request.query.get("addr") if name is None: @@ -277,7 +278,7 @@ async def send_root_resource_chunk(self, request: Request) -> Response: return json_response([]) @exceptions_to_http(SerializedError) - async def create_root_resource(self, request: Request) -> Response: + async def create_chunked_root_resource(self, request: Request) -> Response: name = request.query.get("name") if name is None: return HTTPBadRequest(reason="Missing root resource `name` from request") @@ -300,6 +301,28 @@ async def create_root_resource(self, request: Request) -> Response: self.resource_builder.pop(name) return json_response(self._serialize_resource(root_resource)) + @exceptions_to_http(SerializedError) + async def create_root_resource(self, request: Request) -> Response: + name = request.query.get("name") + if name is None: + return HTTPBadRequest(reason="Missing root resource `name` from request") + resource_data = await request.read() + script_str = rf""" + if root_resource is None: + root_resource = await ofrak_context.create_root_resource_from_file("{name}")""" + try: + root_resource = await self._ofrak_context.create_root_resource( + name, resource_data, (File,) + ) + await self.script_builder.add_action(root_resource, script_str, ActionType.UNPACK) + if request.remote is not None: + self._job_ids[request.remote] = root_resource.get_job_id() + await self.script_builder.commit_to_script(root_resource) + except Exception as e: + await self.script_builder.clear_script_queue(root_resource) + raise e + return json_response(self._serialize_resource(root_resource)) + @exceptions_to_http(SerializedError) async def get_root_resources(self, request: Request) -> Response: roots = await self._ofrak_context.resource_service.get_root_resources() From 08cd92b598b3eef4148b73fc74d9665c4abfe2fd Mon Sep 17 00:00:00 2001 From: Dan Pesce Date: Wed, 7 Jun 2023 16:17:27 -0400 Subject: [PATCH 27/38] Add check in frontnend --- frontend/src/StartView.svelte | 37 ++++++++++++++++++++++------------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/frontend/src/StartView.svelte b/frontend/src/StartView.svelte index 3559165d6..e60394384 100644 --- a/frontend/src/StartView.svelte +++ b/frontend/src/StartView.svelte @@ -112,6 +112,7 @@ const fileChunkSize = warnFileSize; async function createRootResource(f) { + let rootModel; if ( f.size > warnFileSize && !window.confirm( @@ -123,25 +124,33 @@ showRootResource = false; return; } + if (f.size > warnFileSize) { + for (var start = 0; start < f.size; start += fileChunkSize) { + let end = Math.min(start + fileChunkSize, f.size); + const result = await fetch( + `${$settings.backendUrl}/root_resource_chunk?name=${f.name}&addr=${start}`, + { + method: "POST", + body: await f.slice(start, end), + } + ); + } - for (var start = 0; start < f.size; start += fileChunkSize) { - let end = Math.min(start + fileChunkSize, f.size); - const result = await fetch( - `${$settings.backendUrl}/send_root_resource_chunk?name=${f.name}&addr=${start}`, + rootModel = await fetch( + `${$settings.backendUrl}/create_chunked_root_resource?name=${f.name}`, { method: "POST", - body: await f.slice(start, end), } - ); + ).then((r) => r.json()); + } else { + rootModel = await fetch( + `${$settings.backendUrl}/create_root_resource?name=${f.name}`, + { + method: "POST", + body: await f.arrayBuffer(), + } + ).then((r) => r.json()); } - - const rootModel = await fetch( - `${$settings.backendUrl}/create_root_resource?name=${f.name}`, - { - method: "POST", - } - ).then((r) => r.json()); - rootResource = remote_model_to_resource(rootModel, resources); $selected = rootModel.id; } From e50e4faed6e99f182318e45173a55e6c5f23e663 Mon Sep 17 00:00:00 2001 From: Dan Pesce Date: Wed, 7 Jun 2023 16:30:34 -0400 Subject: [PATCH 28/38] add test --- ofrak_core/test_ofrak/unit/test_ofrak_server.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/ofrak_core/test_ofrak/unit/test_ofrak_server.py b/ofrak_core/test_ofrak/unit/test_ofrak_server.py index 09af5b414..ece2492ea 100644 --- a/ofrak_core/test_ofrak/unit/test_ofrak_server.py +++ b/ofrak_core/test_ofrak/unit/test_ofrak_server.py @@ -119,6 +119,23 @@ async def test_create_root_resource( assert body["tags"] == json_result["tags"] +async def test_create_chunked_root_resource( + ofrak_client: TestClient, ofrak_context, ofrak_server, hello_world_elf +): + chunk_size = int(len(hello_world_elf) / 10) + for start in range(0, len(hello_world_elf), chunk_size): + end = min(start + chunk_size, len(hello_world_elf)) + res = await ofrak_client.post( + "/root_resource_chunk", + params={"name": "hello_world_elf", "addr": start}, + data=hello_world_elf[start:end], + ) + create_resp = await ofrak_client.post( + "/create_chunked_root_resource", params={"name": "hello_world_elf"} + ) + assert create_resp.status == 200 + + async def test_get_root_resources( ofrak_client: TestClient, ofrak_context, ofrak_server, hello_world_elf ): From 6c3a2f182ad185d09c575e79777ed12bad8ee1d8 Mon Sep 17 00:00:00 2001 From: Dan Pesce Date: Wed, 14 Jun 2023 11:32:58 -0400 Subject: [PATCH 29/38] Changelog update --- ofrak_core/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/ofrak_core/CHANGELOG.md b/ofrak_core/CHANGELOG.md index a6cc9dc21..94e0aba6f 100644 --- a/ofrak_core/CHANGELOG.md +++ b/ofrak_core/CHANGELOG.md @@ -4,6 +4,7 @@ All notable changes to `ofrak` will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased](https://github.com/redballoonsecurity/ofrak/tree/master) +Upload files in chunks to handle files larger than 2G ([#324](https://github.com/redballoonsecurity/ofrak/pull/324)) ## [3.1.0](https://github.com/redballoonsecurity/ofrak/compare/ofrak-v3.0.0...ofrak-v3.1.0) ### Added From 5c15d5616282a0c19a04fa6d7875eb1f84b5edc9 Mon Sep 17 00:00:00 2001 From: Dan Pesce Date: Wed, 14 Jun 2023 14:47:48 -0400 Subject: [PATCH 30/38] Collect chunk promises and send at once --- frontend/src/StartView.svelte | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/frontend/src/StartView.svelte b/frontend/src/StartView.svelte index 979ea115d..5004c38f1 100644 --- a/frontend/src/StartView.svelte +++ b/frontend/src/StartView.svelte @@ -123,6 +123,17 @@ const warnFileSize = 250 * 1024 * 1024; const fileChunkSize = warnFileSize; + async function sendChunk(start, f){ + let end = Math.min(start + fileChunkSize, f.size); + await fetch( + `${$settings.backendUrl}/root_resource_chunk?name=${f.name}&addr=${start}`, + { + method: "POST", + body: await f.slice(start, end), + } + ) + } + async function createRootResource(f) { let rootModel; if ( @@ -137,16 +148,8 @@ return; } if (f.size > warnFileSize) { - for (var start = 0; start < f.size; start += fileChunkSize) { - let end = Math.min(start + fileChunkSize, f.size); - const result = await fetch( - `${$settings.backendUrl}/root_resource_chunk?name=${f.name}&addr=${start}`, - { - method: "POST", - body: await f.slice(start, end), - } - ); - } + let chunkStartAddrs = Array.from({length:Math.ceil(f.size / fileChunkSize)}, (v, i ) => i * fileChunkSize) + await Promise.all(chunkStartAddrs.map(start => sendChunk(start, f))) rootModel = await fetch( `${$settings.backendUrl}/create_chunked_root_resource?name=${f.name}`, From bb231480d19f24810cbf9d8a5387b8031bfa4e3c Mon Sep 17 00:00:00 2001 From: Dan Pesce Date: Wed, 14 Jun 2023 16:37:25 -0400 Subject: [PATCH 31/38] Index chunked data by resource id --- frontend/src/StartView.svelte | 29 +++++++++++------- ofrak_core/ofrak/gui/server.py | 54 ++++++++++++++++++++++++---------- 2 files changed, 57 insertions(+), 26 deletions(-) diff --git a/frontend/src/StartView.svelte b/frontend/src/StartView.svelte index 5004c38f1..3a8358e98 100644 --- a/frontend/src/StartView.svelte +++ b/frontend/src/StartView.svelte @@ -123,15 +123,15 @@ const warnFileSize = 250 * 1024 * 1024; const fileChunkSize = warnFileSize; - async function sendChunk(start, f){ + async function sendChunk(id, f, start) { let end = Math.min(start + fileChunkSize, f.size); await fetch( - `${$settings.backendUrl}/root_resource_chunk?name=${f.name}&addr=${start}`, - { - method: "POST", - body: await f.slice(start, end), - } - ) + `${$settings.backendUrl}/root_resource_chunk?id=${id}&start=${start}&end=${end}`, + { + method: "POST", + body: await f.slice(start, end), + } + ); } async function createRootResource(f) { @@ -148,11 +148,20 @@ return; } if (f.size > warnFileSize) { - let chunkStartAddrs = Array.from({length:Math.ceil(f.size / fileChunkSize)}, (v, i ) => i * fileChunkSize) - await Promise.all(chunkStartAddrs.map(start => sendChunk(start, f))) + let id = await fetch( + `${$settings.backendUrl}/init_chunked_root_resource?name=${f.name}&size=${f.size}`, + { method: "POST" } + ).then((r) => r.json()); + let chunkStartAddrs = Array.from( + { length: Math.ceil(f.size / fileChunkSize) }, + (v, i) => i * fileChunkSize + ); + await Promise.all( + chunkStartAddrs.map((start) => sendChunk(id, f, start)) + ); rootModel = await fetch( - `${$settings.backendUrl}/create_chunked_root_resource?name=${f.name}`, + `${$settings.backendUrl}/create_chunked_root_resource?id=${id}&name=${f.name}`, { method: "POST", } diff --git a/ofrak_core/ofrak/gui/server.py b/ofrak_core/ofrak/gui/server.py index 8ae8ca1a0..4e47ff54f 100644 --- a/ofrak_core/ofrak/gui/server.py +++ b/ofrak_core/ofrak/gui/server.py @@ -144,10 +144,11 @@ def __init__( self.resource_view_context: ResourceViewContext = ResourceViewContext() self.component_context: ComponentContext = ClientComponentContext() self.script_builder: ScriptBuilder = ScriptBuilder() - self.resource_builder: Dict[str, Dict[int, bytes]] = {} + self.resource_builder: Dict[bytes, Tuple[Resource, memoryview]] = {} self._app.add_routes( [ web.post("/create_root_resource", self.create_root_resource), + web.post("/init_chunked_root_resource", self.init_chunked_root_resource), web.post("/root_resource_chunk", self.root_resource_chunk), web.post("/create_chunked_root_resource", self.create_chunked_root_resource), web.get("/get_root_resources", self.get_root_resources), @@ -264,33 +265,54 @@ async def run_until_cancelled(self): # pragma: no cover await self.runner.cleanup() @exceptions_to_http(SerializedError) - async def root_resource_chunk(self, request: Request) -> Response: + async def init_chunked_root_resource(self, request: Request) -> Response: name = request.query.get("name") - addr = request.query.get("addr") + size = int(request.query.get("size")) if name is None: raise HTTPBadRequest(reason="Missing resource name from request") - if addr is None: - raise HTTPBadRequest(reason="Missing chunk address from request") - if name not in self.resource_builder.keys(): - self.resource_builder[name] = {} + if size is None: + raise HTTPBadRequest(reason="Missing chunk size from request") + root_resource: Resource = await self._ofrak_context.create_root_resource( + name, bytearray(b"\x00" * size), (File,) + ) + self.resource_builder[root_resource.get_id().hex()] = ( + root_resource, + memoryview(bytearray(b"\x00" * size)), + ) + return json_response(root_resource.get_id().hex()) + + @exceptions_to_http(SerializedError) + async def root_resource_chunk(self, request: Request) -> Response: + id = request.query.get("id") + start = int(request.query.get("start")) + end = int(request.query.get("end")) + if id is None: + raise HTTPBadRequest(reason="Missing resource id from request") + if start is None: + raise HTTPBadRequest(reason="Missing chunk start from request") + if end is None: + raise HTTPBadRequest(reason="Missing chunk end from request") chunk_data = await request.read() - self.resource_builder[name][int(addr)] = chunk_data + _, data = self.resource_builder[id] + data[start:end] = chunk_data return json_response([]) @exceptions_to_http(SerializedError) async def create_chunked_root_resource(self, request: Request) -> Response: + id = request.query.get("id") name = request.query.get("name") + if id is None: + return HTTPBadRequest(reason="Missing root resource `id` from request") if name is None: return HTTPBadRequest(reason="Missing root resource `name` from request") - resource_data = b"".join([v for k, v in sorted(self.resource_builder[name].items())]) - script_str = rf""" - if root_resource is None: - root_resource = await ofrak_context.create_root_resource_from_file("{name}")""" try: - root_resource = await self._ofrak_context.create_root_resource( - name, resource_data, (File,) - ) + root_resource, data = self.resource_builder[id] + script_str = rf""" + if root_resource is None: + root_resource = await ofrak_context.create_root_resource_from_file("{name}")""" + root_resource.queue_patch(Range(0, len(bytearray(data))), bytearray(data)) + await root_resource.save() await self.script_builder.add_action(root_resource, script_str, ActionType.UNPACK) if request.remote is not None: self._job_ids[request.remote] = root_resource.get_job_id() @@ -298,7 +320,7 @@ async def create_chunked_root_resource(self, request: Request) -> Response: except Exception as e: await self.script_builder.clear_script_queue(root_resource) raise e - self.resource_builder.pop(name) + self.resource_builder.pop(id) return json_response(self._serialize_resource(root_resource)) @exceptions_to_http(SerializedError) From 9e51c9d71f2b1d60b250254070f8a46eb3f841b7 Mon Sep 17 00:00:00 2001 From: Dan Pesce Date: Thu, 15 Jun 2023 13:20:40 -0400 Subject: [PATCH 32/38] make mypy smile --- ofrak_core/ofrak/gui/server.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/ofrak_core/ofrak/gui/server.py b/ofrak_core/ofrak/gui/server.py index 4e47ff54f..22cfea7ab 100644 --- a/ofrak_core/ofrak/gui/server.py +++ b/ofrak_core/ofrak/gui/server.py @@ -144,7 +144,7 @@ def __init__( self.resource_view_context: ResourceViewContext = ResourceViewContext() self.component_context: ComponentContext = ClientComponentContext() self.script_builder: ScriptBuilder = ScriptBuilder() - self.resource_builder: Dict[bytes, Tuple[Resource, memoryview]] = {} + self.resource_builder: Dict[str, Tuple[Resource, memoryview]] = {} self._app.add_routes( [ web.post("/create_root_resource", self.create_root_resource), @@ -267,7 +267,8 @@ async def run_until_cancelled(self): # pragma: no cover @exceptions_to_http(SerializedError) async def init_chunked_root_resource(self, request: Request) -> Response: name = request.query.get("name") - size = int(request.query.get("size")) + size_param = request.query.get("size") + size = int(size_param) if size_param is not None else None if name is None: raise HTTPBadRequest(reason="Missing resource name from request") if size is None: @@ -284,8 +285,10 @@ async def init_chunked_root_resource(self, request: Request) -> Response: @exceptions_to_http(SerializedError) async def root_resource_chunk(self, request: Request) -> Response: id = request.query.get("id") - start = int(request.query.get("start")) - end = int(request.query.get("end")) + start_param = request.query.get("start") + start = int(start_param) if start_param is not None else None + end_param = request.query.get("end") + end = int(end_param) if end_param is not None else None if id is None: raise HTTPBadRequest(reason="Missing resource id from request") if start is None: From 9ce592995968c6ec21647554c15ffecbb8c7fc17 Mon Sep 17 00:00:00 2001 From: Dan Pesce Date: Thu, 15 Jun 2023 14:27:30 -0400 Subject: [PATCH 33/38] update test --- ofrak_core/test_ofrak/unit/test_ofrak_server.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/ofrak_core/test_ofrak/unit/test_ofrak_server.py b/ofrak_core/test_ofrak/unit/test_ofrak_server.py index ece2492ea..df7decae0 100644 --- a/ofrak_core/test_ofrak/unit/test_ofrak_server.py +++ b/ofrak_core/test_ofrak/unit/test_ofrak_server.py @@ -123,15 +123,20 @@ async def test_create_chunked_root_resource( ofrak_client: TestClient, ofrak_context, ofrak_server, hello_world_elf ): chunk_size = int(len(hello_world_elf) / 10) + init_resp = await ofrak_client.post( + "/init_chunked_root_resource", + params={"name": "hellow_world_elf", "size": len(hello_world_elf)}, + ) + id = await init_resp.json() for start in range(0, len(hello_world_elf), chunk_size): end = min(start + chunk_size, len(hello_world_elf)) res = await ofrak_client.post( "/root_resource_chunk", - params={"name": "hello_world_elf", "addr": start}, + params={"id": id, "start": start, "end": end}, data=hello_world_elf[start:end], ) create_resp = await ofrak_client.post( - "/create_chunked_root_resource", params={"name": "hello_world_elf"} + "/create_chunked_root_resource", params={"name": "hello_world_elf", "id": id} ) assert create_resp.status == 200 From 046a02b97fa376448444e3463b1d3ebaa54b8635 Mon Sep 17 00:00:00 2001 From: Dan Pesce Date: Thu, 15 Jun 2023 14:41:02 -0400 Subject: [PATCH 34/38] Addressing Jacobs review --- ofrak_core/ofrak/gui/server.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/ofrak_core/ofrak/gui/server.py b/ofrak_core/ofrak/gui/server.py index 22cfea7ab..cd521c057 100644 --- a/ofrak_core/ofrak/gui/server.py +++ b/ofrak_core/ofrak/gui/server.py @@ -268,14 +268,12 @@ async def run_until_cancelled(self): # pragma: no cover async def init_chunked_root_resource(self, request: Request) -> Response: name = request.query.get("name") size_param = request.query.get("size") - size = int(size_param) if size_param is not None else None if name is None: raise HTTPBadRequest(reason="Missing resource name from request") - if size is None: + if size_param is None: raise HTTPBadRequest(reason="Missing chunk size from request") - root_resource: Resource = await self._ofrak_context.create_root_resource( - name, bytearray(b"\x00" * size), (File,) - ) + size = int(size_param) + root_resource: Resource = await self._ofrak_context.create_root_resource(name, b"", (File,)) self.resource_builder[root_resource.get_id().hex()] = ( root_resource, memoryview(bytearray(b"\x00" * size)), @@ -286,15 +284,15 @@ async def init_chunked_root_resource(self, request: Request) -> Response: async def root_resource_chunk(self, request: Request) -> Response: id = request.query.get("id") start_param = request.query.get("start") - start = int(start_param) if start_param is not None else None end_param = request.query.get("end") - end = int(end_param) if end_param is not None else None if id is None: raise HTTPBadRequest(reason="Missing resource id from request") - if start is None: + if start_param is None: raise HTTPBadRequest(reason="Missing chunk start from request") - if end is None: + if end_param is None: raise HTTPBadRequest(reason="Missing chunk end from request") + start = int(start_param) + end = int(end_param) chunk_data = await request.read() _, data = self.resource_builder[id] data[start:end] = chunk_data @@ -314,7 +312,7 @@ async def create_chunked_root_resource(self, request: Request) -> Response: script_str = rf""" if root_resource is None: root_resource = await ofrak_context.create_root_resource_from_file("{name}")""" - root_resource.queue_patch(Range(0, len(bytearray(data))), bytearray(data)) + root_resource.queue_patch(Range(0, 0), bytearray(data)) await root_resource.save() await self.script_builder.add_action(root_resource, script_str, ActionType.UNPACK) if request.remote is not None: From c6fbb79ebef567b9360b3f0563ec9c7301cabc22 Mon Sep 17 00:00:00 2001 From: Dan Pesce Date: Mon, 19 Jun 2023 14:02:14 -0400 Subject: [PATCH 35/38] Add large file to test --- .../test_ofrak/unit/test_ofrak_server.py | 29 ++++++++++++++----- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/ofrak_core/test_ofrak/unit/test_ofrak_server.py b/ofrak_core/test_ofrak/unit/test_ofrak_server.py index df7decae0..bf1ae154f 100644 --- a/ofrak_core/test_ofrak/unit/test_ofrak_server.py +++ b/ofrak_core/test_ofrak/unit/test_ofrak_server.py @@ -1,6 +1,9 @@ import itertools import json import os +import tempfile +from ofrak.ofrak_context import OFRAKContext +from ofrak.resource import Resource import pytest import re import sys @@ -26,6 +29,14 @@ def hello_world_elf() -> bytes: return hello_elf() +@pytest.fixture() +async def large_test_file(ofrak_context: OFRAKContext) -> Resource: + with tempfile.NamedTemporaryFile() as temp: + for i in range(0, 256, 1): + temp.write(int.to_bytes(i) * 4096) + yield await ofrak_context.create_root_resource_from_file(temp.name) + + @pytest.fixture(scope="session") def firmware_zip() -> bytes: assets_dir = os.path.abspath( @@ -120,25 +131,29 @@ async def test_create_root_resource( async def test_create_chunked_root_resource( - ofrak_client: TestClient, ofrak_context, ofrak_server, hello_world_elf + ofrak_client: TestClient, ofrak_server, large_test_file ): - chunk_size = int(len(hello_world_elf) / 10) + test_file_data = await large_test_file.get_data() + chunk_size = int(len(test_file_data) / 10) init_resp = await ofrak_client.post( "/init_chunked_root_resource", - params={"name": "hellow_world_elf", "size": len(hello_world_elf)}, + params={"name": "test_file_data", "size": len(test_file_data)}, ) id = await init_resp.json() - for start in range(0, len(hello_world_elf), chunk_size): - end = min(start + chunk_size, len(hello_world_elf)) + for start in range(0, len(test_file_data), chunk_size): + end = min(start + chunk_size, len(test_file_data)) res = await ofrak_client.post( "/root_resource_chunk", params={"id": id, "start": start, "end": end}, - data=hello_world_elf[start:end], + data=test_file_data[start:end], ) create_resp = await ofrak_client.post( - "/create_chunked_root_resource", params={"name": "hello_world_elf", "id": id} + "/create_chunked_root_resource", params={"name": "test_file_data", "id": id} ) assert create_resp.status == 200 + length_resp = await ofrak_client.get(f"/{id}/get_data_length") + length_resp_body = await length_resp.json() + assert length_resp_body == len(test_file_data) async def test_get_root_resources( From 7a5e916e6be782838eec9565e88abf329e2d1b89 Mon Sep 17 00:00:00 2001 From: Dan Pesce Date: Mon, 19 Jun 2023 14:22:36 -0400 Subject: [PATCH 36/38] update int.to_bytes for python3.7 --- ofrak_core/test_ofrak/unit/test_ofrak_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ofrak_core/test_ofrak/unit/test_ofrak_server.py b/ofrak_core/test_ofrak/unit/test_ofrak_server.py index bf1ae154f..2f0ab69dc 100644 --- a/ofrak_core/test_ofrak/unit/test_ofrak_server.py +++ b/ofrak_core/test_ofrak/unit/test_ofrak_server.py @@ -33,7 +33,7 @@ def hello_world_elf() -> bytes: async def large_test_file(ofrak_context: OFRAKContext) -> Resource: with tempfile.NamedTemporaryFile() as temp: for i in range(0, 256, 1): - temp.write(int.to_bytes(i) * 4096) + temp.write(int.to_bytes(i, 1, "big") * 4096) yield await ofrak_context.create_root_resource_from_file(temp.name) From afea4bf8014e696233f6ca0f9fdc528151c5a8f5 Mon Sep 17 00:00:00 2001 From: dannyp303 Date: Mon, 19 Jun 2023 17:21:05 -0400 Subject: [PATCH 37/38] Update ofrak_core/test_ofrak/unit/test_ofrak_server.py Co-authored-by: Jacob Strieb <99368685+rbs-jacob@users.noreply.github.com> --- ofrak_core/test_ofrak/unit/test_ofrak_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ofrak_core/test_ofrak/unit/test_ofrak_server.py b/ofrak_core/test_ofrak/unit/test_ofrak_server.py index 2f0ab69dc..ea693e6f8 100644 --- a/ofrak_core/test_ofrak/unit/test_ofrak_server.py +++ b/ofrak_core/test_ofrak/unit/test_ofrak_server.py @@ -32,7 +32,7 @@ def hello_world_elf() -> bytes: @pytest.fixture() async def large_test_file(ofrak_context: OFRAKContext) -> Resource: with tempfile.NamedTemporaryFile() as temp: - for i in range(0, 256, 1): + for i in range(256): temp.write(int.to_bytes(i, 1, "big") * 4096) yield await ofrak_context.create_root_resource_from_file(temp.name) From 60eb2f0dbb57751aad1ecc8deedcbe3b3a16de93 Mon Sep 17 00:00:00 2001 From: Dan Pesce Date: Mon, 19 Jun 2023 17:22:53 -0400 Subject: [PATCH 38/38] Make file bigger --- ofrak_core/test_ofrak/unit/test_ofrak_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ofrak_core/test_ofrak/unit/test_ofrak_server.py b/ofrak_core/test_ofrak/unit/test_ofrak_server.py index ea693e6f8..28da27aca 100644 --- a/ofrak_core/test_ofrak/unit/test_ofrak_server.py +++ b/ofrak_core/test_ofrak/unit/test_ofrak_server.py @@ -33,7 +33,7 @@ def hello_world_elf() -> bytes: async def large_test_file(ofrak_context: OFRAKContext) -> Resource: with tempfile.NamedTemporaryFile() as temp: for i in range(256): - temp.write(int.to_bytes(i, 1, "big") * 4096) + temp.write(int.to_bytes(i, 1, "big") * 1024 * 1024) yield await ofrak_context.create_root_resource_from_file(temp.name)