Merge pull request #533 from evo-lua/async-file-reader

Add an AsyncFileReader module to the FileSystem API

Benchmarks/async-file-loading.lua

@@ -0,0 +1,87 @@
+local AsyncFileReader = require("AsyncFileReader")
+
+local console = require("console")
+local uv = require("uv")
+
+console.startTimer("Generating test fixtures")
+local SAMPLE_SIZE = 250
+local SMALL_FILE_PATH = path.join("Tests", "Fixtures", "test-small.txt")
+local LARGE_FILE_PATH = path.join("Tests", "Fixtures", "test-large.txt")
+local HUGE_FILE_PATH = path.join("Tests", "Fixtures", "test-huge.txt")
+local SMALL_FILE_SIZE_IN_BYTES = math.min(AsyncFileReader.CHUNK_SIZE_IN_BYTES - 1, 32)
+local LARGE_FILE_SIZE_IN_BYTES = 4 * AsyncFileReader.CHUNK_SIZE_IN_BYTES + 1
+local HUGE_FILE_SIZE_IN_BYTES = 1024 * 1024 * 32
+local SMALL_FILE_CONTENTS = string.rep("A", SMALL_FILE_SIZE_IN_BYTES)
+local LARGE_FILE_CONTENTS = string.rep("A", LARGE_FILE_SIZE_IN_BYTES)
+local HUGE_FILE_CONTENTS = string.rep("A", HUGE_FILE_SIZE_IN_BYTES)
+C_FileSystem.WriteFile(SMALL_FILE_PATH, SMALL_FILE_CONTENTS)
+C_FileSystem.WriteFile(LARGE_FILE_PATH, LARGE_FILE_CONTENTS)
+C_FileSystem.WriteFile(HUGE_FILE_PATH, HUGE_FILE_CONTENTS)
+console.stopTimer("Generating test fixtures")
+
+math.randomseed(os.clock())
+local availableBenchmarks = {
+	function()
+		local label = "[ASYNC] Loading a small file repeatedly, many times"
+		console.startTimer(label)
+		for i = 1, SAMPLE_SIZE, 1 do
+			AsyncFileReader:LoadFileContents(SMALL_FILE_PATH)
+		end
+		uv.run()
+		console.stopTimer(label)
+	end,
+	function()
+		local label = "[ASYNC] Loading a large file repeatedly, many times"
+		console.startTimer(label)
+		for i = 1, SAMPLE_SIZE, 1 do
+			AsyncFileReader:LoadFileContents(LARGE_FILE_PATH)
+		end
+		uv.run()
+		console.stopTimer(label)
+	end,
+	function()
+		local label = "[ASYNC] Loading a huge file repeatedly, many times"
+		console.startTimer(label)
+		for i = 1, SAMPLE_SIZE, 1 do
+			AsyncFileReader:LoadFileContents(HUGE_FILE_PATH)
+		end
+		uv.run()
+		console.stopTimer(label)
+	end,
+	function()
+		local label = "[SYNC] Loading a small file repeatedly, many times"
+		console.startTimer(label)
+		for i = 1, SAMPLE_SIZE, 1 do
+			C_FileSystem.ReadFile(SMALL_FILE_PATH)
+		end
+		console.stopTimer(label)
+	end,
+	function()
+		local label = "[SYNC] Loading a large file repeatedly, many times"
+		console.startTimer(label)
+		for i = 1, SAMPLE_SIZE, 1 do
+			C_FileSystem.ReadFile(LARGE_FILE_PATH)
+		end
+		console.stopTimer(label)
+	end,
+	function()
+		local label = "[SYNC] Loading a huge file repeatedly, many times"
+		console.startTimer(label)
+		for i = 1, SAMPLE_SIZE, 1 do
+			C_FileSystem.ReadFile(HUGE_FILE_PATH)
+		end
+		console.stopTimer(label)
+	end,
+}
+
+table.shuffle(availableBenchmarks)
+
+for _, benchmark in ipairs(availableBenchmarks) do
+	benchmark()
+end
+
+console.startTimer("Removing test fixtures")
+C_FileSystem.Delete(SMALL_FILE_PATH)
+C_FileSystem.Delete(LARGE_FILE_PATH)
+C_FileSystem.Delete(HUGE_FILE_PATH)
+console.stopTimer("Removing test fixtures")

BuildTools/Targets/EvoBuildTarget.lua

@@ -27,6 +27,7 @@ local EvoBuildTarget = {
 		"Runtime/API/C_Runtime.lua",
 		"Runtime/API/C_Timer.lua",
 		"Runtime/API/C_WebView.lua",
+		"Runtime/API/FileSystem/AsyncFileReader.lua",
 		"Runtime/API/Networking/HttpServer.lua",
 		"Runtime/API/Networking/WebSocketTestClient.lua",
 		"Runtime/API/Networking/WebSocketServer.lua",

Runtime/API/FileSystem/AsyncFileReader.lua

@@ -0,0 +1,132 @@
+local etrace = require("etrace")
+local uv = require("uv")
+
+local math_ceil = math.ceil
+
+local AsyncFileReader = {
+	events = {
+		"FILE_REQUEST_STARTED",
+		"FILE_REQUEST_FAILED",
+		"FILE_REQUEST_COMPLETED",
+		"FILE_DESCRIPTOR_OPENED",
+		"FILE_STATUS_AVAILABLE",
+		"FILE_CHUNK_AVAILABLE",
+		"FILE_CONTENTS_AVAILABLE",
+		"FILE_DESCRIPTOR_CLOSED",
+	},
+	FILE_MODE_READONLY = 292, -- Octal: 444
+	CHUNK_SIZE_IN_BYTES = 1024 * 256,
+}
+
+etrace.register(AsyncFileReader.events)
+
+function AsyncFileReader:LoadFileContents(fileSystemPath)
+	local payload = {
+		fileSystemPath = fileSystemPath,
+	}
+
+	uv.fs_open(fileSystemPath, "r", AsyncFileReader.FILE_MODE_READONLY, function(errorMessage, fileDescriptor)
+		payload.errorMessage = errorMessage
+		payload.fileDescriptor = fileDescriptor
+
+		if errorMessage then
+			EVENT("FILE_REQUEST_FAILED", payload)
+			return
+		end
+
+		EVENT("FILE_DESCRIPTOR_OPENED", payload)
+	end)
+
+	EVENT("FILE_REQUEST_STARTED", payload)
+end
+
+function AsyncFileReader:FILE_DESCRIPTOR_OPENED(event, payload)
+	uv.fs_fstat(payload.fileDescriptor, function(errorMessage, stat)
+		payload.errorMessage = errorMessage
+		payload.stat = stat
+
+		if errorMessage then
+			EVENT("FILE_REQUEST_FAILED", payload)
+			return
+		end
+
+		EVENT("FILE_STATUS_AVAILABLE", payload)
+	end)
+end
+
+function AsyncFileReader:FILE_STATUS_AVAILABLE(event, payload)
+	payload.lastChunkIndex = math_ceil(payload.stat.size / AsyncFileReader.CHUNK_SIZE_IN_BYTES)
+	payload.chunkIndex = 0
+	payload.cursorPosition = 0
+
+	if payload.stat.type == "directory" then
+		-- On Windows, read requests on directories succeed without returning any data
+		-- Simulating the error returned on other platforms here allows providing a consistent interface
+		payload.errorMessage = "EISDIR: illegal operation on a directory" -- Should use uv_strerror but it isn't currently bound
+		EVENT("FILE_REQUEST_FAILED", payload)
+		return
+	end
+
+	self:ReadNextFileChunk(payload)
+end
+
+function AsyncFileReader:ReadNextFileChunk(payload)
+	-- Consecutive chunked reads may overwrite the payload unless copied
+	payload = table.scopy(payload)
+
+	local totalFileSizeInBytes = payload.stat.size
+	local startOffset = payload.cursorPosition
+
+	local numLeftoverBytes = math.min(AsyncFileReader.CHUNK_SIZE_IN_BYTES, totalFileSizeInBytes - startOffset)
+	if numLeftoverBytes <= 0 then
+		EVENT("FILE_CONTENTS_AVAILABLE", payload)
+		return
+	end
+
+	uv.fs_read(payload.fileDescriptor, numLeftoverBytes, startOffset, function(errorMessage, chunk)
+		payload.errorMessage = errorMessage
+		payload.chunk = chunk
+
+		if errorMessage then
+			EVENT("FILE_REQUEST_FAILED", payload)
+			return
+		end
+
+		EVENT("FILE_CHUNK_AVAILABLE", payload)
+
+		local newOffset = startOffset + numLeftoverBytes
+		payload.cursorPosition = newOffset
+		if newOffset < totalFileSizeInBytes then
+			return self:ReadNextFileChunk(payload)
+		end
+
+		EVENT("FILE_CONTENTS_AVAILABLE", payload)
+	end)
+
+	payload.chunkIndex = payload.chunkIndex + 1
+end
+
+function AsyncFileReader:FILE_CONTENTS_AVAILABLE(event, payload)
+	uv.fs_close(payload.fileDescriptor, function()
+		EVENT("FILE_DESCRIPTOR_CLOSED", payload)
+	end)
+
+	EVENT("FILE_REQUEST_COMPLETED", payload)
+end
+
+function AsyncFileReader:FILE_REQUEST_FAILED(event, payload)
+	if not payload.fileDescriptor then
+		return
+	end
+
+	uv.fs_close(payload.fileDescriptor, function()
+		EVENT("FILE_DESCRIPTOR_CLOSED", payload)
+	end)
+end
+
+etrace.subscribe("FILE_DESCRIPTOR_OPENED", AsyncFileReader)
+etrace.subscribe("FILE_STATUS_AVAILABLE", AsyncFileReader)
+etrace.subscribe("FILE_CONTENTS_AVAILABLE", AsyncFileReader)
+etrace.subscribe("FILE_REQUEST_FAILED", AsyncFileReader)
+
+return AsyncFileReader

Tests/BDD/FileSystem/AsyncFileReader.spec.lua

@@ -0,0 +1,103 @@
+local etrace = require("etrace")
+local uv = require("uv")
+
+local AsyncFileReader = require("AsyncFileReader")
+
+local OLD_CHUNK_SIZE = AsyncFileReader.CHUNK_SIZE_IN_BYTES
+local NEW_CHUNKS_SIZE = 2 -- No point in generating large payloads here
+local MAX_LENGTH_CHUNK = string.rep("A", NEW_CHUNKS_SIZE)
+AsyncFileReader.CHUNK_SIZE_IN_BYTES = NEW_CHUNKS_SIZE
+
+local SMALL_TEST_FILE = "temp-small.txt"
+local LARGE_TEST_FILE = "temp-large.txt"
+local FILE_CONTENTS_SMALL = string.rep("A", NEW_CHUNKS_SIZE - 1)
+local FILE_CONTENTS_LARGE = MAX_LENGTH_CHUNK .. MAX_LENGTH_CHUNK .. "A"
+C_FileSystem.WriteFile(SMALL_TEST_FILE, FILE_CONTENTS_SMALL)
+C_FileSystem.WriteFile(LARGE_TEST_FILE, FILE_CONTENTS_LARGE)
+
+describe("AsyncFileReader", function()
+	describe("LoadFileContents", function()
+		before(function()
+			etrace.enable(AsyncFileReader.events)
+		end)
+
+		after(function()
+			etrace.clear()
+			etrace.disable(AsyncFileReader.events)
+		end)
+
+		it("should fail if the given path is invalid", function()
+			AsyncFileReader:LoadFileContents("does-not-exist")
+			uv.run()
+
+			local events = etrace.filter("FILE_REQUEST_FAILED")
+			assertEquals(#events, 1)
+
+			assertEquals(events[1].name, "FILE_REQUEST_FAILED")
+			assertEquals(events[1].payload.fileSystemPath, "does-not-exist")
+			assertEquals(events[1].payload.errorMessage, "ENOENT: no such file or directory: does-not-exist")
+		end)
+
+		it("should fail if the given path refers to a directory", function()
+			AsyncFileReader:LoadFileContents("Runtime")
+			uv.run()
+
+			local events = etrace.filter("FILE_REQUEST_FAILED")
+			assertEquals(#events, 1)
+
+			assertEquals(events[1].name, "FILE_REQUEST_FAILED")
+			assertEquals(events[1].payload.fileSystemPath, "Runtime")
+			assertEquals(events[1].payload.errorMessage, "EISDIR: illegal operation on a directory")
+		end)
+
+		it("should read a single chunk if the file isn't large enough to warrant buffering", function()
+			AsyncFileReader:LoadFileContents(SMALL_TEST_FILE)
+			uv.run()
+
+			local events = etrace.filter("FILE_CHUNK_AVAILABLE")
+			local numExpectedChunks = 1
+			assertEquals(#events, numExpectedChunks)
+
+			assertEquals(events[1].name, "FILE_CHUNK_AVAILABLE")
+			assertEquals(events[1].payload.cursorPosition, 1)
+			assertEquals(events[1].payload.chunk, "A")
+			assertEquals(events[1].payload.fileSystemPath, SMALL_TEST_FILE)
+			assertEquals(events[1].payload.lastChunkIndex, 1)
+			assertEquals(events[1].payload.chunkIndex, 1)
+		end)
+
+		it("should read multiple chunks if the file is large enough to warrant buffering", function()
+			AsyncFileReader:LoadFileContents(LARGE_TEST_FILE)
+			uv.run()
+
+			local events = etrace.filter("FILE_CHUNK_AVAILABLE")
+			local numExpectedChunks = 3
+			assertEquals(#events, numExpectedChunks)
+
+			assertEquals(events[1].name, "FILE_CHUNK_AVAILABLE")
+			assertEquals(events[1].payload.cursorPosition, 2)
+			assertEquals(events[1].payload.chunk, "AA")
+			assertEquals(events[1].payload.fileSystemPath, LARGE_TEST_FILE)
+			assertEquals(events[1].payload.lastChunkIndex, 3)
+			assertEquals(events[1].payload.chunkIndex, 1)
+
+			assertEquals(events[2].name, "FILE_CHUNK_AVAILABLE")
+			assertEquals(events[2].payload.cursorPosition, 4)
+			assertEquals(events[2].payload.chunk, "AA")
+			assertEquals(events[2].payload.fileSystemPath, LARGE_TEST_FILE)
+			assertEquals(events[2].payload.lastChunkIndex, 3)
+			assertEquals(events[2].payload.chunkIndex, 2)
+
+			assertEquals(events[3].name, "FILE_CHUNK_AVAILABLE")
+			assertEquals(events[3].payload.cursorPosition, 5)
+			assertEquals(events[3].payload.chunk, "A")
+			assertEquals(events[3].payload.fileSystemPath, LARGE_TEST_FILE)
+			assertEquals(events[3].payload.lastChunkIndex, 3)
+			assertEquals(events[3].payload.chunkIndex, 3)
+		end)
+	end)
+end)
+
+C_FileSystem.Delete(SMALL_TEST_FILE)
+C_FileSystem.Delete(LARGE_TEST_FILE)
+AsyncFileReader.CHUNK_SIZE_IN_BYTES = OLD_CHUNK_SIZE

Tests/unit-test.lua

@@ -41,6 +41,7 @@ local specFiles = {
 	"Tests/BDD/imageprocessing-namespace.spec.lua",
 	"Tests/BDD/runtime-namespace.spec.lua",
 	"Tests/BDD/timer-namespace.spec.lua",
+	"Tests/BDD/FileSystem/AsyncFileReader.spec.lua",
 }
 
 local numFailedSections = C_Runtime.RunDetailedTests(#arg > 0 and arg or specFiles)