Get progress in callback for HTTP requests (#8604)

* work-in-progress * fix callback * fix import * Revert "fix import" This reverts commit bdb4d02. * Revert "fix callback" This reverts commit a500e01. * Refactor script_http -> gamesys * Post progress + send total bytes * Test fixes * HTTP test fixes * Test HTTP progress pt 1 * Test fix * Remove ref count in script_http * Stupid mistake! * Add post test * Minor review fixes * Linux fixes --------- Co-authored-by: AGulev <me@agulev.com>
defold · Mar 13, 2024 · 8d5d85f · 8d5d85f
1 parent a291050
commit 8d5d85f
Show file tree

Hide file tree

Showing 41 changed files with 657 additions and 162 deletions.
diff --git a/engine/dlib/src/dlib/configfile.cpp b/engine/dlib/src/dlib/configfile.cpp
@@ -399,7 +399,7 @@ namespace dmConfigFile
         (void) value;
     }
 
-    void HttpContent(dmHttpClient::HResponse response , void* user_data, int status_code, const void* content_data, uint32_t content_data_size)
+    void HttpContent(dmHttpClient::HResponse response , void* user_data, int status_code, const void* content_data, uint32_t content_data_size, int32_t content_length)
     {
         HttpContext* context = (HttpContext*) user_data;
         if (status_code != 200)

diff --git a/engine/dlib/src/dlib/http_cache.cpp b/engine/dlib/src/dlib/http_cache.cpp
@@ -666,7 +666,7 @@ namespace dmHttpCache
         }
     }
 
-    Result Get(HCache cache, const char* uri, const char* etag, FILE** file, uint64_t* checksum)
+    Result Get(HCache cache, const char* uri, const char* etag, FILE** file, uint32_t* file_size, uint64_t* checksum)
     {
         dmMutex::ScopedLock lock(cache->m_Mutex);
 
@@ -693,6 +693,13 @@ namespace dmHttpCache
             FILE* f = fopen(path, "rb");
             if (f)
             {
+                if (file_size)
+                {
+                    fseek(f, 0L, SEEK_END);
+                    *file_size = ftell(f);
+                    fseek(f, 0L, SEEK_SET);
+                }
+
                 *file = f;
                 entry->m_ReadLockCount++;
                 *checksum = entry->m_Info.m_Checksum;

diff --git a/engine/dlib/src/dlib/http_cache.h b/engine/dlib/src/dlib/http_cache.h
@@ -181,7 +181,7 @@ namespace dmHttpCache
      * @param checksum content checksum (dmHashString64)
      * @return RESULT_OK on success.
      */
-    Result Get(HCache cache, const char* uri, const char* etag, FILE** file, uint64_t* checksum);
+    Result Get(HCache cache, const char* uri, const char* etag, FILE** file, uint32_t* file_size, uint64_t* checksum);
 
     /**
      * Set cache entry to verifed

diff --git a/engine/dlib/src/dlib/http_cache_verify.cpp b/engine/dlib/src/dlib/http_cache_verify.cpp
@@ -119,7 +119,7 @@ namespace dmHttpCacheVerify
         return verify_context->m_Result;
     }
 
-    static void HttpContent(dmHttpClient::HResponse, void* user_data, int status_code, const void* content_data, uint32_t content_data_size)
+    static void HttpContent(dmHttpClient::HResponse, void* user_data, int status_code, const void* content_data, uint32_t content_data_size, int32_t content_length)
     {
         VerifyContext* verify_context = (VerifyContext*) user_data;
 

diff --git a/engine/dlib/src/dlib/http_client.cpp b/engine/dlib/src/dlib/http_client.cpp
@@ -220,13 +220,14 @@ namespace dmHttpClient
         }
     }
 
-    static void DefaultHttpContentData(HResponse response, void* user_data, int status_code, const void* content_data, uint32_t content_data_size)
+    static void DefaultHttpContentData(HResponse response, void* user_data, int status_code, const void* content_data, uint32_t content_data_size, int32_t content_length)
     {
         (void) response;
         (void) user_data;
         (void) status_code;
         (void) content_data;
         (void) content_data_size;
+        (void) content_length;
     }
 
     void SetDefaultParams(NewParams* params)
@@ -653,7 +654,7 @@ if (sock_res != dmSocket::RESULT_OK)\
             } else {
                 n = dmMath::Min(to_transfer - total_transferred, response->m_TotalReceived - response->m_ContentOffset);
             }
-            http_content(response, client->m_Userdata, response->m_Status, client->m_Buffer + response->m_ContentOffset, n);
+            http_content(response, client->m_Userdata, response->m_Status, client->m_Buffer + response->m_ContentOffset, n, response->m_ContentLength);
 
             if (response->m_CacheCreator && add_to_cache)
             {
@@ -726,13 +727,14 @@ if (sock_res != dmSocket::RESULT_OK)\
         }
     }
 
-    static void HttpContentConsume(HResponse response, void* user_data, int status_code, const void* content_data, uint32_t content_data_size)
+    static void HttpContentConsume(HResponse response, void* user_data, int status_code, const void* content_data, uint32_t content_data_size, int32_t content_length)
     {
         (void) response;
         (void) user_data;
         (void) status_code;
         (void) content_data;
         (void) content_data_size;
+        (void) content_length;
     }
 
     static Result HandleCached(HClient client, const char* path, Response* response)
@@ -777,8 +779,9 @@ if (sock_res != dmSocket::RESULT_OK)\
         }
 
         FILE* file = 0;
+        uint32_t file_size = 0;
         uint64_t checksum;
-        cache_result = dmHttpCache::Get(client->m_HttpCache, client->m_URI, cache_etag, &file, &checksum);
+        cache_result = dmHttpCache::Get(client->m_HttpCache, client->m_URI, cache_etag, &file, &file_size, &checksum);
         if (cache_result == dmHttpCache::RESULT_OK)
         {
             // NOTE: We have an extra byte for null-termination so no buffer overrun here.
@@ -787,7 +790,7 @@ if (sock_res != dmSocket::RESULT_OK)\
             {
                 nread = fread(client->m_Buffer, 1, BUFFER_SIZE, file);
                 client->m_Buffer[nread] = '\0';
-                client->m_HttpContent(response, client->m_Userdata, response->m_Status, client->m_Buffer, nread);
+                client->m_HttpContent(response, client->m_Userdata, response->m_Status, client->m_Buffer, nread, file_size);
             }
             while (nread > 0);
             dmHttpCache::Release(client->m_HttpCache, client->m_URI, cache_etag, file);
@@ -806,7 +809,7 @@ if (sock_res != dmSocket::RESULT_OK)\
     {
         Result r = RESULT_OK;
 
-        client->m_HttpContent(response, client->m_Userdata, response->m_Status, 0, 0);
+        client->m_HttpContent(response, client->m_Userdata, response->m_Status, 0, 0, 0);
 
         if (strcmp(method, "HEAD") == 0) {
             // A response from a HEAD request should not attempt to read any body despite
@@ -824,7 +827,6 @@ if (sock_res != dmSocket::RESULT_OK)\
             response->m_ContentOffset = 0;
 
             int chunk_size;
-            int chunk_number = 0;
             while(true)
             {
                 chunk_size = 0;
@@ -855,8 +857,6 @@ if (sock_res != dmSocket::RESULT_OK)\
                         r = RESULT_OK;
                         break;
                     }
-
-                    ++chunk_number;
                 }
                 else
                 {
@@ -1057,9 +1057,10 @@ if (sock_res != dmSocket::RESULT_OK)\
         client->m_Statistics.m_DirectFromCache++;
 
         FILE* file = 0;
+        uint32_t file_size = 0;
         uint64_t checksum;
 
-        dmHttpCache::Result cache_result = dmHttpCache::Get(client->m_HttpCache, client->m_URI, info->m_ETag, &file, &checksum);
+        dmHttpCache::Result cache_result = dmHttpCache::Get(client->m_HttpCache, client->m_URI, info->m_ETag, &file, &file_size, &checksum);
         if (cache_result == dmHttpCache::RESULT_OK)
         {
             // NOTE: We have an extra byte for null-termination so no buffer overrun here.
@@ -1068,7 +1069,7 @@ if (sock_res != dmSocket::RESULT_OK)\
             {
                 nread = fread(client->m_Buffer, 1, BUFFER_SIZE, file);
                 client->m_Buffer[nread] = '\0';
-                client->m_HttpContent(&response, client->m_Userdata, 304, client->m_Buffer, nread);
+                client->m_HttpContent(&response, client->m_Userdata, 304, client->m_Buffer, nread, file_size);
             }
             while (nread > 0);
             dmHttpCache::Release(client->m_HttpCache, client->m_URI, info->m_ETag, file);

diff --git a/engine/dlib/src/dlib/http_client.h b/engine/dlib/src/dlib/http_client.h
@@ -68,7 +68,7 @@ namespace dmHttpClient
      * @param content_data Content data
      * @param content_data_size Content data size
      */
-    typedef void (*HttpContent)(HResponse response, void* user_data, int status_code, const void* content_data, uint32_t content_data_size);
+    typedef void (*HttpContent)(HResponse response, void* user_data, int status_code, const void* content_data, uint32_t content_data_size, int32_t content_size);
 
     /**
      * HTTP content-length callback. Invoked for POST-request prior to HttpWrite-callback to determine content-length

diff --git a/engine/dlib/src/test/test_httpcache.cpp b/engine/dlib/src/test/test_httpcache.cpp
@@ -73,7 +73,7 @@ class dmHttpCacheTest : public jc_test_base_class
     {
         FILE* f = 0;
         dmHttpCache::Result r;
-        r = dmHttpCache::Get(cache, uri, etag, &f, checksum);
+        r = dmHttpCache::Get(cache, uri, etag, &f, 0, checksum);
         if (r != dmHttpCache::RESULT_OK)
             return r;
 
@@ -387,7 +387,7 @@ TEST_F(dmHttpCacheTest, UpdateReadlocked)
 
     FILE* file;
     uint64_t checksum;
-    r = dmHttpCache::Get(cache, "uri", "etag", &file, &checksum);
+    r = dmHttpCache::Get(cache, "uri", "etag", &file, 0, &checksum);
     ASSERT_EQ(dmHashString64("data"), checksum);
     ASSERT_EQ(dmHttpCache::RESULT_OK, r);
 
@@ -417,7 +417,7 @@ TEST_F(dmHttpCacheTest, GetWriteLocked)
 
     FILE* file;
     uint64_t checksum;
-    r = dmHttpCache::Get(cache, "uri", "etag", &file, &checksum);
+    r = dmHttpCache::Get(cache, "uri", "etag", &file, 0, &checksum);
     ASSERT_EQ(dmHttpCache::RESULT_LOCKED, r);
 
     dmHttpCache::Add(cache, cache_creator, "data", 4);
@@ -481,7 +481,7 @@ TEST_F(dmHttpCacheTest, PartialUpdate)
 
     FILE* file;
     uint64_t checksum;
-    r = dmHttpCache::Get(cache, "uri", "etag", &file, &checksum);
+    r = dmHttpCache::Get(cache, "uri", "etag", &file, 0, &checksum);
     ASSERT_EQ(dmHttpCache::RESULT_LOCKED, r);
 
     dmHttpCache::Add(cache, cache_creator, "data", 4);

diff --git a/engine/dlib/src/test/test_httpclient.cpp b/engine/dlib/src/test/test_httpclient.cpp
@@ -85,7 +85,7 @@ class dmHttpClientTest: public jc_test_params_class<const char*>
         self->m_Headers[key] = value;
     }
 
-    static void HttpContent(dmHttpClient::HResponse response, void* user_data, int status_code, const void* content_data, uint32_t content_data_size)
+    static void HttpContent(dmHttpClient::HResponse response, void* user_data, int status_code, const void* content_data, uint32_t content_data_size, int32_t content_length)
     {
         dmHttpClientTest* self = (dmHttpClientTest*) user_data;
         self->m_StatusCode = status_code;
@@ -473,7 +473,7 @@ struct HttpStressHelper
         dmHttpClient::Delete(m_Client);
     }
 
-    static void HttpContent(dmHttpClient::HResponse response, void* user_data, int status_code, const void* content_data, uint32_t content_data_size)
+    static void HttpContent(dmHttpClient::HResponse response, void* user_data, int status_code, const void* content_data, uint32_t content_data_size, int32_t content_length)
     {
         HttpStressHelper* self = (HttpStressHelper*) user_data;
         self->m_StatusCode = status_code;

diff --git a/engine/dlib/src/test/test_httpserver.cpp b/engine/dlib/src/test/test_httpserver.cpp
@@ -131,7 +131,7 @@ class dmHttpServerTest: public jc_test_base_class
         }
     }
 
-    static void ClientHttpContent(dmHttpClient::HResponse response, void* user_data, int status_code, const void* content_data, uint32_t content_data_size)
+    static void ClientHttpContent(dmHttpClient::HResponse response, void* user_data, int status_code, const void* content_data, uint32_t content_data_size, int32_t content_length)
     {
         dmHttpServerTest* self = (dmHttpServerTest*) user_data;
         self->m_ClientData.append((const char*) content_data, content_data_size);

diff --git a/engine/engine/src/engine.cpp b/engine/engine/src/engine.cpp
@@ -1307,13 +1307,16 @@ namespace dmEngine
         script_lib_context.m_JobThread       = engine->m_JobThreadContext;
 
         if (engine->m_SharedScriptContext) {
+            script_lib_context.m_ScriptContext = engine->m_SharedScriptContext;
             script_lib_context.m_LuaState = dmScript::GetLuaState(engine->m_SharedScriptContext);
             if (!dmGameSystem::InitializeScriptLibs(script_lib_context))
                 goto bail;
         } else {
+            script_lib_context.m_ScriptContext = engine->m_GOScriptContext;
             script_lib_context.m_LuaState = dmScript::GetLuaState(engine->m_GOScriptContext);
             if (!dmGameSystem::InitializeScriptLibs(script_lib_context))
                 goto bail;
+            script_lib_context.m_ScriptContext = engine->m_GuiScriptContext;
             script_lib_context.m_LuaState = dmScript::GetLuaState(engine->m_GuiScriptContext);
             if (!dmGameSystem::InitializeScriptLibs(script_lib_context))
                 goto bail;

diff --git a/engine/gamesys/src/gamesys/gamesys.h b/engine/gamesys/src/gamesys/gamesys.h
@@ -155,6 +155,7 @@ namespace dmGameSystem
         dmHID::HContext         m_HidContext;
         dmGraphics::HContext    m_GraphicsContext;
         dmJobThread::HContext   m_JobThread;
+        dmScript::HContext      m_ScriptContext;
     };
 
 

diff --git a/engine/gamesys/src/gamesys/gamesys_script.cpp b/engine/gamesys/src/gamesys/gamesys_script.cpp
@@ -37,6 +37,8 @@
 #include "scripts/script_image.h"
 #include "scripts/script_sys_gamesys.h"
 #include "scripts/script_camera.h"
+#include "scripts/script_http.h"
+
 #include "components/comp_gui.h"
 
 #include <dmsdk/gamesys/script.h>
@@ -146,6 +148,7 @@ namespace dmGameSystem
         ScriptCollectionProxyRegister(context);
         ScriptImageRegister(context);
         ScriptSysGameSysRegister(context);
+        ScriptHttpRegister(context);
 
         assert(top == lua_gettop(L));
         return result;
@@ -159,6 +162,7 @@ namespace dmGameSystem
         ScriptResourceFinalize(context);
         ScriptWindowFinalize(context);
         ScriptSysGameSysFinalize(context);
+        ScriptHttpFinalize(context);
     }
 
     void UpdateScriptLibs(const ScriptLibContext& context)