Merge pull request #88 from KallistiOS/dont-panic-on-oom

Return -1 and set ENOMEM if sbrk fails

examples/dreamcast/cpp/Makefile

@@ -9,17 +9,20 @@ all:
 	$(KOS_MAKE) -C dcplib
 	$(KOS_MAKE) -C clock
 	$(KOS_MAKE) -C modplug_test
+	$(KOS_MAKE) -C out_of_memory
 
 clean:
 	$(KOS_MAKE) -C gltest clean
 	$(KOS_MAKE) -C dcplib clean
 	$(KOS_MAKE) -C clock clean
 	$(KOS_MAKE) -C modplug_test clean
-
+	$(KOS_MAKE) -C out_of_memory clean
+
 dist:
 	$(KOS_MAKE) -C gltest dist
 	$(KOS_MAKE) -C dcplib dist
 	$(KOS_MAKE) -C clock dist
 	$(KOS_MAKE) -C modplug_test dist
+	$(KOS_MAKE) -C out_of_memory dist
 
 

examples/dreamcast/cpp/out_of_memory/Makefile

@@ -0,0 +1,29 @@
+#
+# C++ out-of-memory demonstration
+# (c)2023 Falco Girgis
+#   
+
+TARGET = out_of_memory.elf
+OBJS = out_of_memory.o
+KOS_CPPFLAGS += -fexceptions
+
+all: rm-elf $(TARGET)
+
+include $(KOS_BASE)/Makefile.rules
+
+clean:
+	-rm -f $(TARGET) $(OBJS) 
+
+rm-elf:
+	-rm -f $(TARGET) 
+
+$(TARGET): $(OBJS) 
+	kos-c++ -o $(TARGET) $(OBJS) -lm
+
+run: $(TARGET)
+	$(KOS_LOADER) $(TARGET)
+
+dist:
+	rm -f $(OBJS)
+	$(KOS_STRIP) $(TARGET)
+

examples/dreamcast/cpp/out_of_memory/out_of_memory.cc

@@ -0,0 +1,64 @@
+#include <vector>
+#include <iostream>
+#include <cstdint>
+#include <malloc.h>
+
+KOS_INIT_FLAGS(INIT_MALLOCSTATS);
+
+void new_handler_cb() {
+    std::cout << "new_handler callback invoked!" << std::endl;
+
+    malloc_stats();
+
+    // Unregister ourself as the new handler, so that next 
+    // iteration will hit the exception handler.
+    std::set_new_handler(NULL);
+}
+
+int main(int argc, char **argv) {
+    std::vector<uint8_t> bytes;
+    bool failed_once = false;
+
+    // Sets the global, static C++ handler for when calls to new fail
+    // this can be used without exceptions enabled!
+    std::set_new_handler(new_handler_cb);
+
+    std::cout << "Beginning out-of-memory demonstration." << std::endl;
+
+    while(true) {
+        try {
+            // Just keep adding bytes until something bad happens!
+            bytes.push_back(0xff);
+
+        } catch(std::bad_alloc const&) {
+
+            if(!failed_once) { 
+                // std::bad_alloc is thrown when a call to new fails
+                std::cout << "Caught std::bad_alloc! Current size: " 
+                          << static_cast<double>(bytes.capacity()) 
+                                / 1024.0 / 1024.0 << "MB" 
+                          << std::endl;
+
+                // Remember, std::vector typically requests RAM in 
+                // powers-of-two, so the actual requested allocation
+                // was probably 2x the size of the vector.
+
+                // Lets force the vector to shrink to free up some
+                // space and ensure we can continue to allocate.
+
+                malloc_stats();
+
+                bytes.resize(0);
+                bytes.shrink_to_fit();
+
+                failed_once = true;
+            } else {
+                break;
+            }
+        }
+    }
+
+    std::cout << "All done. Thank you for the RAM!" << std::endl;
+
+    return !failed_once;
+}

kernel/arch/dreamcast/kernel/mm.c

@@ -17,6 +17,7 @@
 #include <arch/types.h>
 #include <arch/arch.h>
 #include <arch/irq.h>
+#include <errno.h>
 #include <stdio.h>
 
 /* The end of the program is always marked by the '_end' symbol. So we'll
@@ -47,9 +48,12 @@ void* mm_sbrk(unsigned long increment) {
     sbrk_base = (void *)(increment + (unsigned long)sbrk_base);
 
     if(((uint32)sbrk_base) >= (_arch_mem_top - 65536)) {
-        dbglog(DBG_DEAD, "Requested sbrk_base %p, was %p, diff %lu\n",
+        dbglog(DBG_CRITICAL, "Out of memory. Requested sbrk_base %p, was %p, diff %lu\n",
                sbrk_base, base, increment);
-        arch_panic("out of memory; about to run over kernel stack");
+        sbrk_base = base;  /* Restore old value and mark failed */
+        errno = ENOMEM;
+        irq_restore(old);
+        return (void*) -1;
     }
 
     irq_restore(old);