lwIP: upgrade to upstream STABLE-2_2_1_RELEASE

download_dependencies.sh

@@ -16,7 +16,7 @@ fi
 ## Download LWIP (upstream, unpatched)
 LWIP_REPO_URL="https://github.com/lwip-tcpip/lwip.git"
 LWIP_REPO_FOLDER="common/external_deps/lwip"
-LWIP_BRANCH_NAME="STABLE-2_0_3_RELEASE"
+LWIP_BRANCH_NAME="STABLE-2_2_1_RELEASE"
 if test ! -d "$LWIP_REPO_FOLDER"; then
   git clone --depth 1 -b $LWIP_BRANCH_NAME $LWIP_REPO_URL "$LWIP_REPO_FOLDER"_inprogress || exit 1
   mv "$LWIP_REPO_FOLDER"_inprogress "$LWIP_REPO_FOLDER"

ee/network/tcpip/Makefile

@@ -44,6 +44,7 @@ ps2api_OBJECTS = \
 	tcpip.o
 
 ps2api_IPV4 = \
+	acd.o \
 	icmp.o \
 	ip.o \
 	ip4.o \
@@ -114,6 +115,9 @@ $(EE_OBJS_DIR)api_msg.o: $(LWIP)/src/api/api_msg.c
 $(EE_OBJS_DIR)api_netbuf.o: $(LWIP)/src/api/netbuf.c
 	$(EE_CC) $(EE_CFLAGS) $(EE_INCS) -c $< -o $@
 
+$(EE_OBJS_DIR)acd.o: $(LWIP)/src/core/ipv4/acd.c
+	$(EE_CC) $(EE_CFLAGS) $(EE_INCS) -c $< -o $@
+
 $(EE_OBJS_DIR)icmp.o: $(LWIP)/src/core/ipv4/icmp.c
 	$(EE_CC) $(EE_CFLAGS) $(EE_INCS) -c $< -o $@
 

ee/network/tcpip/src/include/lwipopts.h

@@ -41,17 +41,47 @@
    ------------------------------------
 */
 /**
- * MEM_LIBC_MALLOC==1: Use malloc/free/realloc provided by the C-library
- * instead of lwIP's internal allocator. Default is 0; enabled on EE
- * because newlib's malloc is already linked in and the heap pool is
- * cheaper than a duplicate lwIP heap.
+ * MEM_LIBC_MALLOC==1: use libc's malloc/free for lwIP's mem_malloc /
+ * mem_free (which serve PBUF_RAM allocations and a handful of other
+ * sites — DHCP options, ARP, DNS, slip/ppp/zepif which we don't build).
+ * Internally lwIP's pbuf_alloc(PBUF_RAM) computes the payload pointer
+ * with `LWIP_MEM_ALIGN(p + SIZEOF_STRUCT_PBUF + offset)`, which only
+ * stays inside the allocated chunk when 'p' is already MEM_ALIGNMENT-
+ * aligned. That is why MEM_ALIGNMENT must match the underlying
+ * allocator's alignment — see MEM_ALIGNMENT below.
  */
 #define MEM_LIBC_MALLOC		1
 
-/* MEM_ALIGNMENT: should be set to the alignment of the CPU for which
-   lwIP is compiled. 4 byte alignment -> define MEM_ALIGNMENT to 4, 2
-   byte alignment -> define MEM_ALIGNMENT to 2. */
-#define MEM_ALIGNMENT		64		//SP193: must be 64, to deal with the EE cache design.
+/* MEM_ALIGNMENT: must equal the alignment libc's malloc returns,
+   because pbuf_alloc(PBUF_RAM) computes
+     payload = LWIP_MEM_ALIGN(p + SIZEOF_STRUCT_PBUF + offset)
+   inside an allocation sized assuming p is already MEM_ALIGNMENT-
+   aligned. If MEM_ALIGNMENT > newlib's alignment the payload pointer
+   overruns the chunk and corrupts the next-chunk header on the
+   freelist (TLB misses inside _malloc_r / _free_r).
+
+   16 is the contract baked into the toolchain configuration. See
+   newlib/newlib/configure.host:
+
+       mips64r5900*)
+           machine_dir=r5900
+           newlib_cflags="${newlib_cflags} -DMALLOC_ALIGNMENT=16"
+           ;;
+
+   so for the mips64r5900el-ps2-elf target newlib is built with
+   -DMALLOC_ALIGNMENT=16, which sets _mallocr.c's MALLOC_ALIGNMENT
+   directly (overriding the SIZE_SZ-derived default of 8). 16 is also
+   what samples/malloc_stress observes empirically.
+
+   The historical value here was 64 with a comment about "the EE cache
+   design" (SP193). That was over-cautious: PBUF_POOL pbufs (the only
+   ones touched by IOP->EE DMA + cache invalidate) come from memp's
+   static pools and are always 64-byte aligned regardless of
+   MEM_ALIGNMENT; PBUF_RAM pbufs (TX, ARP, DNS, ...) only see DMA
+   writeback, where misalignment harmlessly over-flushes extra cache
+   lines. The IOP-side lwipopts has used MEM_ALIGNMENT=4 forever for
+   the same reason. */
+#define MEM_ALIGNMENT		16
 
 /**
  * MEM_SIZE: the size of the heap memory. If the application will send
@@ -103,12 +133,19 @@
 #define MEMP_NUM_TCP_SEG		TCP_SND_QUEUELEN
 
 /**
- * LWIP_TCPIP_CORE_LOCKING_INPUT: when LWIP_TCPIP_CORE_LOCKING is enabled,
- * this lets tcpip_input() grab the mutex for input packets as well,
- * instead of allocating a message and passing it to tcpip_thread.
- *
- * ATTENTION: this does not work when tcpip_input() is called from
- * interrupt context!
+ * LWIP_TCPIP_CORE_LOCKING==1: matches lwIP 2.2.1's upstream default. With
+ * LWIP_COMPAT_MUTEX in arch/cc.h the core lock is a binary semaphore taken
+ * directly on the calling app thread for socket/netconn API calls; lwIP
+ * releases it before any blocking I/O wait so the tcpip thread + netif
+ * input continue to make progress. Saves a context switch + sem wait per
+ * API call versus the message-passing alternative.
+ */
+#define LWIP_TCPIP_CORE_LOCKING		1
+
+/**
+ * LWIP_TCPIP_CORE_LOCKING_INPUT==1: tcpip_input() takes the core mutex
+ * directly instead of allocating a message. Safe here because the netif
+ * input callback runs in a regular thread, not interrupt context.
  */
 #define LWIP_TCPIP_CORE_LOCKING_INPUT	1
 
@@ -134,18 +171,13 @@
 #define LWIP_DHCP		1
 #endif
 
-/**
- * DHCP_DOES_ARP_CHECK==1: Do an ARP check on the offered address.
- */
-#define DHCP_DOES_ARP_CHECK	0	//Don't do the ARP check because an IP address would be first required.
-
-/**
- * LWIP_DHCP_CHECK_LINK_UP==1: dhcp_start() only really starts if the netif has
- * NETIF_FLAG_LINK_UP set in its flags. As this is only an optimization and
- * netif drivers might not set this flag, the default is off. If enabled,
- * netif_set_link_up() must be called to continue dhcp starting.
- */
-#define LWIP_DHCP_CHECK_LINK_UP	1
+/* LWIP_DHCP_DOES_ACD_CHECK / LWIP_ACD left at upstream defaults (=1 when
+ * LWIP_DHCP=1) so the RFC 5227 Address Conflict Detection probe runs on
+ * any DHCP-offered IP. EE applications run on user home networks where
+ * IP collisions are a real (if uncommon) failure mode; the few KB of
+ * code and ~1-2 s extra DHCP-bind delay are worth the robustness. The
+ * IOP lwIP build forces both off because ps2link runs in controlled
+ * bench environments where the IRX size + tick savings matter more. */
 
 /*
    ----------------------------------
@@ -208,10 +240,6 @@
    ---------- Socket options ----------
    ------------------------------------
 */
-/* LWIP_SOCKET_SET_ERRNO==1: Set errno when socket functions cannot complete
- * successfully, as required by POSIX. Default is POSIX-compliant.
- */
-#define LWIP_SOCKET_SET_ERRNO	0
 /**
  * LWIP_POSIX_SOCKETS_IO_NAMES==1: Enable POSIX-style sockets functions names.
  * Disable this option if you use a POSIX operating system that uses the same

ee/network/tcpip/src/sys_arch.c

@@ -338,6 +338,15 @@ err_t sys_mbox_trypost(sys_mbox_t *mbox, void *sys_msg)
 	return result;
 }
 
+/* lwIP 2.2.x distinguishes ISR-context posts from task-context posts.
+   The PS2 EE has preemptive scheduling, but our netif input does not run
+   in interrupt context (it goes through ps2ip / SIF callbacks on the EE
+   tcpip thread), so the two paths are equivalent here. */
+err_t sys_mbox_trypost_fromisr(sys_mbox_t *mbox, void *msg)
+{
+	return sys_mbox_trypost(mbox, msg);
+}
+
 void sys_mbox_post(sys_mbox_t *mbox, void *sys_msg)
 {
 	SendMbx(mbox, alloc_msg(), sys_msg);
@@ -415,6 +424,52 @@ void sys_sem_set_invalid(sys_sem_t *sem){
 	*sem=SYS_SEM_NULL;
 }
 
+/* Semaphore-based critical section for lwIP. Replaces the previous
+ * DIntr/EIntr approach, which was unsafe: any code path inside a
+ * SYS_ARCH_PROTECT region that ended up calling newlib's malloc/free
+ * would try to WaitSema on the heap recursive mutex with interrupts
+ * disabled, deadlocking the EE. The sema-based variant lets nested
+ * waits work normally and removes the EE-specific incompatibility
+ * between lwIP and any other library that uses newlib's locks.
+ *
+ * Recursive ownership: lwIP allows SYS_ARCH_PROTECT to nest, so we
+ * track the owning thread + a recursion counter and only Wait/Signal
+ * on the outermost transitions.
+ */
+static int s_protect_sem = -1;
+static int s_protect_count = 0;
+static int s_protect_owner = -1;
+
+sys_prot_t sys_arch_protect(void)
+{
+	int tid = GetThreadId();
+	if (s_protect_count > 0 && s_protect_owner == tid)
+	{
+		s_protect_count++;
+		return 0; /* nested re-entry; outer call will release */
+	}
+	WaitSema(s_protect_sem);
+	s_protect_owner = tid;
+	s_protect_count = 1;
+	return 1; /* outermost level; matching unprotect will release */
+}
+
+void sys_arch_unprotect(sys_prot_t level)
+{
+	if (level == 0)
+	{
+		/* nested unprotect; just decrement */
+		if (s_protect_count > 0)
+		{
+			s_protect_count--;
+		}
+		return;
+	}
+	s_protect_count = 0;
+	s_protect_owner = -1;
+	SignalSema(s_protect_sem);
+}
+
 void sys_init(void)
 {
 	arch_message *prev;
@@ -428,6 +483,15 @@ void sys_init(void)
 	sema.init_count = sema.max_count = SYS_MAX_MESSAGES;
 	MsgCountSema=CreateSema(&sema);
 
+	/* Critical-section sema: binary mutex (init=1, max=1). */
+	sema.attr = 0;
+	sema.option = (u32)"PS2IP_PROTECT";
+	sema.init_count = 1;
+	sema.max_count = 1;
+	s_protect_sem = CreateSema(&sema);
+	s_protect_count = 0;
+	s_protect_owner = -1;
+
 	free_head = &msg_pool[0];
 	prev = &msg_pool[0];
 
@@ -446,17 +510,6 @@ u32_t sys_now(void)
 	return(clock()/1000);
 }
 
-sys_prot_t sys_arch_protect(void)
-{
-	return DIntr();
-}
-
-void sys_arch_unprotect(sys_prot_t level)
-{
-	if(level)
-		EIntr();
-}
-
 void *ps2ip_calloc64(size_t n, size_t size)
 {
 	void *ptr = NULL;

iop/network/smap/src/imports.lst

@@ -66,7 +66,7 @@ I_inet_addr
 I_tcpip_input
 I_netif_set_link_up
 I_netif_set_link_down
-I_tcpip_callback_with_block
+I_tcpip_callback
 ps2ip_IMPORTS_end
 #endif
 

iop/tcpip/tcpip-base/include/lwipopts.h

@@ -8,9 +8,21 @@
 
 /* ---------- Thread options ---------- */
 /**
- * DEFAULT_THREAD_STACKSIZE: The stack size used by any other lwIP thread.
- * The stack size value itself is platform-dependent, but is passed to
- * sys_thread_new() when the thread is created.
+ * DEFAULT_THREAD_STACKSIZE: The stack size used by any other lwIP thread
+ * spawned via sys_thread_new(). In our build that's just the tcpip thread.
+ *
+ * With LWIP_TCPIP_CORE_LOCKING=1 the deep socket-API call chains run on
+ * the calling app thread, not on the tcpip thread; the tcpip thread itself
+ * only dispatches timer callbacks and the occasional tcpip_callback (e.g.
+ * link up/down). Worst-case chain on the tcpip thread is roughly
+ *
+ *   tcpip_thread (56) -> sys_check_timeouts (32) -> lwip_cyclic_timer (32)
+ *     -> tcp_slowtmr (72) or dhcp_fine_tmr -> ~150-200 of inner work
+ *   ~= 350-450 bytes + register-save overhead.
+ *
+ * 0x600 (1.5 KB) is the historical 2.0.3 value and matches the call-chain
+ * profile under LWIP_TCPIP_CORE_LOCKING=1. ~2x margin over the measured
+ * worst case.
  */
 #define DEFAULT_THREAD_STACKSIZE	0x600
 
@@ -108,12 +120,20 @@
 #define PBUF_POOL_SIZE		32	//SP193: should be at least ((TCP_WND/PBUF_POOL_BUFSIZE)+1). But that is too small to handle simultaneous connections.
 
 /**
- * LWIP_TCPIP_CORE_LOCKING_INPUT: when LWIP_TCPIP_CORE_LOCKING is enabled,
- * this lets tcpip_input() grab the mutex for input packets as well,
- * instead of allocating a message and passing it to tcpip_thread.
- *
- * ATTENTION: this does not work when tcpip_input() is called from
- * interrupt context!
+ * LWIP_TCPIP_CORE_LOCKING==1: matches lwIP 2.2.1's upstream default. Socket
+ * and netconn API calls take the core mutex on the calling app thread and
+ * run synchronously, instead of round-tripping through the tcpip thread's
+ * mailbox. Saves a context switch + sem wait per API call. lwIP releases
+ * the core lock before any blocking I/O wait (mbox_fetch on connection
+ * mboxes), so the tcpip thread and SMAP RX can still run to deliver data.
+ */
+#define LWIP_TCPIP_CORE_LOCKING		1
+
+/**
+ * LWIP_TCPIP_CORE_LOCKING_INPUT==1: tcpip_input() takes the core mutex
+ * directly instead of allocating a message. Safe here because the netif
+ * input callback runs in IntrHandlerThread (smap.c) — a normal thread,
+ * not interrupt context — so the lock acquire is allowed.
  */
 #define LWIP_TCPIP_CORE_LOCKING_INPUT	1
 
@@ -140,17 +160,14 @@
 #endif
 
 /**
- * DHCP_DOES_ARP_CHECK==1: Do an ARP check on the offered address.
- */
-#define DHCP_DOES_ARP_CHECK	0	//Don't do the ARP check because an IP address would be first required.
-
-/**
- * LWIP_DHCP_CHECK_LINK_UP==1: dhcp_start() only really starts if the netif has
- * NETIF_FLAG_LINK_UP set in its flags. As this is only an optimization and
- * netif drivers might not set this flag, the default is off. If enabled,
- * netif_set_link_up() must be called to continue dhcp starting.
+ * LWIP_DHCP_DOES_ACD_CHECK==0: skip RFC 5227 Address Conflict Detection on
+ * the DHCP-offered address (replaces the pre-2.2.0 DHCP_DOES_ARP_CHECK).
+ * PS2 networking targets a controlled LAN; the saved code+timer/RAM beats
+ * guarding against a vanishingly unlikely IP collision. Combined with
+ * LWIP_AUTOIP=0 (default) this lets LWIP_ACD default to 0 too.
  */
-#define LWIP_DHCP_CHECK_LINK_UP	1
+#define LWIP_DHCP_DOES_ACD_CHECK	0
+#define LWIP_ACD			0
 
 /*
    ----------------------------------
@@ -213,10 +230,6 @@
    ---------- Socket options ----------
    ------------------------------------
 */
-/* LWIP_SOCKET_SET_ERRNO==1: Set errno when socket functions cannot complete
- * successfully, as required by POSIX. Default is POSIX-compliant.
- */
-#define LWIP_SOCKET_SET_ERRNO	0
 /**
  * LWIP_POSIX_SOCKETS_IO_NAMES==1: Enable POSIX-style sockets functions names.
  * Disable this option if you use a POSIX operating system that uses the same

iop/tcpip/tcpip-base/sys_arch.c

@@ -182,6 +182,13 @@ err_t sys_mbox_trypost(sys_mbox_t *mbox, void *msg){
 	return result;
 }
 
+/* lwIP 2.2.x distinguishes ISR-context posts from task-context posts. The
+   IOP has cooperative scheduling and no preemptive ISRs that interact with
+   the lwIP message queue, so the two are equivalent here. */
+err_t sys_mbox_trypost_fromisr(sys_mbox_t *mbox, void *msg){
+	return sys_mbox_trypost(mbox, msg);
+}
+
 void sys_mbox_post(sys_mbox_t *mbox, void *msg)
 {
 	arch_message *MsgPkt;

iop/tcpip/tcpip-netman/src/exports.tab

@@ -65,7 +65,7 @@ DECLARE_EXPORT_TABLE(ps2ip, 2, 6)
 #endif
 	DECLARE_EXPORT(netif_set_link_up)
 	DECLARE_EXPORT(netif_set_link_down)	//55
-	DECLARE_EXPORT(tcpip_callback_with_block)
+	DECLARE_EXPORT(tcpip_callback)
 	DECLARE_EXPORT(pbuf_coalesce)
 END_EXPORT_TABLE
 

iop/tcpip/tcpip-netman/src/imports.lst

@@ -54,13 +54,16 @@ I_memset
 I_strcpy
 I_strncpy
 I_memcpy
+I_memmove
 I_strlen
 I_strncmp
 I_strtok
 I_strtoul
 I_memcmp
 I_strtol
 I_strcmp
+I_tolower
+I_look_ctype_table
 sysclib_IMPORTS_end
 
 sysmem_IMPORTS_start

iop/tcpip/tcpip-netman/src/ps2ip.c

@@ -35,6 +35,13 @@
 
 #include "ps2ip_internal.h"
 
+/* lwIP 2.2.1's sockets.c writes errno via set_errno(); the IOP IRX has no
+   libc-provided errno storage, so we define it here. The ".data" section
+   name (with leading dot) is critical: a bare "data" attribute creates a
+   separate section that the IRX loader never allocates into IOP RAM, so
+   every set_errno() write would corrupt random memory. */
+int errno __attribute__((section(".data")));
+
 typedef struct pbuf	PBuf;
 typedef struct netif	NetIF;
 typedef struct ip4_addr	IPAddr;