8880 improve DTrace error checking

Reviewed by: Tim Kordas <tim.kordas@joyent.com>
Reviewed by: Bryan Cantrill <bryan@joyent.com>
Reviewed by: Richard Lowe <richlowe@richlowe.net>
Approved by: Dan McDonald <danmcd@joyent.com>

usr/src/uts/common/dtrace/dtrace.c

@@ -13127,6 +13127,7 @@ static int
 dtrace_dof_relocate(dof_hdr_t *dof, dof_sec_t *sec, uint64_t ubase)
 {
 	uintptr_t daddr = (uintptr_t)dof;
+	uintptr_t ts_end;
 	dof_relohdr_t *dofr =
 	    (dof_relohdr_t *)(uintptr_t)(daddr + sec->dofs_offset);
 	dof_sec_t *ss, *rs, *ts;
@@ -13142,6 +13143,7 @@ dtrace_dof_relocate(dof_hdr_t *dof, dof_sec_t *sec, uint64_t ubase)
 	ss = dtrace_dof_sect(dof, DOF_SECT_STRTAB, dofr->dofr_strtab);
 	rs = dtrace_dof_sect(dof, DOF_SECT_RELTAB, dofr->dofr_relsec);
 	ts = dtrace_dof_sect(dof, DOF_SECT_NONE, dofr->dofr_tgtsec);
+	ts_end = (uintptr_t)ts + sizeof (dof_sec_t);
 
 	if (ss == NULL || rs == NULL || ts == NULL)
 		return (-1); /* dtrace_dof_error() has been called already */
@@ -13168,6 +13170,11 @@ dtrace_dof_relocate(dof_hdr_t *dof, dof_sec_t *sec, uint64_t ubase)
 				return (-1);
 			}
 
+			if (taddr >= (uintptr_t)ts && taddr < ts_end) {
+				dtrace_dof_error(dof, "bad relocation offset");
+				return (-1);
+			}
+
 			if (!IS_P2ALIGNED(taddr, sizeof (uint64_t))) {
 				dtrace_dof_error(dof, "misaligned setx relo");
 				return (-1);