Merge branch 'master' into tmp/ruby-2.0.x

HISTORY

@@ -1,5 +1,39 @@
 History of Onigmo (Oniguruma-mod)
 
+2014/04/18: [bug]  fix warning about alloca() with MinGW (Issue #35)
+2014/04/17: [spec] add checking the number of capture groups (Issue #24)
+                   Now the maximum number of capture groups is explicitly
+                   limited to ONIG_MAX_CAPTURE_GROUP_NUM (32767).
+                   Also add a new error code: ONIGERR_TOO_MANY_CAPTURE_GROUPS.
+2014/04/17: [bug]  double free may occur
+2014/04/15: [dist] testpy.py: change 'region' to a local variable
+                   Now, it becomes easy to execute a test manually from the
+                   python's interactive shell.
+
+                   E.g.:
+                        $ LD_LIBRARY_PATH=.libs python
+                        >>> import testpy
+                        >>> testpy.x2(r"a+b+", "aab", 0, 3)
+                        OK: /a+b+/ 'aab'
+
+2014/04/12: [new]  (thanks mattn)
+                   Fix build on mingw (Issue #33)
+2014/04/12: [new]  improve multithread support
+                   Add the default multithread implementations for Win32 and
+                   Unix. Add --enable-multithread option in configure.
+2014/04/12: [bug]  deadlock occurs when recursive lock is not allowed
+2014/04/12: [bug]  fix: segmation fault occurs when many groups are used
+                   (Issue #24)
+                   see: https://bugs.ruby-lang.org/issues/8716
+2014/01/22: [new]  (thanks Allan Odgaard)
+                   Add option to indicate search range is not begin/end of
+                   string (Issue #27)
+2014/01/22: [bug]  fix: Backwards search not respecting range (Issue #22)
+2014/01/21: [dist] Merge Oniguruma 5.9.5
+2013/07/29: [new]  (thanks Allan Odgaard)
+                   Add 'ifndef ... endif' guard around thread locking macros
+                   (Issue #23)
+
 2013/07/03: Version 5.13.5
 
 2013/07/03: [dist] update LTVERSION to "5:0:5".

oniguruma.h

@@ -341,6 +341,7 @@ int onigenc_str_bytelen_null P_((OnigEncoding enc, const OnigUChar* p));
 /* config parameters */
 #define ONIG_NREGION                          10
 #define ONIG_MAX_BACKREF_NUM                1000
+#define ONIG_MAX_CAPTURE_GROUP_NUM         32767
 #define ONIG_MAX_REPEAT_NUM               100000
 #define ONIG_MAX_MULTI_BYTE_RANGES_NUM     10000
 /* constants */
@@ -587,6 +588,7 @@ ONIG_EXTERN const OnigSyntaxType*   OnigDefaultSyntax;
 #define ONIGERR_NEVER_ENDING_RECURSION                       -221
 #define ONIGERR_GROUP_NUMBER_OVER_FOR_CAPTURE_HISTORY        -222
 #define ONIGERR_INVALID_CHAR_PROPERTY_NAME                   -223
+#define ONIGERR_TOO_MANY_CAPTURE_GROUPS                      -224
 #define ONIGERR_INVALID_CODE_POINT_VALUE                     -400
 #define ONIGERR_INVALID_WIDE_CHAR_VALUE                      -400
 #define ONIGERR_TOO_BIG_WIDE_CHAR_VALUE                      -401

regerror.c

@@ -133,6 +133,8 @@ onig_error_code_to_format(OnigPosition code)
     p = "too short multibyte code string"; break;
   case ONIGERR_TOO_BIG_BACKREF_NUMBER:
     p = "too big backref number"; break;
+  case ONIGERR_TOO_MANY_CAPTURE_GROUPS:
+    p = "too many capture groups are specified"; break;
   case ONIGERR_INVALID_BACKREF:
 #ifdef USE_NAMED_GROUP
     p = "invalid backref number/name"; break;

regint.h

@@ -280,8 +280,8 @@ extern pthread_mutex_t gOnigMutex;
 # include <stddef.h>
 #endif
 
-#ifdef __BORLANDC__
-#include <malloc.h>
+#ifdef _WIN32
+#include <malloc.h>	/* for alloca() */
 #endif
 
 #ifdef ONIG_DEBUG

regparse.c

@@ -986,6 +986,8 @@ scan_env_add_mem_entry(ScanEnv* env)
   Node** p;
 
   need = env->num_mem + 1;
+  if (need > ONIG_MAX_CAPTURE_GROUP_NUM)
+    return ONIGERR_TOO_MANY_CAPTURE_GROUPS;
   if (need >= SCANENV_MEMNODES_SIZE) {
     if (env->mem_alloc <= need) {
       if (IS_NULL(env->mem_nodes_dynamic)) {
@@ -5000,14 +5002,10 @@ parse_enclose(Node** np, OnigToken* tok, int term, UChar** src, UChar* end,
 	*np = node_new_enclose_memory(env->option, 0);
 	CHECK_NULL_RETURN_MEMERR(*np);
 	num = scan_env_add_mem_entry(env);
-	if (num < 0) {
-	  onig_node_free(*np);
-	  return num;
-	}
-	else if (num >= (int )BIT_STATUS_BITS_NUM) {
-	  onig_node_free(*np);
+	if (num < 0) return num;
+	if (num >= (int )BIT_STATUS_BITS_NUM)
 	  return ONIGERR_GROUP_NUMBER_OVER_FOR_CAPTURE_HISTORY;
-	}
+
 	NENCLOSE(*np)->regnum = num;
 	BIT_STATUS_ON_AT_SIMPLE(env->capture_history, num);
       }

testpy.py

@@ -11,7 +11,6 @@
 nerror = 0
 nsucc = 0
 nfail = 0
-region = 0
 
 onig_encoding = onig.ONIG_ENCODING_EUC_JP
 encoding = onig_encoding[0].name.decode()
@@ -61,7 +60,6 @@ def xx(pattern, target, s_from, s_to, mem, not_match):
     global nerror
     global nsucc
     global nfail
-    global region
 
     reg = onig.OnigRegex()
     einfo = onig.OnigErrorInfo()
@@ -94,6 +92,7 @@ def xx(pattern, target, s_from, s_to, mem, not_match):
                 file=sys.stderr)
         return
 
+    region = onig.onig_region_new()
     r = onig.onig_search(reg, targetp.getptr(), targetp.getptr(-1),
                     targetp.getptr(), targetp.getptr(-1),
                     region, onig.ONIG_OPTION_NONE);
@@ -102,6 +101,7 @@ def xx(pattern, target, s_from, s_to, mem, not_match):
         nerror += 1
         print_result("ERROR", "%s (/%s/ '%s')" % (msg.value, pattern, target),
                 file=sys.stderr)
+        onig.onig_region_free(region, 1)
         return
 
     if r == onig.ONIG_MISMATCH:
@@ -126,6 +126,7 @@ def xx(pattern, target, s_from, s_to, mem, not_match):
                 print_result("FAIL", "/%s/ '%s' %d-%d : %d-%d" % (pattern, target,
                         s_from, s_to, start, end))
     onig.onig_free(reg)
+    onig.onig_region_free(region, 1)
 
 def x2(pattern, target, s_from, s_to):
     xx(pattern, target, s_from, s_to, 0, False)
@@ -143,12 +144,9 @@ def is_unicode_encoding(enc):
                    onig.ONIG_ENCODING_UTF8)
 
 def main():
-    global region
     global onig_encoding
     global encoding
 
-    region = onig.onig_region_new()
-
     # set encoding of the test target
     if len(sys.argv) > 1:
         encs = {"EUC-JP": onig.ONIG_ENCODING_EUC_JP,
@@ -960,7 +958,7 @@ def get_text_writer(fileno, **kwargs):
     x2("\\z", "hello", 5, 5)
     x2("\\n?\\z", "こんにちは", 5, 5)
     x2("\\z", "こんにちは", 5, 5)
-    x2("()" * 32768, "", 0, 0)      # Issue #24
+    x2("()" * 32767, "", 0, 0)      # Issue #24
 
     # character classes (tests for character class optimization)
     x2("[@][a]", "@a", 0, 2);
@@ -1132,7 +1130,6 @@ def get_text_writer(fileno, **kwargs):
     print("\nRESULT   SUCC: %d,  FAIL: %d,  ERROR: %d      (by Onigmo %s)" % (
           nsucc, nfail, nerror, onig.onig_version()))
 
-    onig.onig_region_free(region, 1)
     onig.onig_end()
 
     if (nfail == 0 and nerror == 0):

win32/Makefile

@@ -25,7 +25,7 @@ CC = cl
 RC = rc
 DEFS = -DHAVE_CONFIG_H -DNOT_RUBY -DEXPORT
 RUBYDIR = ..
-PYTHON = python
+PYTHON = py
 
 # Command string to get the version of cl.exe
 _MSC_VER = [for /f %i in ('cmd /c "(echo _MSC_VER>mscver.c) && ($(CC) /EP mscver.c 2>nul) && del mscver.c"') do @exit %i]
@@ -112,7 +112,7 @@ makeargs = $(MFLAGS) CPPFLAGS='$(CPPFLAGS)' CFLAGS='$(CFLAGS)' CC='$(CC)'
 !else
 # batch-mode inference rules
 .c.obj::
-	$(CC) $(CFLAGS) $(CPPFLAGS) $(DEFS) /I. /I.. /Fo.\ /c $<
+	$(CC) $(CFLAGS) $(CPPFLAGS) $(DEFS) /I. /Fo.\ /c $<
 {$(encdir)\}.c{$(encdir)\}.obj::
 	$(CC) $(CFLAGS) $(CPPFLAGS) $(DEFS) /I. /I.. /Fo$(encdir)\ /c $<
 !endif
@@ -214,7 +214,7 @@ $(libbase).def: win32\makedef.py oniguruma.h regenc.h oniggnu.h onigposix.h
 	$(PYTHON) win32\makedef.py > $@
 
 clean:
-	del *.obj $(encdir)\*.obj *.lib *.exp *.dll $(testp).exe $(testc).exe $(testc).obj $(testcu).exe $(testcu).obj win32\*.res
+	del *.obj $(encdir)\*.obj *.lib *.exp *.dll $(testp).exe $(testc).exe $(testcu).exe win32\*.res
 
 
 # backup file suffix

win32/Makefile.mingw

@@ -20,12 +20,12 @@ ARDLL_FLAGS = -shared $(LINKFLAGS)
 LINKFLAGS =
 
 INSTALL = install -c
-CP      = copy
+CP      = cp
 CC = gcc
 RC = windres
 DEFS = -DHAVE_CONFIG_H -DNOT_RUBY -DEXPORT
 RUBYDIR = ..
-PYTHON = python
+PYTHON = py
 
 ifdef DEFFILE
 # Use a .def file to export APIs.
@@ -97,14 +97,14 @@ makeargs = $(MFLAGS) CPPFLAGS='$(CPPFLAGS)' CFLAGS='$(CFLAGS)' CC='$(CC)'
 	$(CC) $(CFLAGS) $(CPPFLAGS) $(DEFS) -I. -I.. -o $@ -c $<
 
 .rc.res:
-	$(RC) -O coff $< $@
+	$(RC) -O coff -I. $< $@
 
 # targets
 default: all
 
 setup:
-	$(CP) win32\config.h config.h
-	$(CP) win32\testc.c  testc.c
+	$(CP) win32/config.h config.h
+	$(CP) win32/testc.c  testc.c
 
 
 all: $(libname) $(dllname) 
@@ -113,7 +113,7 @@ $(libname): $(libobjs) $(encobjs)
 	$(ARLIB) $(ARLIB_FLAGS) $@ $(libobjs) $(encobjs)
 
 $(dllname): $(libobjs) $(encobjs) $(resobj) $(DEFFILE)
-	$(ARDLL) $(libobjs) $(encobjs) $(resobj) -o $@ $(ARDLL_FLAGS)
+	$(ARDLL) $(libobjs) $(encobjs) $(resobj) -o $@ $(ARDLL_FLAGS) -Wl,--out-implib,$(dlllib)
 
 regparse.o:  regparse.c $(onigheaders) config.h st.h
 regext.o:    regext.c   $(onigheaders) config.h
@@ -178,22 +178,22 @@ ptest: $(testp)
 	.\$(testp)
 
 $(testc): $(testc).c $(libname)
-	$(CC) -nologo -o $(testc) -DONIG_EXTERN=extern $(testc).c $(libname)
+	$(CC) -o $(testc) -DONIG_EXTERN=extern $(testc).c $(libname)
 
 $(testp): $(testc).c $(dlllib)
-	$(CC) -nologo -DPOSIX_TEST -o $(testp) $(testc).c $(dlllib)
+	$(CC) -DPOSIX_TEST -o $(testp) $(testc).c $(dlllib)
 
 #$(testc)u.c: test.rb testconvu.rb
 #	ruby -Ke testconvu.rb test.rb > $@
 
 $(testcu): $(testcu).c $(libname)
-	$(CC) -nologo -o $(testcu) -DONIG_EXTERN=extern $(testcu).c $(libname)
+	$(CC) -o $(testcu) -DONIG_EXTERN=extern $(testcu).c $(libname)
 
 $(libbase).def: win32\makedef.py oniguruma.h regenc.h oniggnu.h onigposix.h
 	$(PYTHON) win32\makedef.py > $@
 
 clean:
-	del *.o $(encdir)\*.o *.a *.dll $(testp).exe $(testc).exe $(testc).o $(testcu).exe $(testcu).o win32\*.res
+	rm *.o $(encdir)/*.o *.a *.dll $(testp).exe $(testc).exe $(testcu).exe win32/*.res
 
 
 # backup file suffix

win32/config.h

@@ -29,7 +29,7 @@
 #define HAVE_LIMITS_H 1
 #define HAVE_FCNTL_H 1
 #define HAVE_SYS_UTIME_H 1
-#if defined(__MINGW32__ ) || (defined(_MSC_VER) && _MSC_VER >= 1600)
+#if defined(__MINGW32__) || (defined(_MSC_VER) && _MSC_VER >= 1600)
 #define HAVE_STDINT_H 1
 #endif
 #define uid_t int

win32/makedef.py

@@ -8,41 +8,47 @@
     "oniggnu.h", "onigposix.h"
 )
 
-exclude_symbols = (
+exclude_symbols = [
     "OnigEncodingKOI8",
 
     # USE_UPPER_CASE_TABLE
     "OnigEncAsciiToUpperCaseTable",
     "OnigEncISO_8859_1_ToUpperCaseTable",
-
-    # USE_RECOMPILE_API
-    "onig_recompile",
-    "onig_recompile_deluxe",
-    "re_recompile_pattern",
-
-    # USE_VARIABLE_META_CHARS
-    #"onig_set_meta_char",
-
-    # USE_CAPTURE_HISTORY
-    #"onig_get_capture_tree",
-)
+]
+
+features = {
+    "USE_RECOMPILE_API": ("onig_recompile",
+                "onig_recompile_deluxe",
+                "re_recompile_pattern"),
+    "USE_VARIABLE_META_CHARS": ("onig_set_meta_char",),
+    "USE_CAPTURE_HISTORY": ("onig_get_capture_tree",)
+}
+
+for v in features.values():
+    exclude_symbols += list(v)
+
+# Check if the features are enabled
+with open("regint.h", "r") as f:
+    e = set()
+    for line in f:
+        for k, v in features.items():
+            if re.match(r"^#define\s+" + k + r"\b", line):
+                e |= set(v)
+    exclude_symbols = list(set(exclude_symbols) - e)
 
 symbols = set()
 
 rx1 = re.compile("(ONIG_EXTERN.*)$")
 rx2 = re.compile(r"(\w+)( +PV?_\(\(.*\)\)|\[.*\])?;\s*(/\*.*\*/)?$")
 for filename in header_files:
     with open(filename, "r") as f:
-        while True:
-            line = f.readline()
-            if not line:
-                break
+        for line in f:
             m = rx1.match(line)
             if not m:
                 continue
             s = m.group(1)
             if s[-1] != ';':
-                s += ' ' + f.readline()
+                s += ' ' + next(f)
             m2 = rx2.search(s)
             if m2 and (not m2.group(1) in exclude_symbols):
                 symbols.add(m2.group(1))