Refactored openage/util/fslike to behave alike pathlib

.travis.yml

@@ -10,16 +10,15 @@ compiler:
 env:
   - BUILDMODE=release
   - STATICCHECKER=scan-build
-  - SANITIZER=addr LSAN_OPTIONS=suppressions=lsan.supp ASAN_OPTIONS=check_initialization_order=1
+  - SANITIZER=yes LSAN_OPTIONS=suppressions=etc/lsan.supp ASAN_OPTIONS=check_initialization_order=1
   - SANITIZER=mem     # TODO this needs a custom-built libstd++
-  - SANITIZER=thread
-  - SANITIZER=undef
+  - SANITIZER=thread SANITIZER_FATAL=1
 
 matrix:
   allow_failures:
     - env: SANITIZER=mem
     - compiler: gcc
-      env: SANITIZER=thread
+      env: SANITIZER=thread SANITIZER_FATAL=1
   exclude:
     - compiler: gcc                   # gcc can't do that yet
       env: SANITIZER=mem

CMakeLists.txt

@@ -63,6 +63,7 @@ doxygen_configure(libopenage/ openage/)
 
 add_subdirectory(libopenage/)
 set(PYEXT_LINK_LIBRARY libopenage)
+set(PYEXT_CXXFLAGS "${PYEXT_CXXFLAGS} -include \"${CMAKE_SOURCE_DIR}/libopenage/pyinterface/hacks.h\"")
 add_cython_modules(EMBED NOINSTALL run.py)
 add_subdirectory(openage/)
 add_subdirectory(dist/)

Makefile

@@ -32,9 +32,9 @@ tests: build
 build: $(BUILDDIR)
 	@$(MAKE) $(MAKEARGS) -C $(BUILDDIR)
 
-.PHONY: openage
-openage: $(BUILDDIR)
-	$(MAKE) $(MAKEARGS) -C $(BUILDDIR) openage
+.PHONY: libopenage
+libopenage: $(BUILDDIR)
+	@$(MAKE) $(MAKEARGS) -C $(BUILDDIR) libopenage
 
 .PHONY: codegen
 codegen: $(BUILDDIR)
@@ -142,7 +142,7 @@ help: $(BUILDDIR)/Makefile
 	@echo "targets:"
 	@echo ""
 	@echo "build              -> build entire project"
-	@echo "openage            -> build libopenage"
+	@echo "libopenage         -> build libopenage"
 	@echo "pxdgen             -> generate .pxd files"
 	@echo "cythonize          -> compile .pyx files to .cpp"
 	@echo "compilepy          -> compile .py files to .pyc"

buildsystem/codecompliance/pylint.py

@@ -19,7 +19,7 @@ def find_pyx_modules(dirnames):
 def find_issues(check_files, dirnames):
     """ Invokes the external utility. """
 
-    invocation = ['pylint', '--rcfile=.pylintrc', '--reports=n']
+    invocation = ['pylint', '--rcfile=etc/pylintrc', '--reports=n']
 
     ignored_modules = list(find_pyx_modules(dirnames))
     ignored_modules.append('numpy')

buildsystem/pxdgen.py

@@ -251,17 +251,6 @@ def get_pxd_lines(self):
 
         yield "# " + self.filename
 
-        hacksheader = os.path.abspath("libopenage/pyinterface/hacks.h")
-
-        # Include hacks.h, which contains various hacks and workarounds
-        # that should be run on Cython-compiled files.
-        # See the header file for more info.
-        yield ""
-        yield '# Auto-added by pxdgen. Makes Cython include "hacks.h".'
-        yield '# See that header file for further docs.'
-        yield 'cdef extern from "%s":' % hacksheader
-        yield '    int hacks'
-
         self.parse()
 
         # namespace of the previous pxd annotation
@@ -313,7 +302,7 @@ def postprocess_annotation_line(self, annotation):
 
         if 'cdef ' in annotation:
             self.warnings.append(
-                "there's no need to use 'cdef' in PXD files:\n" +
+                "there's no need to use 'cdef' in PXD annotations:\n" +
                 annotation)
 
         return annotation

buildsystem/python.cmake

@@ -63,8 +63,8 @@ function(python_init)
 
 	set(PYEXT_CXXFLAGS "${PYEXT_CXXFLAGS}" PARENT_SCOPE)
 	set(PYEXT_PYLIB "${PYTHON_LIBRARY}" PARENT_SCOPE)
-	set(PYEXT_SUFFIX "${PYEXT_SUFFIX}" PARENT_SCOPE)
 	set(PYEXT_INCLUDE_DIR "${PYTHON_INCLUDE_DIR}" PARENT_SCOPE)
+	set(PYEXT_SUFFIX "${PYEXT_SUFFIX}" PARENT_SCOPE)
 
 	if(NOT CMAKE_PY_INSTALL_PREFIX)
 		py_exec("import site; print(site.getsitepackages()[0])" PREFIX)
@@ -142,7 +142,7 @@ function(add_cython_modules)
 
 				set_target_properties("${TARGETNAME}" PROPERTIES
 					PREFIX ""
-					SUFFIX ".cpython-34m.so"
+					SUFFIX "${PYEXT_SUFFIX}"
 				)
 			endif()
 
@@ -166,6 +166,7 @@ function(add_cython_modules)
 				set(PRETTY_MODULE_PROPERTIES "${PRETTY_MODULE_PROPERTIES} [standalone]")
 				set(STANDALONE_NEXT FALSE)
 			else()
+				set_target_properties("${TARGETNAME}" PROPERTIES CMAKE_LINK_DEPENDS_NO_SHARED 1)
 				target_link_libraries("${TARGETNAME}" "${PYEXT_LINK_LIBRARY}")
 			endif()
 
@@ -314,7 +315,7 @@ function(python_finalize)
 	file(WRITE "${CMAKE_BINARY_DIR}/py/py_files" "${py_files}")
 	set(BUILDPY_TIMEFILE "${CMAKE_BINARY_DIR}/py/compilepy_timefile")
 	add_custom_command(OUTPUT "${COMPILEPY_TIMEFILE}"
-		COMMAND "${PYTHON}" -m compileall openage
+		COMMAND "${PYTHON}" -m buildsystem.compileall openage
 		COMMAND "${CMAKE_COMMAND}" -E touch "${COMPILEPY_TIMEFILE}"
 		WORKING_DIRECTORY "${CMAKE_SOURCE_DIRECTORY}"
 		DEPENDS ${py_files}

buildsystem/simple

@@ -71,7 +71,7 @@ run python3 -m buildsystem.pxdgen libopenage/**/*.h
 for file in openage/**/*.pyx; do
 
 run python3 -m cython --cplus -3 $file || exit
-run g++ -Wall -Wextra -pedantic -std=c++14 -O2 -fPIC -shared -fwrapv -fstack-protector-strong -D_FORTIFY_SOURCE=2 \
+run g++ -Wall -Wextra -pedantic -std=c++14 -O2 -fPIC -shared -fwrapv -fstack-protector-strong -D_FORTIFY_SOURCE=2 -include libopenage/pyinterface/hacks.h \
         -I/usr/include/python${PYVER} \
         -pthread -Wl,-O2,-Bsymbolic-functions,-z,relro,-rpath,`pwd` -lopenage -L`pwd` \
         -o ${file%.pyx}.cpython-${PYVERDOTLESS}.so ${file%.pyx}.cpp

configure

@@ -31,6 +31,18 @@ def getenv(*varnames, default=""):
     return default
 
 
+def getenv_bool(varname):
+    """
+    fetches a "boolean" environment variable.
+    """
+    value = os.environ.get(varname)
+    if isinstance(value, str):
+        if value.lower() in {"0", "false", "no", "off", "n"}:
+            value = False
+
+    return bool(value)
+
+
 # argparsing
 description = """./configure is a convenience script that creates the build \
 directory,
@@ -59,9 +71,13 @@ ap.add_argument("--optimize", "-O",
                      "is set according to mode if 'auto'. " +
                      "conflicts with --flags")
 ap.add_argument("--sanitize",
-                choices=["none", "addr", "mem", "thread", "undef"],
+                choices=["none", "yes", "mem", "thread"],
                 default=getenv("SANITIZER", default="none"),
-                help="enable one of those (run-time) code sanitizers")
+                help=("enable one of those (run-time) code sanitizers."
+                      "'yes' enables the address and undefined sanitizers."))
+ap.add_argument("--sanitize-fatal", action='store_true',
+                default=getenv_bool("SANITIZER_FATAL"),
+                help="With --sanitize, abort program execution on first find.")
 ap.add_argument("--compiler", "-c",
                 default="",
                 help="compiler suite")
@@ -260,22 +276,27 @@ else:
 
 if args.sanitize != 'none':
     flags.append('-fno-omit-frame-pointer')
+    if args.sanitize_fatal and csuite=='llvm':
+        flags.append('-fno-sanitize-recover')
+else:
+    if args.sanitize_fatal:
+        ap.error('--sanitize-fatal only valid with --sanitize')
 
 if args.sanitize == 'none':
     pass
-elif args.sanitize == 'addr':
+elif args.sanitize == 'yes':
     flags.append('-fsanitize=address')
+    flags.append('-fsanitize=undefined')
 elif args.sanitize == 'mem':
     flags.append('-fsanitize=memory')
-    flags.append('-fsanitize-memory-track-origins=2')
 elif args.sanitize == 'thread':
     flags.append('-fsanitize=thread')
-elif args.sanitize == 'undef':
-    flags.append('-fsanitize=undefined')
+    if csuite in {'gnu'}:
+        flags.append('-fPIC')
+        flags.append('-pie')
 else:
     ap.error("unknown sanitizer mode")
-
-
+all
 flags = shlex.join(flags + shlex.split(args.flags))
 ldflags = shlex.join(ldflags + shlex.split(args.ldflags))
 defines.update(c_flags=flags, cxx_flags=flags, exe_linker_flags=ldflags,

doc/buildsystem.md

@@ -4,7 +4,7 @@ Steps in building openage:
  - run `openage.codegen` (python module) to generate C++ source files (recipe: `codegen`)
  - generate `.pxd` cython extension declaration files from annotated `.h` files (recipe: `pxdgen`)
  - build and link `libopenage.so` (recipe: `openage`)
- - build Cython extension modules (generate cpp files and compile them, via `distutils.cythonize`) (recipe: `python`); those link against libopenage.
+ - build Cython extension modules (generate cpp files and compile them, via `buildsystem.cythonize`) (recipe: `cython`); those link against libopenage.
 
 Additional recipes:
 

doc/code/logger.md

@@ -31,7 +31,7 @@ The `MSG` macro collects all sorts of information, including `__FILE__` and `__L
  - "c++-style" with `operator <<` ("iostreams")
  - "c-style" with `.fmt()` ("printf")
 
-All input is appended to the internal `log::message` object, which can be obtained by auto-conversion to `log::message`.
+All input is appended to the internal `log::message` object. The `MessageBuilder` is auto-converted to `log::message` if needed.
 
 #### message
 

doc/code/pyinterface.md

@@ -6,8 +6,8 @@ and the library `libopenage.so`, which contains all C++ code.
 
 Cython is used for glue code.
 
-Quick overview of Cython
-------------------------
+Cython crash course
+-------------------
 
 Cython modules are written in `.pyx` files, and roughly equivalent to .py files.
 In addition to regular Python syntax, `.pyx` files allow you to define typed
@@ -24,20 +24,30 @@ cdef cppclass Rectangle:
         return this.h * this.w
 
 def foo():
-    Rectangle r
+    cdef Rectangle r
     r.h = 5
     r.w = 6
     return square(r.size())
 ```
 
 `.pyx` files are translated to `.cpp` by Cython ("cythonized") as part of the
 openage build process; syntax errors are shown in this step.
+
 Each `.cpp` file is then compiled to a Python extension module, which may be
 used from everywhere.
 
     - `def` functions and `cdef class` classes can be used from Python.
     - `cdef` functions are suitable for storage in a `C` function pointer.
 
+```
+$ cython --cplus -3 test.pyx
+$ g++ -shared -fPIC -I/usr/include/python3.4m test.cpp -o test.so
+$ python3
+>>> import test
+>>> test.foo()
+900
+```
+
 Cython can use any regular C++ function or type; for that purpose, they
 need to be declared in `.pxd` files (which are analogous to C++ `.h` files):
 
@@ -82,7 +92,7 @@ print(vector_sin(range(10)))
 
 From time to time, it may be useful to have a look at the generated `.cpp` file,
 especially if you require more exotic functionality or something doesn't work
-out as you expected.
+out as you expected. For that purpose, `.html` files are generated next to the `.cpp` files.
 
 
 Calling C++ functions from Python

doc/code/testing.md

@@ -37,7 +37,7 @@ The header `libopenage/testing/testing.h` provides `TestError` and some convenie
  - `TESTFAIL`
     throws `TestError`
  - `TESTFAILMSG("a" << "b")`
-    throws `TestError("AB")`
+    throws `TestError("ab")`
  - `TESTEQUALS(left, right)`
     evaluates `left` and `right`
     throws `TestError(left)` if `left != right`
@@ -95,9 +95,9 @@ Example doctest for a function:
         """
         High-performance, state-of-the-art primality tester.
 
-        > is_prime(23)
+        >>> is_prime(23)
         True
-        > is_prime(42)
+        >>> is_prime(42)
         False
         """
         return not any(p % x == 0 for x in range(2, p))
@@ -121,9 +121,9 @@ Example demo:
     def prime_demo(argv):
         import argparse
         cli = argparse.ArgumentParser()
-        cli.add_argument(max_number, type=int)
+        cli.add_argument('max_number', type=int)
         args = cli.parse_args(argv)
 
-        for p in range(2, max_number):
+        for p in range(2, args.max_number):
             if is_prime(p):
                 print(p)

doc/code_style/mom.cpp

@@ -15,7 +15,7 @@
 
 // Headers from different openage folders.
 #include "../valve.h"
-#include "crossplattform/opengl.h"
+#include "log/log.h"
 
 // Lastly, headers from the same folder.
 #include "half_life.h"

doc/contributing.md

@@ -62,8 +62,10 @@ Rebasing - What the hell is it, and (why) do I need it?
 git checkout master
 git pull
 git checkout my_feature_branch
-git checkout -b my_feature_branch-backup       # you never know, you know?
+# make backup (you never know, you know?)
+git checkout -b my_feature_branch-backup
 get checkout my_feature_branch
+# rebase
 git rebase master
 ```