Build fixes to prepare for release.

'scripts/release.sh build-and-test' works again. Details: - Generate ID/AST/lexer code in _devbuild/gen because fastlex.so needs it. It was getting blown away by the 'make clean' in scripts/release.sh build-and-test. - Fix include paths - Fix line count script. - Make the generated MatchToken function 'static inline' rather than 'inline'. Other: - .c files shouldn't be executable.
oilshell · Dec 10, 2017 · 6593c24b15266cf767f4a3e64e71c6c58e6fcac2 · 6593c24
1 parent 4dcfb29
commit 6593c24b15266cf767f4a3e64e71c6c58e6fcac2
diff --git a/benchmarks/auto.sh b/benchmarks/auto.sh
@@ -111,7 +111,6 @@ all() {
  make clean

  build/dev.sh all
-  build/codegen.sh lexer

  _banner 'OSH dev build'
  bin/osh -c 'echo OSH dev build'
diff --git a/build/codegen.sh b/build/codegen.sh
@@ -16,6 +16,8 @@ set -o nounset
 set -o pipefail
 set -o errexit

+source build/common.sh
+
 # Files
 #
 # native/lex.c -- calls generated function?
@@ -24,7 +26,10 @@ set -o errexit
 #
 #  ReadToken(lexer_mode, line, s) -> (t, e)

-# _build/gen/
+# NOTE: These are in _devbuild because fastlex.so need them, and fastlex.so is
+# needed for the Makefile to properly crawl dependencies.
+#
+# _devbuild/gen/
 #    osh-ast.h - lex_mode_e for now
 #    id_kind.h - id_e for now
 #    osh-lex.re2c.c  
@@ -47,19 +52,19 @@ install-re2c() {
 re2c() { _deps/re2c-1.0.3/re2c "$@"; }

 ast-gen() {
-  PYTHONPATH=. osh/ast_gen.py "$@" > _build/gen/osh-ast.h
+  PYTHONPATH=. osh/ast_gen.py "$@" > _devbuild/gen/osh-ast.h
 }

 id-gen() {
-  PYTHONPATH=. core/id_kind_gen.py c > _build/gen/id.h
+  PYTHONPATH=. core/id_kind_gen.py c > _devbuild/gen/id.h
 }

 lexer-gen() { PYTHONPATH=. core/lexer_gen.py "$@"; }

 # _gen/osh_lex.re2c.c
 # This includes osh_ast.h
 osh-lex-gen() {
-  lexer-gen c > _build/gen/osh-lex.re2c.h
+  lexer-gen c > _devbuild/gen/osh-lex.re2c.h
 }

 print-regex() { lexer-gen print-regex; }
@@ -71,13 +76,14 @@ osh-lex-gen-native() {
  # The COMMENT state can match an empty string at the end of a line, e.g.
  # '#\n'.  So we have to turn that warning off.
  re2c -W -Wno-match-empty-string -Werror \
-    -o _build/gen/osh-lex.h _build/gen/osh-lex.re2c.h
+    -o _devbuild/gen/osh-lex.h _devbuild/gen/osh-lex.re2c.h
 }

 # Called by build/dev.sh for fastlex.so.
 ast-id-lex() {
-  mkdir -p _build/gen
+  mkdir -p _devbuild/gen

+  log "-- Generating AST, IDs, and lexer in _devbuild/gen"
  ast-gen
  id-gen
  osh-lex-gen
diff --git a/build/common.sh b/build/common.sh
@@ -17,8 +17,12 @@ readonly PY27=Python-2.7.13

 readonly PREPARE_DIR=_devbuild/cpython-full

+log() {
+  echo "$@" >&2
+}
+
 die() {
-  echo "FATAL: $@" 1>&2
+  log "FATAL: $@"
  exit 1
 }

diff --git a/build/compile.sh b/build/compile.sh
@@ -138,7 +138,7 @@ readonly PREPROC_FLAGS=(
  -D Py_BUILD_CORE
 )

-readonly INCLUDE_PATHS=(-I . -I Include -I ../_build/gen)
+readonly INCLUDE_PATHS=(-I . -I Include -I ../_devbuild/gen)
 readonly CC=${CC:-cc}  # cc should be on POSIX systems

 # BASE_CFLAGS is copied by observation from what configure.ac does on my Ubuntu
diff --git a/build/dev.sh b/build/dev.sh
@@ -59,6 +59,7 @@ pylibc() {
 }

 fastlex() {
+  build/codegen.sh ast-id-lex
  py-ext fastlex build/setup_fastlex.py
  PYTHONPATH=. native/fastlex_test.py
 }
@@ -78,8 +79,6 @@ minimal() {
 all() {
  minimal
  build/codegen.sh
-
-  build/codegen.sh ast-id-lex
  fastlex
 }

diff --git a/build/setup_fastlex.py b/build/setup_fastlex.py
@@ -10,5 +10,5 @@
 setup(name = 'fastlex',
      version = '1.0',
      description = 'Module to speed up lexers',
-      include_dirs = ['_build/gen'],
+      include_dirs = ['_devbuild/gen'],
      ext_modules = [module])
diff --git a/core/lexer_gen.py b/core/lexer_gen.py
@@ -180,6 +180,12 @@ def TranslateRegex(pat):


 def TranslateLexer(lexer_def):
+  # https://stackoverflow.com/questions/12836171/difference-between-an-inline-function-and-static-inline-function
+  # Has to be 'static inline' rather than 'inline', otherwise the
+  # _bin/oil.ovm-dbg build fails (but the _bin/oil.ovm doesn't!).
+  # Since we reference this function in exactly one translation unit --
+  # fastlex.c, the difference is moot, and we just satisfy the compiler.
+
  print r"""
 /* Common stuff */

@@ -189,7 +195,7 @@ def TranslateLexer(lexer_def):
  re2c:yyfill:enable = 0;  // generated code doesn't ask for more input
 */

-inline void MatchToken(int lex_mode, unsigned char* line, int line_len,
+static inline void MatchToken(int lex_mode, unsigned char* line, int line_len,
                       int start_pos, int* id, int* end_pos) {

  // bounds checking
diff --git a/native/fastlex.c b/native/fastlex.c
diff --git a/native/libc.c b/native/libc.c
diff --git a/scripts/count.sh b/scripts/count.sh
@@ -49,7 +49,7 @@ all() {
  echo

  echo 'GENERATED CODE'
-  wc -l _devbuild/*.py _build/gen/* | sort --numeric
+  wc -l _devbuild/*.py _devbuild/gen/* | sort --numeric
  echo

  echo 'TOOLS'
diff --git a/scripts/release.sh b/scripts/release.sh
@@ -81,17 +81,17 @@ remove-files() {
    _devbuild _build _release \
    _tmp/{spec,wild,unit} \
    _tmp/{osh-parser,osh-runtime,vm-baseline,oheap} \
-    _tmp/metrics
+    _tmp/metrics \
+    _tmp/oil-tar-test
 }

 build-and-test() {
  remove-files
-  rm -f _bin/oil.*
+  rm -f -v _bin/oil.* *.so

-  build/dev.sh pylibc  # for libc.so, needed to crawl deps
-  build/doc.sh osh-quick-ref  # for _devbuild/osh_help.py
+  build/dev.sh all  # for {libc,fastlex}.so, needed to crawl deps

-  # TODO: publish these
+  # TODO: publish unit tests.
  test/unit.sh all

  build/prepare.sh configure