Implement osh --runtime-mem-dump for benchmarking.

Rename --dump-proc-status-to to --parser-mem-dump.

Don't exit on --parser-mem-dump.  We might want both flags, so rely on
-n to do this.

Problem: this doesn't catch SystemExit from a user.

benchmarks/osh-parser.sh

@@ -63,12 +63,11 @@ parser-task() {
  # Can't use array because of set -u bug!!!  Only fixed in bash 4.4.
  extra_args=''
  if test "$shell_name" = 'osh'; then
    #extra_args='--ast-format none'
    local script_name
    local vm_out_path
    script_name=$(basename $script_path)
    vm_out_path="${vm_out_dir}/${shell_name}-${shell_hash}__${script_name}.txt"
    extra_args="--dump-proc-status-to $vm_out_path"
    extra_args="--ast-format none --parser-mem-dump $vm_out_path"

    # Should we add a field here to say it has VM stats?
  fi

benchmarks/osh-runtime.sh

@@ -111,10 +111,9 @@ conf-task() {
  # Can't use array because of set -u bug!!!  Only fixed in bash 4.4.
  extra_args=''
  if test "$shell_name" = 'osh'; then
    local vm_out_path
    vm_out_path="${vm_out_dir}/$task_label.txt"
    # TODO: Implement this flag
    #extra_args="--runtime-dump-mem $vm_out_path"
    local pdump="${vm_out_dir}/${task_label}__parser.txt"
    local rdump="${vm_out_dir}/${task_label}__runtime.txt"
    extra_args="--parser-mem-dump $pdump --runtime-mem-dump $rdump"

    # Should we add a field here to say it has VM stats?
  fi

benchmarks/virtual-memory.sh

@@ -64,21 +64,33 @@ baseline-csv() {

# Demo of the --dump-proc-status-to flag.
# NOTE: Could also add Python introspection.
dump-demo() {
  local out=_tmp/virtual-memory
  mkdir -p $out
parser-dump-demo() {
  local out_dir=_tmp/virtual-memory
  mkdir -p $out_dir

  # VmRSS: 46 MB for abuild, 200 MB for configure!  That is bad.  This
  # benchmark really is necessary.

  local input=benchmarks/testdata/abuild
  #local input=benchmarks/testdata/configure

  bin/osh \
    --dump-proc-status-to $out/demo.txt \
    --parser-mem-dump $out_dir/parser.txt -n --ast-format none \
    $input

  cat $out/demo.txt
  grep '^Vm' $out_dir/parser.txt
}

runtime-dump-demo() {
  # Multiple processes
  #OIL_TIMING=1 bin/osh -c 'echo $(echo hi)'

  local out_dir=_tmp/virtual-memory
  mkdir -p $out_dir
  bin/osh \
    --parser-mem-dump $out_dir/parser.txt \
    --runtime-mem-dump $out_dir/runtime.txt \
    -c 'echo $(echo hi)'

  grep '^Vm' $out_dir/parser.txt $out_dir/runtime.txt
}

"$@"

bin/oil.py

@@ -23,6 +23,7 @@

import os
import sys
import time  # for perf measurement

# TODO: Set PYTHONPATH from outside?
this_dir = os.path.dirname(os.path.abspath(sys.argv[0]))
@@ -38,7 +39,6 @@

# Uncomment this to see startup time problems.
if os.environ.get('OIL_TIMING'):
  import time
  start_time = time.time()
  def _tlog(msg):
    pid = os.getpid()  # TODO: Maybe remove PID later.
@@ -52,7 +52,7 @@ def _tlog(msg):
import errno
import platform
import re
import traceback  # for debugging
#import traceback  # for debugging

# Set in Modules/main.c.
HAVE_READLINE = os.getenv('_HAVE_READLINE') != ''
@@ -194,8 +194,9 @@ def OshMain(argv, login_shell):
  spec.LongFlag('--trace', ['cmd-parse', 'word-parse', 'lexer'])  # NOTE: can only trace one now
  spec.LongFlag('--hijack-shebang')

  # For benchmarks/virtual-memory.sh.
  spec.LongFlag('--dump-proc-status-to', args.Str)
  # For benchmarks/*.sh
  spec.LongFlag('--parser-mem-dump', args.Str)
  spec.LongFlag('--runtime-mem-dump', args.Str)

  builtin.AddOptionsToArgSpec(spec)

@@ -360,18 +361,17 @@ def OshMain(argv, login_shell):

    # Do this after parsing the entire file.  There could be another option to
    # do it before exiting runtime?
    if opts.dump_proc_status_to:
      import time
    if opts.parser_mem_dump:
      # This might be superstition, but we want to let the value stabilize
      # after parsing.  bash -c 'cat /proc/$$/status' gives different results
      # with a sleep.
      time.sleep(0.001)
      input_path = '/proc/%d/status' % os.getpid()
      with open(input_path) as f, open(opts.dump_proc_status_to, 'w') as f2:
      with open(input_path) as f, open(opts.parser_mem_dump, 'w') as f2:
        contents = f.read()
        f2.write(contents)
        log('Wrote %s to %s', input_path, opts.dump_proc_status_to)
      sys.exit(0)
        log('Wrote %s to %s (--parser-mem-dump)', input_path,
            opts.parser_mem_dump)

    # -n prints AST, --show-ast prints and executes
    if exec_opts.noexec or opts.show_ast:
@@ -410,6 +410,21 @@ def OshMain(argv, login_shell):
    if do_exec:
      _tlog('Execute(node)')
      status = ex.Execute(node)

      # We only do this in the "happy" case for now.  ex.Execute() can raise
      # exceptions.
      if opts.runtime_mem_dump:
        # This might be superstition, but we want to let the value stabilize
        # after parsing.  bash -c 'cat /proc/$$/status' gives different results
        # with a sleep.
        time.sleep(0.001)
        input_path = '/proc/%d/status' % os.getpid()
        with open(input_path) as f, open(opts.runtime_mem_dump, 'w') as f2:
          contents = f.read()
          f2.write(contents)
          log('Wrote %s to %s (--runtime-mem-dump)', input_path,
              opts.runtime_mem_dump)

    else:
      status = 0