Move OPy shell functions out of run.sh.

- Line counts are now in count.sh, and enhanced.
- compare-opcodes updated and moved to new metrics.sh script>
- Remove a function now in fix.sh.

build/metrics.sh

@@ -24,19 +24,20 @@ linecount-nativedeps() {

readonly BYTECODE='bytecode-opy'

# NOTE: Copy to _tmp/{pydeps,pycdeps}.txt for ad hoc analysis.
linecount-pydeps() {
  local app_name=${1:-oil}

  awk '/\.py$/ { print $1 }' _build/$app_name/${BYTECODE}-manifest.txt |
    sort | uniq | xargs wc -l | sort -n
    tee _tmp/pydeps.txt | sort | uniq | xargs wc -l | sort -n
}

# Print table of [num_bytes pyc path]
pyc-bytes() {
  local app_name=${1:-oil}

  awk '/\.pyc$/ { print $1 }' _build/$app_name/${BYTECODE}-manifest.txt |
    sort | uniq | xargs wc --bytes | sort -n
    tee _tmp/pycdeps.txt | sort | uniq | xargs wc --bytes | sort -n
}

_tar-lines() {

opy/count.sh

@@ -0,0 +1,47 @@
#!/bin/bash
#
# Usage:
#   ./count.sh <function name>

set -o nounset
set -o pipefail
set -o errexit

_count() {
  xargs wc -l | sort -n
}

# 8700 lines for tokenizer -> tokens -> parser -> homogeneous nodes ->
# transformer -> ast -> compiler -> byte code
all() {
  echo COMMON
  echo opy_main.py util_opy.py | _count
  echo

  echo PARSER GENERATOR
  echo pytree.py pgen2/*.py | _count
  echo

  # ast is generated
  echo COMPILER2
  ls compiler2/*.py | grep -v ast.py | xargs wc -l | sort -n
  echo

  echo GENERATED CODE
  wc -l compiler2/ast.py
  echo

  echo BYTERUN
  ls byterun/*.py | grep -v 'test' | xargs wc -l | sort -n
  echo

  echo MISC
  echo {misc,tools}/*.py | _count
  echo

  echo SCRIPTS
  echo *.sh */*.sh | _count
  echo
}

"$@"

opy/metrics.sh

@@ -0,0 +1,38 @@
#!/bin/bash
#
# Somewhat like build/metrics.sh for OPy.
#
# Usage:
#   ./metrics.sh <function name>

set -o nounset
set -o pipefail
set -o errexit

# Compare opcodes emitted by compiler2 vs. ones defined.
compare-opcodes() {
  # 67 opcodes emitted
  local pat='self.emit|self.unaryOp|self.binaryOp|self._nameOp|self._implicitNameOp|INPLACE|CALL_FUNCTION'
  egrep "$pat" compiler2/pycodegen.py | egrep -o '[A-Z][A-Z_]+' |
    sort | uniq > _tmp/opcodes-emitted.txt

  # 119 ops?
  PYTHONPATH=. python2 > _tmp/opcodes-defined.txt -c '
from compiler2 import opcode
names = sorted(opcode.opmap)
for n in names:
  print(n)
'

  diff -u _tmp/opcodes-{defined,emitted}.txt | tee _tmp/opcode-diff.txt || true
  
  # Opcodes emitted but not defined?  This is approximate because some opcodes
  # are dynamically defined, like SLICE+0 vs SLICE.
  grep '^+' _tmp/opcode-diff.txt | grep -v SLICE

  # Found issue: VERSION=1 is still there, and UNPACK_TUPLE isn't used!

  wc -l _tmp/opcodes-{defined,emitted}.txt  # 119 defined
}

"$@"

opy/run.sh

@@ -307,50 +307,6 @@ copy-pycompiler-tools() {
  cp -v ~/src/Python-2.7.6/Tools/compiler/{ast.txt,ACKS,README,*.py} tools/
}

compare-emitted() {
  # 67 opcodes emitted
  grep emit compiler/pycodegen.py | egrep -o '[A-Z][A-Z_]+' |
    sort | uniq > _tmp/opcodes-emitted.txt

  # 119 ops?
  PYTHONPATH=. python3 > _tmp/opcodes-defined.txt -c '
from compiler import opcode27
names = sorted(opcode27.opmap)
for n in names:
  print(n)
'

  wc -l _tmp/opcodes-defined.txt

  diff -u _tmp/opcodes-{emitted,defined}.txt
}

# 8700 lines for tokenizer -> tokens -> parser -> homogeneous nodes ->
# transformer -> ast -> compiler -> byte code
count() {
  echo PARSER GENERATOR
  wc -l *.py pgen2/*.py | sort -n
  echo

  # ast is generated
  echo COMPILER2
  ls compiler2/*.py | grep -v ast.py | xargs wc -l | sort -n
  echo

  echo GENERATED CODE
  wc -l compiler2/ast.py
  echo

  echo BYTERUN
  ls byterun/*.py | grep -v 'test' | xargs wc -l | sort -n
  echo
}

# We don't want print statements.
2to3-print() {
  2to3 --fix print --write "$@" 
}

# Features from Python 3 used?  Static types?  I guess Python 3.6 has locals with
# foo: str = 1
#