View
57 core/expr_eval.py
@@ -18,18 +18,19 @@
from core.id_kind import BOOL_OPS, OperandType, Id, IdName
from core.util import log
from core.value import TValue
from core import runtime
from osh import ast_ as ast
arith_expr_e = ast.arith_expr_e
bool_expr_e = ast.bool_expr_e # used for dispatch
word_e = ast.word_e
part_value_e = runtime.part_value_e
value_e = runtime.value_e
#from core import word_eval
class ExprEvalError(RuntimeError): pass
class ExprEvalError(RuntimeError):
pass
# In C++ is there a compact notation for {true, i+i}? ArithEvalResult,
# BoolEvalResult, CmdExecResult? Word is handled differntly because it's a
@@ -83,10 +84,11 @@ def _ValToInteger(self, val):
bare word: variable
quoted word: string
"""
is_str, s = val.AsString()
if not is_str:
assert isinstance(val, runtime.value), val
if val.tag != value_e.Str:
# TODO: Error message: expected string but got integer/array
return False, 0
s = val.s
if s.startswith('0x'):
try:
@@ -160,10 +162,10 @@ def _Eval(self, node):
# handle that as a special case.
#if node.id == Id.Node_ArithVar:
if node.tag == arith_expr_e.RightVar:
defined, val = self.mem.Get(node.name)
val = self.mem.Get(node.name)
# By default, undefined variables are the ZERO value. TODO: Respect
# nounset and raise an exception.
if not defined:
if val.tag == value_e.Undef:
return 0
ok, i = self._ValToInteger(val)
@@ -173,7 +175,7 @@ def _Eval(self, node):
raise ExprEvalError()
elif node.tag == arith_expr_e.ArithWord: # constant string
ok, val = self.word_ev.EvalCompoundWord(node.w, elide_empty=False)
ok, val = self.word_ev.EvalWordToString(node.w)
if not ok:
raise ExprEvalError(self.word_ev.Error())
@@ -236,46 +238,22 @@ def _Eval(self, node):
raise AssertionError("Shouldn't get here")
# NOTE: Not used now
def _ValuesAreEqual(x, y):
"""Equality is used for [[.
NOTE: Equality of arrays works!
"""
if x.type != y.type:
# TODO: should we throw an INCOMPARABLE error? Same with -eq on strings.
return False
if x.type == TValue.STRING:
#return x.s == y.s
# RHS is the PATTERN. LHS is the value.
return libc.fnmatch(y.s, x.s)
raise NotImplementedError
class BoolEvaluator(ExprEvaluator):
def _SetRegexMatches(self, matches):
"""For ~= to set the BASH_REMATCH array."""
self.mem
def _EvalCompoundWord(self, word, do_glob=False):
def _EvalCompoundWord(self, word, do_fnmatch=False):
"""
Args:
node: Id.Word_Compound
do_glob: TOOD: rename this
"""
ok, val = self.word_ev.EvalCompoundWord(word, do_glob=do_glob,
elide_empty=False)
ok, val = self.word_ev.EvalWordToString(word, do_fnmatch=do_fnmatch)
if not ok:
raise ExprEvalError(self.word_ev.Error())
is_str, s = val.AsString()
if not is_str:
raise ExprEvaluator("Expected string, got array")
return s
return val.s
def _Eval(self, node):
#print('!!', node.tag)
@@ -334,9 +312,9 @@ def _Eval(self, node):
s1 = self._EvalCompoundWord(node.left)
# Whehter to glob escape
do_glob = op_id in (
do_fnmatch = op_id in (
Id.BoolBinary_Equal, Id.BoolBinary_DEqual, Id.BoolBinary_NEqual)
s2 = self._EvalCompoundWord(node.right, do_glob=do_glob)
s2 = self._EvalCompoundWord(node.right, do_fnmatch=do_fnmatch)
# Now dispatch on arg type
arg_type = BOOL_OPS[op_id]
@@ -372,11 +350,10 @@ def _Eval(self, node):
# - Compare arrays. (Although bash coerces them to string first)
if op_id in (Id.BoolBinary_Equal, Id.BoolBinary_DEqual):
#return True, _ValuesAreEqual(val1, val2)
#log('Comparing %s and %s', s2, s1)
return libc.fnmatch(s2, s1)
if op_id == Id.BoolBinary_NEqual:
#return True, not _ValuesAreEqual(val1, val2)
return not libc.fnmatch(s2, s1)
if op_id == Id.BoolBinary_EqualTilde:
View
68 core/glob_.py
@@ -8,6 +8,8 @@
except ImportError:
from core import fake_libc as libc
from core.util import log
# EXAMPLES
#
# Splitting happens before globbing:
@@ -71,7 +73,7 @@
#
# WordEvaluator(mem, exec_opts)
# EvalCompoundWord
# EvalWords
# EvalWordSequence
# EvalEnv
#
# PartEvaluator(mem, exec_opts)
@@ -90,11 +92,7 @@
# Globber(exec_opts)
# arg_value[] -> string[]
# Expand()
#
# Vertical slice:
# - start by evaluating all parts, no splitting, joining, or globbing
# - just the trivial algorithm of joining all the parts.
# - like IFS='' and noglob?
def LooksLikeGlob():
"""
@@ -171,33 +169,31 @@ def __init__(self, exec_opts):
# TODO: Figure out which ones are in other shells, and only support those?
# - Include globstar since I use it, and zsh has it.
def Expand(self, argv):
result = []
for arg in argv:
# TODO: Only try to glob if there are any glob metacharacters.
# Or maybe it is a conservative "avoid glob" heuristic?
#
# Non-glob but with glob characters:
# echo ][
# echo [] # empty
# echo []LICENSE # empty
# echo [L]ICENSE # this one is good
# So yeah you need to test the validity somehow.
try:
#g = glob.glob(arg) # Bad Python glob
# PROBLEM: / is significant and can't be escaped! Hav eto avoid globbing it.
g = libc.glob(arg)
except Exception as e:
# - [C\-D] is invalid in Python? Regex compilation error.
# - [:punct:] not supported
print("Error expanding glob %r: %s" % (arg, e))
raise
#print('G', arg, g)
if g:
result.extend(g)
else:
u = _GlobUnescape(arg)
result.append(u)
return result
def Expand(self, arg):
# TODO: Only try to glob if there are any glob metacharacters.
# Or maybe it is a conservative "avoid glob" heuristic?
#
# Non-glob but with glob characters:
# echo ][
# echo [] # empty
# echo []LICENSE # empty
# echo [L]ICENSE # this one is good
# So yeah you need to test the validity somehow.
try:
#g = glob.glob(arg) # Bad Python glob
# PROBLEM: / is significant and can't be escaped! Hav eto avoid globbing it.
g = libc.glob(arg)
except Exception as e:
# - [C\-D] is invalid in Python? Regex compilation error.
# - [:punct:] not supported
print("Error expanding glob %r: %s" % (arg, e))
raise
#print('G', arg, g)
#log('Globbing %s', arg)
if g:
return g
else:
u = _GlobUnescape(arg)
return [u]
View
51 core/runtime.asdl
@@ -1,31 +1,46 @@
-- Data types for evaluating an AST (or LST at the moment.)
-- TODO: add ASDL option to put constructors under the vraitn namespace:
-- part_value.String, part_value.Array
-- fragment
-- arg.Const, arg.Glob
-- value.Str, value StrArray,
module runtime
{
-- A part value is the result of evaluating a WordPart.
-- A static word_part from os.asdl is evaluated to a dynamic part_value.
part_value =
StringPartValue(string s, bool do_split_elide, bool do_glob)
-- "$@" or "${a[@]}" -- never split or globbed since double quoted
| ArrayPartValue(string* s)
-- A string part after splitting. Parts may still be elided.
| SplitPartValue(string* s, bool do_elide, bool do_glob)
-- UndefPartValue is for internal processing only.
UndefPartValue
-- Substitutions get split/elided/globbed. Unquoted literals also get
-- globbed.
| StringPartValue(string s, bool? do_split_elide, bool? do_glob)
-- "$@" or "${a[@]}" -- never split or globbed since double quoted.
| ArrayPartValue(string* strs)
-- Split() algorithm works on StringPartValue only
-- JoinAndGlobEscape() works on ArrayPartValue() and SplitPartValue()
-- Elide() -- after joining.
-- Example: {a,b,} has two args (third is elided)(, but {a,b,}'' has 3
-- if empty and do_elide?
-- part_values are split into fragments. Fragments may still be elided
-- and globbed.
fragment = (string s, bool do_elide, bool do_glob)
-- With regard to joining, ArrayPartValue() behaves just like
-- SplitPartValue(do_glob=False), but we model them separately for a little
-- type safety.
-- TODO: ArgValue value? Or those could just be strings?
-- After word splitting, we will get of arg_values.
-- We reframe and join fragments into an array of arg_value. If any
-- fragment in an arg had do_glob set, the whole arg is globbed, with
-- quoted fragments being glob-escaped.
-- e.g. "my[]dir/"*.py -> my\[\]dir/*.py.
arg_value =
ConstArg(string s)
| GlobArg(string s) -- non-glob parts glob-escaped
-- After glob expansion, we get an array of simple strings.
-- A static word from osh.asdl is evaluted to a dynamic value. value
-- instances are stored in memory.
value =
-- Undef isn't visible at the language level. We model it as a value
-- rather than an error code because it's manipulated with ${a:-default}
-- and such.
Undef
| Str(string s)
| StrArray(string* strs)
-- For Oil?
-- | ArrayInt(int* array_int)
-- | ArrayBool(bool* a)
}
View
122 core/value.py

This file was deleted.

Oops, something went wrong.
View
953 core/word_eval.py

Large diffs are not rendered by default.

Oops, something went wrong.
View
8 core/word_eval_test.py
@@ -22,7 +22,7 @@ def testIfsSplitEmpty(self):
self.assertEqual(
[], word_eval._IfsSplit(' ', ' \t\n'))
self.assertEqual(
[], word_eval._IfsSplit('', ' '))
[''], word_eval._IfsSplit('', ' '))
# No word splitting when no IFS. Hm.
self.assertEqual(
@@ -59,16 +59,16 @@ def testIfsSplit(self):
word_eval._IfsSplit('abbc', 'b '))
self.assertEqual(
['a', '', '', 'cd'],
['', 'a', '', '', 'cd', ''],
word_eval._IfsSplit('\ta b\tb cd\n', 'b \t\n'))
self.assertEqual(
['a', 'cd'],
['', 'a', 'cd', ''],
word_eval._IfsSplit('\tabcd\n', 'b \t\n'))
# No non-whitespace IFS
self.assertEqual(
['a', 'c'],
['', 'a', 'c', ''],
word_eval._IfsSplit(' a c ', ' '))
View
1 sh_spec.py
@@ -818,6 +818,7 @@ def main(argv):
stats = RunCases(cases, case_predicate, shell_pairs, env, out)
out.EndCases(stats)
stats['osh_failures_allowed'] = opts.osh_failures_allowed
if opts.stats_file:
with open(opts.stats_file, 'w') as f:
f.write(opts.stats_template % stats)
View
9 spec-runner.sh
@@ -49,7 +49,8 @@ run-cases() {
./spec.sh $spec_name \
--format html \
--stats-file _tmp/spec/${spec_name}.stats.txt \
--stats-template '%(num_cases)d %(osh_num_passed)d %(osh_num_failed)d' \
--stats-template \
'%(num_cases)d %(osh_num_passed)d %(osh_num_failed)d %(osh_failures_allowed)d' \
> _tmp/spec/${spec_name}.html
}
@@ -100,7 +101,7 @@ _html-summary() {
<thead>
<tr>
<td>name</td> <td>Exit Code</td> <td>Elapsed Seconds</td>
<td># cases</td> <td>osh # passed</td> <td>osh # failed</td>
<td># cases</td> <td>osh # passed</td> <td>osh # failed</td> <td>osh failures allowed </d>
</tr>
</thead>
EOF
@@ -123,12 +124,14 @@ EOF
num_cases = $1
osh_num_passed = $2
osh_num_failed = $3
osh_failures_allowed = $4
sum_status += status
sum_wall_secs += wall_secs
sum_num_cases += num_cases
sum_osh_num_passed += osh_num_passed
sum_osh_num_failed += osh_num_failed
sum_osh_failures_allowed += osh_failures_allowed
num_rows += 1
# For the console
@@ -155,6 +158,7 @@ EOF
print "<td>" num_cases "</td>"
print "<td>" osh_num_passed "</td>"
print "<td>" osh_num_failed "</td>"
print "<td>" osh_failures_allowed "</td>"
print "</tr>"
}
@@ -167,6 +171,7 @@ EOF
print "<td>" sum_num_cases "</td>"
print "<td>" sum_osh_num_passed "</td>"
print "<td>" sum_osh_num_failed "</td>"
print "<td>" sum_osh_failures_allowed "</td>"
print "</tr>"
print "</tfoot>"
View
15 spec.sh
@@ -168,12 +168,16 @@ comments() {
sh-spec tests/comments.test.sh ${REF_SHELLS[@]} $OSH "$@"
}
# TODO(pysh): Implement ${foo:-a b c}
word-split() {
sh-spec tests/word-split.test.sh --osh-failures-allowed 3 \
${REF_SHELLS[@]} $OSH "$@"
}
word-eval() {
sh-spec tests/word-eval.test.sh --osh-failures-allowed 3 \
${REF_SHELLS[@]} $OSH "$@"
}
# 'do' -- detected statically as syntax error? hm.
assign() {
sh-spec tests/assign.test.sh --osh-failures-allowed 1 \
@@ -282,6 +286,11 @@ tilde() {
sh-spec tests/tilde.test.sh ${REF_SHELLS[@]} $OSH "$@"
}
var-op-test() {
sh-spec tests/var-op-test.test.sh --osh-failures-allowed 1 \
${REF_SHELLS[@]} $OSH "$@"
}
var-sub() {
# NOTE: ZSH has interesting behavior, like echo hi > "$@" can write to TWO
# FILES! But ultimately we don't really care, so I disabled it.
@@ -315,10 +324,8 @@ arith-context() {
$BASH $MKSH $ZSH $OSH "$@"
}
# TODO: array= (a b c) vs array=(a b c). I think LookAheadForOp might still be
# messed up.
array() {
sh-spec tests/array.test.sh --osh-failures-allowed 25 \
sh-spec tests/array.test.sh --osh-failures-allowed 22 \
$BASH $MKSH $OSH "$@"
}
View
17 tests/array.test.sh
@@ -10,10 +10,15 @@ a=(1 '2 3')
echo $a
# stdout: 1
### ${a[@]} and ${a[*]} give all elements of array
### "${a[@]}" and "${a[*]}"
a=(1 '2 3')
echo "${a[@]}" "${a[*]}"
# stdout: 1 2 3 1 2 3
argv.py "${a[@]}" "${a[*]}"
# stdout: ['1', '2 3', '1 2 3']
### ${a[@]} and ${a[*]}
a=(1 '2 3')
argv.py ${a[@]} ${a[*]}
# stdout: ['1', '2', '3', '1', '2', '3']
### local array
# mksh support local variables, but not local arrays, oddly.
@@ -298,3 +303,9 @@ echo "${a[@]}"
a=(-{a,b} {c,d}-)
echo "${a[@]}"
# stdout: -a -b c- d-
### array default
default=('1 2' '3')
argv.py "${undef[@]:-${default[@]}}"
# stdout: ['1 2', '3']
View
13 tests/var-op-test.test.sh
@@ -0,0 +1,13 @@
#!/bin/bash
### Lazy Evaluation of Alternative
i=0
x=x
echo ${x:-$((i++))}
echo $i
echo ${undefined:-$((i++))}
echo $i # i is one because the alternative was only evaluated once
# status: 0
# stdout-json: "x\n0\n0\n1\n"
# N-I dash status: 2
# N-I dash stdout-json: "x\n0\n"
View
2 tests/var-sub.test.sh
@@ -135,7 +135,7 @@ echo ref ${!a}
# BUG mksh stdout: ref a
# N-I dash/zsh stdout-json: ""
### Local Var
### Dynamic Scope
# Oh this is interesting. Local vars in a function are visible to the function
# it calls. That is not how functions work! Functions are supposed to take
# params.
View
86 tests/word-eval.test.sh
@@ -0,0 +1,86 @@
#!/bin/bash
#
# word-eval.test.sh: Test the word evaluation pipeline in order.
#
# Part evaluation, splitting, joining, elision, globbing.
# TODO: Rename word-eval-smoke.test.sh?
# Word sequence evaluation.
# This is more like a vertical slice. For exhaustive tests, see:
#
# word-split.test.sh (perhaps rename word-reframe?)
# glob.test.sh
### Evaluation of constant parts
argv.py bare 'sq'
# stdout: ['bare', 'sq']
### Evaluation of each part
#set -o noglob
HOME=/home/bob
str=s
array=(a1 a2)
argv.py bare 'sq' ~ $str "-${str}-" "${array[@]}" $((1+2)) $(echo c) `echo c`
# stdout: ['bare', 'sq', '/home/bob', 's', '-s-', 'a1', 'a2', '3', 'c', 'c']
# N-I dash stdout-json: ""
# N-I dash status: 2
### Word splitting
s1='1 2'
s2='3 4'
s3='5 6'
argv.py $s1$s2 "$s3"
# stdout: ['1', '23', '4', '5 6']
### Word joining
set -- x y z
s1='1 2'
array=(a1 a2)
argv.py $s1"${array[@]}"_"$@"
# stdout: ['1', '2a1', 'a2_x', 'y', 'z']
# N-I dash stdout-json: ""
# N-I dash status: 2
### Word elision
s1=''
argv.py $s1 - "$s1"
# stdout: ['-', '']
### Word elision with space
s1=' '
argv.py $s1
# stdout: []
### Word elision with IFS
# Treated differently than the default IFS. What is the rule here?
IFS=_
s1='_'
argv.py $s1
# stdout: ['']
### Default value with single quotes
argv ${undef:-'a b'}
# stdout: ['a b']
### Default values -- more cases
argv ${undef:-hi} "${undef:-c d}" "${un:-"e f"}" "${un:-'g h'}"
# stdout: ['hi', 'c d', 'e f', "'g h'"]
### Globbing after splitting
touch _tmp/foo.gg _tmp/bar.gg _tmp/foo.hh
pat='_tmp/*.hh _tmp/*.gg'
argv $pat
# stdout: ['_tmp/foo.hh', '_tmp/bar.gg', '_tmp/foo.gg']
### Globbing escaping
touch '_tmp/[bc]ar.mm' # file that looks like a glob pattern
touch _tmp/bar.mm _tmp/car.mm
argv '_tmp/[bc]'*.mm - _tmp/?ar.mm
# stdout: ['_tmp/[bc]ar.mm', '-', '_tmp/bar.mm', '_tmp/car.mm']
### Assignment Causes Array Decay
set -- x y z
#argv "[$@]" # NOT DECAYED here.
var="[$@]"
argv "$var"
# stdout: ['[x y z]']