View
@@ -17,7 +17,7 @@
import libc
from core.id_kind import Id
from osh.meta import Id
from core import glob_
from core import util
View
@@ -18,7 +18,7 @@
from core import runtime
from core import util
from core.id_kind import Id
from osh.meta import Id
redirect_e = runtime.redirect_e
e_die = util.e_die
View
@@ -6,7 +6,7 @@
import os
import unittest
from core.id_kind import Id
from osh.meta import Id
from core import builtin
from osh import ast_ as ast
View
@@ -11,7 +11,7 @@
from asdl import py_meta
from asdl import asdl_ as asdl
from core import util
from core.id_kind import Id
from osh.meta import Id
def _LoadSchema(f):
View
@@ -17,7 +17,7 @@
from core import legacy
from core import runtime
from core import util
from core.id_kind import Id
from osh.meta import Id
from osh import ast_ as ast
View
@@ -3,7 +3,7 @@
tdop.py - Library for expression parsing.
"""
from core.id_kind import Id
from osh.meta import Id
from core import word
from core import util
View
@@ -13,14 +13,15 @@
from osh import bool_parse
from osh import ast_ as ast
from osh.meta import Id
from osh import meta
Id = id_kind.Id
log = util.log
_UNARY_LOOKUP = id_kind.TEST_UNARY_LOOKUP
_BINARY_LOOKUP = id_kind.TEST_BINARY_LOOKUP
_OTHER_LOOKUP = id_kind.TEST_OTHER_LOOKUP
_UNARY_LOOKUP = meta.TEST_UNARY_LOOKUP
_BINARY_LOOKUP = meta.TEST_BINARY_LOOKUP
_OTHER_LOOKUP = meta.TEST_OTHER_LOOKUP
class _StringWordEmitter:
View
@@ -10,7 +10,7 @@
"""
from core import alloc
from core.id_kind import Id
from osh.meta import Id
from asdl import py_meta
View
@@ -2,9 +2,9 @@
word.py -- Functions for using words as "tokens".
"""
from osh import ast_ as ast
from core.id_kind import Id, Kind, LookupKind
from osh.meta import Id, Kind, LookupKind
from core import util
from osh import ast_ as ast
from asdl import const
p_die = util.p_die
View
@@ -6,7 +6,7 @@
doesn't depend on any values at runtime.
"""
from core.id_kind import Id
from osh.meta import Id
from core import runtime
var_flags_e = runtime.var_flags_e
View
@@ -9,7 +9,7 @@
from core import expr_eval
from core import libstr
from core import glob_
from core.id_kind import Id, Kind, LookupKind
from osh.meta import Id, Kind, LookupKind
from core import runtime
from core import state
from core import word_compile
View
@@ -1,9 +1,8 @@
OSH Quick Reference
- Below is a list of topics, organized into [Sections].
- Features not yet implemented have an X prefix.
- Oil features are all unimplemented!
- View it on the web at https://www.oilshell.org/release/0.4.0/doc/osh-quick-ref.html
- The X prefix means "unimplemented". Oil features are all unimplemented!
- HTML version: https://www.oilshell.org/release/0.4.0/doc/osh-quick-ref.html
INTRO
[Overview] overview osh-vs-oil command-vs-expr
View
@@ -11,8 +11,7 @@
import unittest
from core import id_kind
from core.id_kind import Id
from osh.meta import Id, IdInstance
from osh import ast_ as ast
import fastlex # module under test
@@ -22,7 +21,7 @@
def MatchToken(lex_mode, line, start_pos):
tok_type, end_pos = fastlex.MatchToken(lex_mode.enum_id, line, start_pos)
return id_kind.IdInstance(tok_type), end_pos
return IdInstance(tok_type), end_pos
def TokenizeLineOuter(line):
View
@@ -5,7 +5,7 @@
from core import tdop
from core import util
from core.id_kind import Id
from osh.meta import Id
from core import word
from osh import ast_ as ast
View
@@ -6,110 +6,10 @@
import sys
from asdl import asdl_ as asdl
from asdl import format as fmt
from asdl import py_meta
from core.id_kind import Id
from core import util
_ColoredString = fmt._ColoredString
MakeTree = fmt.MakeTree
_STRING_LITERAL = fmt._STRING_LITERAL
_OTHER_TYPE = fmt._OTHER_TYPE
def _AbbreviateToken(token, out):
if token.id != Id.Lit_Chars:
c = _ColoredString(str(token.id), _OTHER_TYPE)
out.append(c)
out.append(_ColoredString(token.val, _STRING_LITERAL))
def _GetFieldNames(node):
# Don't let the 'spids' field disable abbreviation
return [n for n, _ in node.fields if n != 'spids']
def AbbreviateNodes(obj, node):
"""
Args:
obj: py_meta.Obj to print
node: homogeneous node after MakeTree prints it; can be mutated
"""
if node.node_type == 'token':
node.abbrev = True
node.node_type = 'T'
node.show_node_type = False
node.left = '<'
node.right = '>'
_AbbreviateToken(obj, node.unnamed_fields)
elif node.node_type == 'LiteralPart':
node.abbrev = True
node.node_type = 'L'
node.show_node_type = False
_AbbreviateToken(obj.token, node.unnamed_fields)
elif node.node_type == 'SimpleVarSub':
node.abbrev = True
node.node_type = '$'
_AbbreviateToken(obj.token, node.unnamed_fields)
elif node.node_type == 'BracedVarSub':
if _GetFieldNames(node) != ['token']:
return # we have other fields to display; don't abbreviate
node.abbrev = True
node.node_type = '${'
_AbbreviateToken(obj.token, node.unnamed_fields)
elif node.node_type == 'DoubleQuotedPart':
node.abbrev = True
node.node_type = 'DQ'
for part in obj.parts:
node.unnamed_fields.append(MakeTree(part, AbbreviateNodes))
# Only abbreviate 'foo', not $'foo\n'
elif (node.node_type == 'SingleQuotedPart' and
obj.left.id == Id.Left_SingleQuote):
node.abbrev = True
node.node_type = 'SQ'
for token in obj.tokens:
node.unnamed_fields.append(MakeTree(token, AbbreviateNodes))
elif node.node_type == 'CompoundWord':
node.abbrev = True
node.node_type = 'W'
node.show_node_type = False
node.left = '{'
node.right = '}'
for part in obj.parts:
node.unnamed_fields.append(MakeTree(part, AbbreviateNodes))
elif node.node_type == 'SimpleCommand':
if _GetFieldNames(node) != ['words']:
return # we have other fields to display; don't abbreviate
node.abbrev = True
node.node_type = 'C'
for w in obj.words:
# Recursively call MakeTree here?
# Well actually then the printer needs to recursively handle it
node.unnamed_fields.append(MakeTree(w, AbbreviateNodes))
def PrettyPrint(node, f=sys.stdout):
ast_f = fmt.DetectConsoleOutput(f)
tree = fmt.MakeTree(node, AbbreviateNodes)
fmt.PrintTree(tree, ast_f)
f.write('\n')
from osh.meta import Id
def LoadSchema(f):
@@ -124,9 +24,6 @@ def LoadSchema(f):
return asdl_module, type_lookup
# TODO: This should be the only lines in this module?
# PrettyPrint can go in osh/ast_lib ? or ast_util?
f = util.GetResourceLoader().open('osh/osh.asdl')
asdl_module, type_lookup = LoadSchema(f)
View
@@ -0,0 +1,112 @@
#!/usr/bin/python
"""
ast_lib.py
Helpers for osh/osh.asdl
"""
import sys
from asdl import format as fmt
from osh.meta import Id
_ColoredString = fmt._ColoredString
MakeTree = fmt.MakeTree
_STRING_LITERAL = fmt._STRING_LITERAL
_OTHER_TYPE = fmt._OTHER_TYPE
def _AbbreviateToken(token, out):
if token.id != Id.Lit_Chars:
c = _ColoredString(str(token.id), _OTHER_TYPE)
out.append(c)
out.append(_ColoredString(token.val, _STRING_LITERAL))
def _GetFieldNames(node):
# Don't let the 'spids' field disable abbreviation
return [n for n, _ in node.fields if n != 'spids']
def AbbreviateNodes(obj, node):
"""
Args:
obj: py_meta.Obj to print
node: homogeneous node after MakeTree prints it; can be mutated
"""
if node.node_type == 'token':
node.abbrev = True
node.node_type = 'T'
node.show_node_type = False
node.left = '<'
node.right = '>'
_AbbreviateToken(obj, node.unnamed_fields)
elif node.node_type == 'LiteralPart':
node.abbrev = True
node.node_type = 'L'
node.show_node_type = False
_AbbreviateToken(obj.token, node.unnamed_fields)
elif node.node_type == 'SimpleVarSub':
node.abbrev = True
node.node_type = '$'
_AbbreviateToken(obj.token, node.unnamed_fields)
elif node.node_type == 'BracedVarSub':
if _GetFieldNames(node) != ['token']:
return # we have other fields to display; don't abbreviate
node.abbrev = True
node.node_type = '${'
_AbbreviateToken(obj.token, node.unnamed_fields)
elif node.node_type == 'DoubleQuotedPart':
node.abbrev = True
node.node_type = 'DQ'
for part in obj.parts:
node.unnamed_fields.append(MakeTree(part, AbbreviateNodes))
# Only abbreviate 'foo', not $'foo\n'
elif (node.node_type == 'SingleQuotedPart' and
obj.left.id == Id.Left_SingleQuote):
node.abbrev = True
node.node_type = 'SQ'
for token in obj.tokens:
node.unnamed_fields.append(MakeTree(token, AbbreviateNodes))
elif node.node_type == 'CompoundWord':
node.abbrev = True
node.node_type = 'W'
node.show_node_type = False
node.left = '{'
node.right = '}'
for part in obj.parts:
node.unnamed_fields.append(MakeTree(part, AbbreviateNodes))
elif node.node_type == 'SimpleCommand':
if _GetFieldNames(node) != ['words']:
return # we have other fields to display; don't abbreviate
node.abbrev = True
node.node_type = 'C'
for w in obj.words:
# Recursively call MakeTree here?
# Well actually then the printer needs to recursively handle it
node.unnamed_fields.append(MakeTree(w, AbbreviateNodes))
def PrettyPrint(node, f=sys.stdout):
ast_f = fmt.DetectConsoleOutput(f)
tree = fmt.MakeTree(node, AbbreviateNodes)
fmt.PrintTree(tree, ast_f)
f.write('\n')
View
@@ -35,7 +35,7 @@
from osh import ast_ as ast
from core import word
from core.id_kind import Id, Kind, LookupKind
from osh.meta import Id, Kind, LookupKind
from core import util
try:
View
@@ -11,7 +11,7 @@
import unittest
from core.id_kind import Id
from osh.meta import Id
from core import test_lib
from osh import ast_ as ast
View
@@ -14,7 +14,7 @@
from core import braces
from core import word
from core.id_kind import Id, Kind
from osh.meta import Id, Kind
from core import util
from osh import ast_ as ast
View
@@ -7,11 +7,12 @@
import unittest
from core import ui
from core.id_kind import Id
from osh.meta import Id
from core import word
from core import test_lib
from osh import ast_ as ast
from osh import ast_lib
from osh import parse_lib
from osh.cmd_parse import CommandParser # module under test
from osh.word_parse import WordParser
@@ -34,7 +35,7 @@ def _assertParseMethod(test, code_str, method, expect_success=True):
node = m()
if node:
ast.PrettyPrint(node)
ast_lib.PrettyPrint(node)
if not expect_success:
test.fail('Expected %r to fail ' % code_str)
else:
@@ -52,7 +53,7 @@ def _assertParseCommandListError(test, code_str):
node = c_parser.ParseCommandLine()
if node:
print('UNEXPECTED:')
ast.PrettyPrint(node)
ast_lib.PrettyPrint(node)
test.fail("Expected %r to fail" % code_str)
return
err = c_parser.Error()
View
@@ -102,7 +102,7 @@
import re
from core.id_kind import Id, Kind, ID_SPEC
from osh.meta import Id, Kind, ID_SPEC
from core.lexer import C, R
from osh import ast_ as ast
View
@@ -5,7 +5,7 @@
import unittest
from core.id_kind import Id, Kind, LookupKind
from osh.meta import Id, Kind, LookupKind
from core.lexer import CompileAll, LineLexer
from core import test_lib
View
@@ -0,0 +1,145 @@
#!/usr/bin/python
"""
meta.py
Another "thin waist" of the interpreter. It can be happen at compile time!
We are following the code <-> data pattern, and this is the "data" module.
id_kind and ASDL are the code.
Usage:
from osh.meta import Id, Kind, ast, ID_SPEC
"""
import sys
# These are metaprogramming libraries. Everything can happen at compile time.
# Could move these to a dir like meta? From meta import id_kind? From meta
# import asdl?
from core import id_kind
#from osh import ast_ # TODO: could be ast_lib?
class Id(object):
"""Token and op type.
The evaluator must consider all Ids.
NOTE: We add a bunch of class attributes that are INSTANCES of this class,
e.g. Id.Lit_Chars.
"""
def __init__(self, enum_value):
self.enum_value = enum_value
def __repr__(self):
return IdName(self)
class Kind(object):
"""A coarser version of Id, used to make parsing decisions."""
# TODO: The Kind type should be folded into ASDL. It can't print itself,
# which is inconsistent with Id.
pass
_ID_TO_KIND = {} # int -> Kind
def LookupKind(id_):
return _ID_TO_KIND[id_.enum_value]
_ID_NAMES = {} # int -> string
def IdName(id_):
return _ID_NAMES[id_.enum_value]
# Keep one instance of each Id, to save memory and enable comparison by
# OBJECT IDENTITY.
# Do NOT create any any more instances of them! Always used IdInstance().
# TODO: Fold this into ASDL, which will enforce this?
_ID_INSTANCES = {} # int -> Id
def IdInstance(i):
return _ID_INSTANCES[i]
# Id -> OperandType
BOOL_OPS = {} # type: dict
# Used by test_builtin.py
TEST_UNARY_LOOKUP = {}
TEST_BINARY_LOOKUP = {}
TEST_OTHER_LOOKUP = {}
#
# Instantiate the spec
#
ID_SPEC = id_kind.IdSpec(Id, Kind,
_ID_NAMES, _ID_INSTANCES, _ID_TO_KIND,
BOOL_OPS)
id_kind._AddKinds(ID_SPEC)
id_kind._AddBoolKinds(ID_SPEC, Id) # must come second
id_kind._SetupTestBuiltin(Id, Kind, ID_SPEC,
TEST_UNARY_LOOKUP, TEST_BINARY_LOOKUP,
TEST_OTHER_LOOKUP)
# Debug
_kind_sizes = ID_SPEC.kind_sizes
#
# Redirect Tables associated with IDs
#
# These might be osh specific.
#
REDIR_DEFAULT_FD = {
# filename
Id.Redir_Less: 0, # cat <input.txt means cat 0<input.txt
Id.Redir_Great: 1,
Id.Redir_DGreat: 1,
Id.Redir_Clobber: 1,
Id.Redir_LessGreat: 1, # TODO: What does echo <>foo do?
# descriptor
Id.Redir_GreatAnd: 1, # echo >&2 means echo 1>&2
Id.Redir_LessAnd: 0, # echo <&3 means echo 0<&3, I think
Id.Redir_TLess: 0, # here word
# here docs included
Id.Redir_DLess: 0,
Id.Redir_DLessDash: 0,
}
def _InitRedirType():
# To break circular import. TODO: Id should really be metaprogrammed in the
# same module!
from osh import ast_ as ast
redir_type_e = ast.redir_type_e
return {
# filename
Id.Redir_Less: redir_type_e.Path,
Id.Redir_Great: redir_type_e.Path,
Id.Redir_DGreat: redir_type_e.Path,
Id.Redir_Clobber: redir_type_e.Path,
Id.Redir_LessGreat: redir_type_e.Path,
# descriptor
Id.Redir_GreatAnd: redir_type_e.Desc,
Id.Redir_LessAnd: redir_type_e.Desc,
Id.Redir_TLess: redir_type_e.Here, # here word
# note: here docs aren't included
}
View
@@ -223,15 +223,12 @@ module osh
--
-- These types are not used in the LST. But they could be statically
-- from values in the LST.
-- derived from values in the LST.
--
-- NOTE: This is invalid: ASDL is case-sensitive!
-- OperandType = Undefined | Path | Int | Str | Other
-- RedirType = Path | Desc | Here
-- Also invalid because of duplicate 'Path' -- to fix
-- bool_operand_type = Undefined | Path | Int | Str | Other
-- redir_type = Path | Desc | Here
redir_type = Path | Desc | Here
--
-- APIs
View
@@ -9,6 +9,7 @@
from osh import lex
from osh import word_parse
from osh import cmd_parse
from osh.meta import Id, IdInstance
# bin/osh should work without compiling fastlex? But we want all the unit
# tests to run with a known version of it.
@@ -17,8 +18,6 @@
except ImportError:
fastlex = None
Id = id_kind.Id
class MatchToken_Slow(object):
"""An abstract matcher that doesn't depend on OSH."""
@@ -50,7 +49,7 @@ def MatchToken_Fast(lex_mode, line, start_pos):
tok_type, end_pos = fastlex.MatchToken(lex_mode.enum_id, line, start_pos)
# IMPORTANT: We're reusing Id instances here. Ids are very common, so this
# saves memory.
return id_kind.IdInstance(tok_type), end_pos
return IdInstance(tok_type), end_pos
def _MakeMatcher():
View
@@ -11,7 +11,7 @@
from asdl import const
from core.id_kind import Id, Kind, LookupKind
from osh.meta import Id, Kind, LookupKind
from core import braces
from core import word
from core import tdop
View
@@ -14,11 +14,12 @@
from asdl import const
from core import alloc
from core.id_kind import Id
from osh.meta import Id
from core import test_lib
from core import word
from osh import ast_ as ast
from osh import ast_lib
from osh import parse_lib
from osh.word_parse import WordParser # module under test
@@ -45,7 +46,7 @@ def _assertReadWordWithArena(test, word_str):
arena, w_parser = _InitWordParserWithArena(word_str)
w = w_parser.ReadWord(lex_mode_e.OUTER)
if w:
ast.PrettyPrint(w)
ast_lib.PrettyPrint(w)
else:
err = w_parser.Error()
test.fail("Couldn't parse %r: %s" % (word_str, err))
@@ -81,7 +82,7 @@ def _assertReadWordFailure(test, word_str):
w_parser = InitWordParser(word_str)
w = w_parser.ReadWord(lex_mode_e.OUTER)
if w:
ast.PrettyPrint(w)
ast_lib.PrettyPrint(w)
test.fail('Expected a parser error, got %r' % w)
else:
print(w_parser.Error())
@@ -347,7 +348,7 @@ def testRead(self):
print('Error in word parser: %s' % e)
self.fail(e)
ast.PrettyPrint(w)
ast_lib.PrettyPrint(w)
if word.CommandId(w) == Id.Eof_Real:
break
@@ -449,7 +450,7 @@ def testReadArith(self):
print('ERROR', err)
self.fail(err)
break
ast.PrettyPrint(w)
ast_lib.PrettyPrint(w)
if word.CommandId(w) in (Id.Eof_Real, Id.Unknown_Tok):
break
View
@@ -53,6 +53,10 @@ replace() {
done < $file
}
replace2() {
sed -r -i "s/^from core.id_kind import/from osh.meta import/g" */*.py
}
trailing-ws() {
sed -r -i 's/[ ]+$//g' "$@"
}
View
@@ -9,7 +9,7 @@
from core import util
from core import word
from core.id_kind import Id
from osh.meta import Id
from asdl import const
from osh import ast_ as ast