Permalink
Switch branches/tags
Nothing to show
Find file Copy path
Fetching contributors…
Cannot retrieve contributors at this time
211 lines (187 sloc) 6.37 KB
with builtins;
# Tokenizer.
let
layout_pat = "[ \n]+";
layout_pat_opt = "[ \n]*";
token_pat = ''=|[[][[][a-zA-Z0-9_."*-]+[]][]]|[[][a-zA-Z0-9_."*-]+[]]|[a-zA-Z0-9_-]+|"[^"]*"''; #"
tokenizer_1_11 = str:
let
tokenizer_rec = len: prevTokens: patterns: str:
let
pattern = head patterns;
layoutAndTokens = match pattern str;
matchLength = stringLength (head layoutAndTokens);
tokens = prevTokens ++ tail layoutAndTokens;
in
if layoutAndTokens == null then
# if we cannot reduce the pattern, return the list of token
if tail patterns == [] then prevTokens
# otherwise, take the next pattern, which only captures half the token.
else tokenizer_rec len prevTokens (tail patterns) str
else tokenizer_rec len tokens patterns (substring matchLength len str);
avgTokenSize = 100;
ceilLog2 = v:
let inner = n: i: if i < v then inner (n + 1) (i * 2) else n; in
inner 1 1;
# The builtins.match function match the entire string, and generate a list of all captured
# elements. This is the most efficient way to make a tokenizer, if we can make a pattern which
# capture all token of the file. Unfortunately C++ std::regex does not support captures in
# repeated patterns. As a work-around, we generate patterns which are matching tokens in multiple
# of 2, such that we can avoid iterating too many times over the content.
generatePatterns = str:
let
depth = ceilLog2 (stringLength str / avgTokenSize);
inner = depth:
if depth == 0 then [ "(${token_pat})" ]
else
let next = inner (depth - 1); in
[ "${head next}${layout_pat}${head next}" ] ++ next;
in
map (pat: "(${layout_pat_opt}${pat}).*" ) (inner depth);
in
tokenizer_rec (stringLength str) [] (generatePatterns str) str;
tokenizer_1_12 = str:
let
# Nix 1.12 has the builtins.split function which allow to tokenize the
# file quickly. by iterating with a simple regexp.
layoutTokenList = split "(${token_pat})" str;
isLayout = s: match layout_pat_opt s != null;
filterLayout = list:
filter (s:
if isString s then
if isLayout s then false
else throw "Error: Unexpected token: '${s}'"
else true) list;
removeTokenWrapper = list:
map (x: assert tail x == []; head x) list;
in
removeTokenWrapper (filterLayout layoutTokenList);
tokenizer =
if builtins ? split
then tokenizer_1_12
else tokenizer_1_11;
in
# Parse entry headers
let
unescapeString = str:
# Let's ignore any escape character for the moment.
assert match ''"[^"]*"'' str != null; #"
substring 1 (stringLength str - 2) str;
# Match the content of TOML format section names.
ident_pat = ''[a-zA-Z0-9_-]+|"[^"]*"''; #"
removeBraces = token: wrapLen:
substring wrapLen (stringLength token - 2 * wrapLen) token;
# Note, this implementation is limited to 11 identifiers.
matchPathFun_1_11 = token:
let
# match header_pat "a.b.c" == [ "a" ".b" "b" ".c" "c" ]
header_pat =
foldl' (pat: n: "(${ident_pat})([.]${pat})?")
"(${ident_pat})" (genList (n: 0) 10);
matchPath = match header_pat token;
filterDot = filter (s: substring 0 1 s != ".") matchPath;
in
filterDot;
matchPathFun_1_12 = token:
map (e: head e)
(filter (s: isList s)
(split "(${ident_pat})" token));
matchPathFun =
if builtins ? split
then matchPathFun_1_12
else matchPathFun_1_11;
headerToPath = token: wrapLen:
let
token' = removeBraces token wrapLen;
matchPath = matchPathFun token';
path =
map (s:
if substring 0 1 s != ''"'' then s #"
else unescapeString s
) matchPath;
in
assert matchPath != null;
# assert trace "Path: ${token'}; match as ${toString path}" true;
path;
in
# Reconstruct the equivalent attribute set.
let
tokenToValue = token:
if token == "true" then true
else if token == "false" then false
else unescapeString token;
parserInitState = {
idx = 0;
path = [];
isList = false;
output = [];
elem = {};
};
# Imported from nixpkgs library.
setAttrByPath = attrPath: value:
if attrPath == [] then value
else listToAttrs
[ { name = head attrPath; value = setAttrByPath (tail attrPath) value; } ];
closeSection = state:
state // {
output = state.output ++ [ (setAttrByPath state.path (
if state.isList then [ state.elem ]
else state.elem
)) ];
};
readToken = state: token:
# assert trace "Read '${token}'" true;
if state.idx == 0 then
if substring 0 2 token == "[[" then
(closeSection state) // {
path = headerToPath token 2;
isList = true;
elem = {};
}
else if substring 0 1 token == "[" then
(closeSection state) // {
path = headerToPath token 1;
isList = false;
elem = {};
}
else
assert match "[a-zA-Z0-9_-]+" token != null;
state // { idx = 1; name = token; }
else if state.idx == 1 then
assert token == "=";
state // { idx = 2; }
else
assert state.idx == 2;
state // {
idx = 0;
elem = state.elem // {
"${state.name}" = tokenToValue token;
};
};
# aggregate each section as individual attribute sets.
parser = str:
closeSection (foldl' readToken parserInitState (tokenizer str));
fromTOML = toml:
let
sections = (parser toml).output;
# Inlined from nixpkgs library functions.
zipAttrs = sets:
listToAttrs (map (n: {
name = n;
value =
let v = catAttrs n sets; in
# assert trace "Visiting ${n}" true;
if tail v == [] then head v
else if isList (head v) then concatLists v
else if isAttrs (head v) then zipAttrs v
else throw "cannot merge sections";
}) (concatLists (map attrNames sets)));
in
zipAttrs sections;
in
{
testing = fromTOML (builtins.readFile ./channel-rust-nightly.toml);
testing_url = fromTOML (builtins.readFile (builtins.fetchurl
https://static.rust-lang.org/dist/channel-rust-nightly.toml));
inherit fromTOML;
}