From 841c391b989a17068356dbcd5dec330ed433c04d Mon Sep 17 00:00:00 2001 From: Dan Bennett Date: Mon, 14 Jun 2021 21:56:00 -0500 Subject: [PATCH] Basic lexer for Microsoft Power Query M (#462) --- lexers/p/powerquery.go | 38 ++++++++ lexers/testdata/powerquery.actual | 21 ++++ lexers/testdata/powerquery.expected | 146 ++++++++++++++++++++++++++++ 3 files changed, 205 insertions(+) create mode 100644 lexers/p/powerquery.go create mode 100644 lexers/testdata/powerquery.actual create mode 100644 lexers/testdata/powerquery.expected diff --git a/lexers/p/powerquery.go b/lexers/p/powerquery.go new file mode 100644 index 000000000..0302420ea --- /dev/null +++ b/lexers/p/powerquery.go @@ -0,0 +1,38 @@ +package p + +import ( + . "github.com/alecthomas/chroma" // nolint + "github.com/alecthomas/chroma/lexers/internal" +) + +// PowerQuery lexer. +var PowerQuery = internal.Register(MustNewLazyLexer( + &Config{ + Name: "PowerQuery", + Aliases: []string{"powerquery", "pq"}, + Filenames: []string{"*.pq"}, + MimeTypes: []string{"text/x-powerquery"}, + DotAll: true, + CaseInsensitive: true, + }, + powerqueryRules, +)) + +func powerqueryRules() Rules { + return Rules{ + "root": { + {`\s+`, Text, nil}, + {`//.*?\n`, CommentSingle, nil}, + {`/\*.*?\*/`, CommentMultiline, nil}, + {`"(\\\\|\\"|[^"])*"`, LiteralString, nil}, + {`(and|as|each|else|error|false|if|in|is|let|meta|not|null|or|otherwise|section|shared|then|true|try|type)\b`, Keyword, nil}, + {`(#binary|#date|#datetime|#datetimezone|#duration|#infinity|#nan|#sections|#shared|#table|#time)\b`, KeywordType, nil}, + {`(([a-zA-Z]|_)[\w|._]*|#"[^"]+")`, Name, nil}, + {`0[xX][0-9a-fA-F][0-9a-fA-F_]*[lL]?`, LiteralNumberHex, nil}, + {`([0-9]+\.[0-9]+|\.[0-9]+)([eE][0-9]+)?`, LiteralNumberFloat, nil}, + {`[0-9]+`, LiteralNumberInteger, nil}, + {`[\(\)\[\]\{\}]`, Punctuation, nil}, + {`\.\.|\.\.\.|=>|<=|>=|<>|[@!?,;=<>\+\-\*\/&]`, Operator, nil}, + }, + } +} diff --git a/lexers/testdata/powerquery.actual b/lexers/testdata/powerquery.actual new file mode 100644 index 000000000..d33810621 --- /dev/null +++ b/lexers/testdata/powerquery.actual @@ -0,0 +1,21 @@ +let + GetLabel = (subject as text) as nullable table => + let + // Request Ntriples from the LDF Service + Options = [Headers = [#"Accept"="text/csv"]], + FixedSubject = if Text.At(subject,0) <> "<" then Text.Combine({"<",subject,">"}) else subject, + Query = Text.Combine({"SELECT ?label WHERE { + SERVICE wikibase:label { + bd:serviceParam wikibase:language ""en"" . + ",FixedSubject," ?label + } + }"}), + /* this is a multiline comment, see https://docs.microsoft.com/en-us/powerquery-m/comments + */ + url = Text.Combine({"https://query.wikidata.org/sparql?",Uri.BuildQueryString([query=Query])}), + data = Csv.Document(Web.Contents(url,Options)), + promoted = Table.PromoteHeaders(data, [PromoteAllScalars=true]) + in + promoted +in + GetLabel \ No newline at end of file diff --git a/lexers/testdata/powerquery.expected b/lexers/testdata/powerquery.expected new file mode 100644 index 000000000..ee2aec8d9 --- /dev/null +++ b/lexers/testdata/powerquery.expected @@ -0,0 +1,146 @@ +[ + {"type":"Keyword","value":"let"}, + {"type":"Text","value":"\n "}, + {"type":"Name","value":"GetLabel"}, + {"type":"Text","value":" "}, + {"type":"Operator","value":"="}, + {"type":"Text","value":" "}, + {"type":"Punctuation","value":"("}, + {"type":"Name","value":"subject"}, + {"type":"Text","value":" "}, + {"type":"Keyword","value":"as"}, + {"type":"Text","value":" "}, + {"type":"Name","value":"text"}, + {"type":"Punctuation","value":")"}, + {"type":"Text","value":" "}, + {"type":"Keyword","value":"as"}, + {"type":"Text","value":" "}, + {"type":"Name","value":"nullable"}, + {"type":"Text","value":" "}, + {"type":"Name","value":"table"}, + {"type":"Text","value":" "}, + {"type":"Operator","value":"=\u003e"}, + {"type":"Text","value":"\n "}, + {"type":"Keyword","value":"let"}, + {"type":"Text","value":" \n "}, + {"type":"CommentSingle","value":"// Request Ntriples from the LDF Service\n"}, + {"type":"Text","value":" "}, + {"type":"Name","value":"Options"}, + {"type":"Text","value":" "}, + {"type":"Operator","value":"="}, + {"type":"Text","value":" "}, + {"type":"Punctuation","value":"["}, + {"type":"Name","value":"Headers"}, + {"type":"Text","value":" "}, + {"type":"Operator","value":"="}, + {"type":"Text","value":" "}, + {"type":"Punctuation","value":"["}, + {"type":"Name","value":"#\"Accept\""}, + {"type":"Operator","value":"="}, + {"type":"LiteralString","value":"\"text/csv\""}, + {"type":"Punctuation","value":"]]"}, + {"type":"Operator","value":","}, + {"type":"Text","value":"\n "}, + {"type":"Name","value":"FixedSubject"}, + {"type":"Text","value":" "}, + {"type":"Operator","value":"="}, + {"type":"Text","value":" "}, + {"type":"Keyword","value":"if"}, + {"type":"Text","value":" "}, + {"type":"Name","value":"Text.At"}, + {"type":"Punctuation","value":"("}, + {"type":"Name","value":"subject"}, + {"type":"Operator","value":","}, + {"type":"LiteralNumberInteger","value":"0"}, + {"type":"Punctuation","value":")"}, + {"type":"Text","value":" "}, + {"type":"Operator","value":"\u003c\u003e"}, + {"type":"Text","value":" "}, + {"type":"LiteralString","value":"\"\u003c\""}, + {"type":"Text","value":" "}, + {"type":"Keyword","value":"then"}, + {"type":"Text","value":" "}, + {"type":"Name","value":"Text.Combine"}, + {"type":"Punctuation","value":"({"}, + {"type":"LiteralString","value":"\"\u003c\""}, + {"type":"Operator","value":","}, + {"type":"Name","value":"subject"}, + {"type":"Operator","value":","}, + {"type":"LiteralString","value":"\"\u003e\""}, + {"type":"Punctuation","value":"})"}, + {"type":"Text","value":" "}, + {"type":"Keyword","value":"else"}, + {"type":"Text","value":" "}, + {"type":"Name","value":"subject"}, + {"type":"Operator","value":","}, + {"type":"Text","value":"\n "}, + {"type":"Name","value":"Query"}, + {"type":"Text","value":" "}, + {"type":"Operator","value":"="}, + {"type":"Text","value":" "}, + {"type":"Name","value":"Text.Combine"}, + {"type":"Punctuation","value":"({"}, + {"type":"LiteralString","value":"\"SELECT ?label WHERE {\n SERVICE wikibase:label {\n bd:serviceParam wikibase:language \"\"en\"\" .\n \""}, + {"type":"Operator","value":","}, + {"type":"Name","value":"FixedSubject"}, + {"type":"Operator","value":","}, + {"type":"LiteralString","value":"\" \u003chttp://www.w3.org/2000/01/rdf-schema#label\u003e ?label\n }\n }\""}, + {"type":"Punctuation","value":"})"}, + {"type":"Operator","value":","}, + {"type":"Text","value":"\n "}, + {"type":"CommentMultiline","value":"/* this is a multiline comment, see https://docs.microsoft.com/en-us/powerquery-m/comments\n */"}, + {"type":"Text","value":"\n "}, + {"type":"Name","value":"url"}, + {"type":"Text","value":" "}, + {"type":"Operator","value":"="}, + {"type":"Text","value":" "}, + {"type":"Name","value":"Text.Combine"}, + {"type":"Punctuation","value":"({"}, + {"type":"LiteralString","value":"\"https://query.wikidata.org/sparql?\""}, + {"type":"Operator","value":","}, + {"type":"Name","value":"Uri.BuildQueryString"}, + {"type":"Punctuation","value":"(["}, + {"type":"Name","value":"query"}, + {"type":"Operator","value":"="}, + {"type":"Name","value":"Query"}, + {"type":"Punctuation","value":"])})"}, + {"type":"Operator","value":","}, + {"type":"Text","value":"\n "}, + {"type":"Name","value":"data"}, + {"type":"Text","value":" "}, + {"type":"Operator","value":"="}, + {"type":"Text","value":" "}, + {"type":"Name","value":"Csv.Document"}, + {"type":"Punctuation","value":"("}, + {"type":"Name","value":"Web.Contents"}, + {"type":"Punctuation","value":"("}, + {"type":"Name","value":"url"}, + {"type":"Operator","value":","}, + {"type":"Name","value":"Options"}, + {"type":"Punctuation","value":"))"}, + {"type":"Operator","value":","}, + {"type":"Text","value":"\n "}, + {"type":"Name","value":"promoted"}, + {"type":"Text","value":" "}, + {"type":"Operator","value":"="}, + {"type":"Text","value":" "}, + {"type":"Name","value":"Table.PromoteHeaders"}, + {"type":"Punctuation","value":"("}, + {"type":"Name","value":"data"}, + {"type":"Operator","value":","}, + {"type":"Text","value":" "}, + {"type":"Punctuation","value":"["}, + {"type":"Name","value":"PromoteAllScalars"}, + {"type":"Operator","value":"="}, + {"type":"Keyword","value":"true"}, + {"type":"Punctuation","value":"])"}, + {"type":"Text","value":"\n "}, + {"type":"Keyword","value":"in"}, + {"type":"Text","value":" \n "}, + {"type":"Name","value":"promoted"}, + {"type":"Text","value":" \n"}, + {"type":"Keyword","value":"in"}, + {"type":"Text","value":"\n "}, + {"type":"Name","value":"GetLabel"}, + {"type":"Text","value":" "} +]