From f1894de8d4f38c9838be044baac4e3d43e224da3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aslak=20Helles=C3=B8y?= Date: Tue, 28 Jul 2009 01:00:11 +0200 Subject: [PATCH 1/2] Added Rake script that generates lexer from cucumber's languages. Requires at least cucumber 039eaba (where texan was escaped). Now the gherkin lexer can be used with Jekyll. There are still some issues. Steps starting with 'Given I ' are not correctly lexed. 'Given I ' becomes one token, but only 'Given ' ought to. --- README.textile | 1 + Rakefile | 41 ++++++++++++++++++++ gherkin_lexer/__init__.py | 4 +- lexer.erb.py | 81 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 125 insertions(+), 2 deletions(-) create mode 100644 Rakefile create mode 100644 lexer.erb.py diff --git a/README.textile b/README.textile index da9816f..702ddb2 100644 --- a/README.textile +++ b/README.textile @@ -6,6 +6,7 @@ For an idea of what it looks like "click here.":http://www.benmabey.com/cucumber h2. Installing +rake generate sudo python setup.py install h2. Making Perty Cucumber features diff --git a/Rakefile b/Rakefile new file mode 100644 index 0000000..9f1e3dc --- /dev/null +++ b/Rakefile @@ -0,0 +1,41 @@ +class SyntaxGenerator + def generate + require 'yaml' + require 'erb' + require 'cucumber' + + scenario_keywords_array = [] + step_keywords_array = [] + + Cucumber::LANGUAGES.each do |lang, words| + scenario_keywords_array << words.delete('scenario').split(/\|/) rescue nil + scenario_keywords_array << words.delete('scenario_outline').split(/\|/) rescue nil + scenario_keywords_array << words.delete('background').split(/\|/) rescue nil + + step_keywords_array << keywords(lang, words) + end + + scenario_keywords = scenario_keywords_array.flatten.compact.sort.reverse.uniq.join('|') + step_keywords = step_keywords_array.flatten.compact.sort.reverse.uniq.join('|') + + template = ERB.new(IO.read(File.dirname(__FILE__) + '/lexer.erb.py')) + syntax = template.result(binding) + + syntax_file = File.dirname(__FILE__) + '/gherkin_lexer/__init__.py' + File.open(syntax_file, "w") do |io| + io.write(syntax) + end + end + + def keywords(lang, words) + space = words['space_after_keyword'] ? '\s+' : '' + %w{given when then and but}.map do |key| + words[key].split(/\|/).map{|w| "#{w}#{space}"} + end + end +end + +desc 'Generate Gherkin lexer for all languages supported by Cucumber' +task :generate do + SyntaxGenerator.new.generate +end diff --git a/gherkin_lexer/__init__.py b/gherkin_lexer/__init__.py index b552d6a..b7c8c02 100755 --- a/gherkin_lexer/__init__.py +++ b/gherkin_lexer/__init__.py @@ -8,8 +8,8 @@ class GherkinLexer(RegexLexer): aliases = ['Cucumber', 'cucumber', 'Gherkin', 'gherkin'] filenames = ['*.feature', '*.story'] - step_keywords_regexp = r'(Given|When|Then|And|But)' - scenario_sections_regexp = r'(\s+)(Background|Scenario|Scenario Outline)(:)(.*)$' + step_keywords_regexp = r'(하지만|조건|만일|그리고|그러면|那麼|那么|而且|當|当|前提|假設|假如|但是|但し|並且|もし|ならば|しかし|かつ|و\s+|متى\s+|لكن\s+|عندما\s+|ثم\s+|بفرض\s+|اذاً\s+|כאשר\s+|וגם\s+|בהינתן\s+|אזי\s+|אז\s+|אבל\s+|То\s+|Но\s+|Когато\s+|К тому же\s+|И\s+|Если\s+|Допустим\s+|Дадено\s+|А\s+|Și\s+|És\s+|anrhegedig a\s+|Zakładając\s+|Yna\s+|Ya know how\s+|Ya gotta\s+|Y\s+|Wtedy\s+|When\s+|When y\'all\s+|Wenn\s+|WEN\s+|Và\s+|Und\s+|Un\s+|Thì\s+|Then\s+|Then y\'all\s+|Tapi\s+|Tak\s+|Tada\s+|Tad\s+|Så\s+|Soit\s+|Siis\s+|Si\s+|Quando\s+|Quan\s+|Pryd\s+|Pokud\s+|Pokiaľ\s+|Però\s+|Pero\s+|Pak\s+|Oraz\s+|Ond\s+|Oletetaan\s+|Og\s+|Och\s+|Når\s+|När\s+|Niin\s+|Nhưng\s+|N\s+|Mutta\s+|Men\s+|Mas\s+|Maka\s+|Majd\s+|Mais\s+|Maar\s+|Ma\s+|Lorsque\s+|Kun\s+|Kuid\s+|Kui\s+|Khi\s+|Keď\s+|Ketika\s+|Když\s+|Kai\s+|Kad\s+|Jeżeli\s+|Ja\s+|Ir\s+|I\s+|I CAN HAZ\s+|Ha\s+|Givet\s+|Given\s+|Given y\'all\s+|Gitt\s+|Gegeven\s+|Gegeben sei\s+|Et\s+|Então\s+|Entonces\s+|Entao\s+|En\s+|Eeldades\s+|E\s+|Duota\s+|Donat\s+|Donada\s+|Dengan\s+|De\s+|Dato\s+|Dar\s+|Dann\s+|Dan\s+|Dado\s+|Dacă\s+|Daca\s+|DEN\s+|Când\s+|Cuando\s+|Cho\s+|Cept\s+|Cand\s+|But\s+|But y\'all\s+|Biết\s+|Bet\s+|BUT\s+|Atunci\s+|And\s+|And y\'all\s+|Als\s+|Alors\s+|Allora\s+|Aleshores\s+|Ale\s+|Akkor\s+|Aber\s+|A\s+|AN\s+|A také\s+)' + scenario_sections_regexp = r'(\s+)(시나리오 개요|시나리오|배경|背景|場景大綱|場景|场景大纲|场景|劇本大綱|劇本|テンプレ|シナリオテンプレート|シナリオテンプレ|シナリオアウトライン|シナリオ|سيناريو مخطط|سيناريو|الخلفية|תרחיש|תבנית תרחיש|רקע|Сценарий|Структура сценария|Рамка на сценарий|Предыстория|Предистория|Założenia|Tình huống|Tausta|Taust|Tapausaihio|Tapaus|Szenariogrundriss|Szenario|Szablon scenariusza|Stsenaarium|Skenario konsep|Skenario|Situācija|Scénář|Scénario|Schema dello scenario|Scenārijs pēc parauga|Scenārijs|Scenár|Scenariusz|Scenariu|Scenario Outline|Scenario|Scenarijus|Scenarijaus šablonas|Scenarie|Rerefons|Raamstsenaarium|Pozadí|Pozadie|Plan du Scénario|Osnova scénáře|Náčrt Scénáře|Náčrt Scenáru|Mate|MISHUN SRSLY|MISHUN|Kịch bản|Kontext|Konteksts|Kontekstas|Khung tình huống|Khung kịch bản|Háttér|Grundlage|Forgatókönyv vázlat|Forgatókönyv|Esquema do Cenário|Esquema do Cenario|Esquema del escenario|Esquema de l\'escenari|Escenario|Escenari|Dasar|Contexto|Contexte|Contesto|Cenário|Cenario|Bối cảnh|Blokes|Bakgrunn|Bakgrund|Baggrund|Background|B4|Antecedents|Antecedentes|All y\'all|Achtergrond|Abstrakt Scenario|Abstract Scenario)(:)(.*)$' tokens = { 'comments': [ diff --git a/lexer.erb.py b/lexer.erb.py new file mode 100644 index 0000000..619cc2b --- /dev/null +++ b/lexer.erb.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python +from pygments.lexer import RegexLexer, bygroups, include +from pygments.token import \ + Text, Comment, Literal, Operator, Keyword, Name, String + +class GherkinLexer(RegexLexer): + name = 'Gherkin' + aliases = ['Cucumber', 'cucumber', 'Gherkin', 'gherkin'] + filenames = ['*.feature', '*.story'] + + step_keywords_regexp = r'(<%= step_keywords %>)' + scenario_sections_regexp = r'(\s+)(<%= scenario_keywords %>)(:)(.*)$' + + tokens = { + 'comments': [ + (r'#.*$', Comment), + ], + 'multiline_descriptions' : [ + (step_keywords_regexp, Keyword, "#pop"), + include('comments'), + (r"(\s|.)", Name.Constant), + ], + 'multiline_descriptions_on_stack' : [ + (step_keywords_regexp, Keyword, "#pop:2"), + include('comments'), + (r"(\s|.)", Name.Constant), + ], + 'scenario_table_description': [ + (r"\s+\|", Text, 'scenario_table_header'), + include('comments'), + (r"(\s|.)", Name.Constant), + ], + 'scenario_table_header': [ + (r"\s+\|\s*$", Text, "#pop:2"), + (r"(\s+\|\s*)(#.*)$", bygroups(Text, Comment), "#pop:2"), + include('comments'), + (r"\s+\|", Text), + (r"[^\|]", Literal.String.Symbol), + ], + 'scenario_sections_on_stack': [ + (scenario_sections_regexp, bygroups(Text, Name.Class, Name.Class, Name.Constant), "multiline_descriptions_on_stack"), + ], + 'narrative': [ + include('scenario_sections_on_stack'), + (r"(\s|.)", Name.Builtin), + ], + 'table_vars': [ + (r'(<)([^>]*)(>)', bygroups(Operator, Literal.String.Symbol, Operator)), + ], + 'string': [ + include('table_vars'), + (r'(\s|.)', String), + ], + 'py_string': [ + (r'"""', String, "#pop"), + include('string'), + ], + 'double_string': [ + (r'"', String, "#pop"), + include('string'), + ], + 'single_string': [ + (r"'", String, "#pop"), + include('string'), + ], + 'root': [ + (r'\n', Text), + include('comments'), + (r'"""', String, "py_string"), + (r'"', String, "double_string"), + (r"'", String, "single_string"), + include('table_vars'), + (r'@[^@\s]+', Name.Namespace), + (step_keywords_regexp, Keyword), + (r'^(Feature|Story)(:)(.*)$', bygroups(Name.Class, Name.Class, Name.Constant), 'narrative'), + (scenario_sections_regexp, bygroups(Text, Name.Class, Name.Class, Name.Constant), "multiline_descriptions"), + (r'(\s+)(Scenarios|Examples)(:)(.*)$', bygroups(Text, Name.Class, Name.Class, Name.Constant), "scenario_table_description"), + (r'(\s|.)', Text), + ] + + } From c2bf0890fc393c8ab762e912298bf92b47d6c24e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aslak=20Helles=C3=B8y?= Date: Thu, 30 Jul 2009 00:38:43 +0200 Subject: [PATCH 2/2] Better handling of spaces --- gherkin_lexer/__init__.py | 4 ++-- lexer.erb.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/gherkin_lexer/__init__.py b/gherkin_lexer/__init__.py index b7c8c02..1ce6286 100755 --- a/gherkin_lexer/__init__.py +++ b/gherkin_lexer/__init__.py @@ -8,7 +8,7 @@ class GherkinLexer(RegexLexer): aliases = ['Cucumber', 'cucumber', 'Gherkin', 'gherkin'] filenames = ['*.feature', '*.story'] - step_keywords_regexp = r'(하지만|조건|만일|그리고|그러면|那麼|那么|而且|當|当|前提|假設|假如|但是|但し|並且|もし|ならば|しかし|かつ|و\s+|متى\s+|لكن\s+|عندما\s+|ثم\s+|بفرض\s+|اذاً\s+|כאשר\s+|וגם\s+|בהינתן\s+|אזי\s+|אז\s+|אבל\s+|То\s+|Но\s+|Когато\s+|К тому же\s+|И\s+|Если\s+|Допустим\s+|Дадено\s+|А\s+|Și\s+|És\s+|anrhegedig a\s+|Zakładając\s+|Yna\s+|Ya know how\s+|Ya gotta\s+|Y\s+|Wtedy\s+|When\s+|When y\'all\s+|Wenn\s+|WEN\s+|Và\s+|Und\s+|Un\s+|Thì\s+|Then\s+|Then y\'all\s+|Tapi\s+|Tak\s+|Tada\s+|Tad\s+|Så\s+|Soit\s+|Siis\s+|Si\s+|Quando\s+|Quan\s+|Pryd\s+|Pokud\s+|Pokiaľ\s+|Però\s+|Pero\s+|Pak\s+|Oraz\s+|Ond\s+|Oletetaan\s+|Og\s+|Och\s+|Når\s+|När\s+|Niin\s+|Nhưng\s+|N\s+|Mutta\s+|Men\s+|Mas\s+|Maka\s+|Majd\s+|Mais\s+|Maar\s+|Ma\s+|Lorsque\s+|Kun\s+|Kuid\s+|Kui\s+|Khi\s+|Keď\s+|Ketika\s+|Když\s+|Kai\s+|Kad\s+|Jeżeli\s+|Ja\s+|Ir\s+|I\s+|I CAN HAZ\s+|Ha\s+|Givet\s+|Given\s+|Given y\'all\s+|Gitt\s+|Gegeven\s+|Gegeben sei\s+|Et\s+|Então\s+|Entonces\s+|Entao\s+|En\s+|Eeldades\s+|E\s+|Duota\s+|Donat\s+|Donada\s+|Dengan\s+|De\s+|Dato\s+|Dar\s+|Dann\s+|Dan\s+|Dado\s+|Dacă\s+|Daca\s+|DEN\s+|Când\s+|Cuando\s+|Cho\s+|Cept\s+|Cand\s+|But\s+|But y\'all\s+|Biết\s+|Bet\s+|BUT\s+|Atunci\s+|And\s+|And y\'all\s+|Als\s+|Alors\s+|Allora\s+|Aleshores\s+|Ale\s+|Akkor\s+|Aber\s+|A\s+|AN\s+|A také\s+)' + step_keywords_regexp = r'(\s+)(하지만|조건|만일|그리고|그러면|那麼|那么|而且|當|当|前提|假設|假如|但是|但し|並且|もし|ならば|しかし|かつ|و\s+|متى\s+|لكن\s+|عندما\s+|ثم\s+|بفرض\s+|اذاً\s+|כאשר\s+|וגם\s+|בהינתן\s+|אזי\s+|אז\s+|אבל\s+|То\s+|Но\s+|Когато\s+|К тому же\s+|И\s+|Если\s+|Допустим\s+|Дадено\s+|А\s+|Și\s+|És\s+|anrhegedig a\s+|Zakładając\s+|Yna\s+|Ya know how\s+|Ya gotta\s+|Y\s+|Wtedy\s+|When\s+|When y\'all\s+|Wenn\s+|WEN\s+|Và\s+|Und\s+|Un\s+|Thì\s+|Then\s+|Then y\'all\s+|Tapi\s+|Tak\s+|Tada\s+|Tad\s+|Så\s+|Soit\s+|Siis\s+|Si\s+|Quando\s+|Quan\s+|Pryd\s+|Pokud\s+|Pokiaľ\s+|Però\s+|Pero\s+|Pak\s+|Oraz\s+|Ond\s+|Oletetaan\s+|Og\s+|Och\s+|Når\s+|När\s+|Niin\s+|Nhưng\s+|N\s+|Mutta\s+|Men\s+|Mas\s+|Maka\s+|Majd\s+|Mais\s+|Maar\s+|Ma\s+|Lorsque\s+|Kun\s+|Kuid\s+|Kui\s+|Khi\s+|Keď\s+|Ketika\s+|Když\s+|Kai\s+|Kad\s+|Jeżeli\s+|Ja\s+|Ir\s+|I\s+|I CAN HAZ\s+|Ha\s+|Givet\s+|Given\s+|Given y\'all\s+|Gitt\s+|Gegeven\s+|Gegeben sei\s+|Et\s+|Então\s+|Entonces\s+|Entao\s+|En\s+|Eeldades\s+|E\s+|Duota\s+|Donat\s+|Donada\s+|Dengan\s+|De\s+|Dato\s+|Dar\s+|Dann\s+|Dan\s+|Dado\s+|Dacă\s+|Daca\s+|DEN\s+|Când\s+|Cuando\s+|Cho\s+|Cept\s+|Cand\s+|But\s+|But y\'all\s+|Biết\s+|Bet\s+|BUT\s+|Atunci\s+|And\s+|And y\'all\s+|Als\s+|Alors\s+|Allora\s+|Aleshores\s+|Ale\s+|Akkor\s+|Aber\s+|A\s+|AN\s+|A také\s+)' scenario_sections_regexp = r'(\s+)(시나리오 개요|시나리오|배경|背景|場景大綱|場景|场景大纲|场景|劇本大綱|劇本|テンプレ|シナリオテンプレート|シナリオテンプレ|シナリオアウトライン|シナリオ|سيناريو مخطط|سيناريو|الخلفية|תרחיש|תבנית תרחיש|רקע|Сценарий|Структура сценария|Рамка на сценарий|Предыстория|Предистория|Założenia|Tình huống|Tausta|Taust|Tapausaihio|Tapaus|Szenariogrundriss|Szenario|Szablon scenariusza|Stsenaarium|Skenario konsep|Skenario|Situācija|Scénář|Scénario|Schema dello scenario|Scenārijs pēc parauga|Scenārijs|Scenár|Scenariusz|Scenariu|Scenario Outline|Scenario|Scenarijus|Scenarijaus šablonas|Scenarie|Rerefons|Raamstsenaarium|Pozadí|Pozadie|Plan du Scénario|Osnova scénáře|Náčrt Scénáře|Náčrt Scenáru|Mate|MISHUN SRSLY|MISHUN|Kịch bản|Kontext|Konteksts|Kontekstas|Khung tình huống|Khung kịch bản|Háttér|Grundlage|Forgatókönyv vázlat|Forgatókönyv|Esquema do Cenário|Esquema do Cenario|Esquema del escenario|Esquema de l\'escenari|Escenario|Escenari|Dasar|Contexto|Contexte|Contesto|Cenário|Cenario|Bối cảnh|Blokes|Bakgrunn|Bakgrund|Baggrund|Background|B4|Antecedents|Antecedentes|All y\'all|Achtergrond|Abstrakt Scenario|Abstract Scenario)(:)(.*)$' tokens = { @@ -71,7 +71,7 @@ class GherkinLexer(RegexLexer): (r"'", String, "single_string"), include('table_vars'), (r'@[^@\s]+', Name.Namespace), - (step_keywords_regexp, Keyword), + (step_keywords_regexp, bygroups(Text, Keyword)), (r'^(Feature|Story)(:)(.*)$', bygroups(Name.Class, Name.Class, Name.Constant), 'narrative'), (scenario_sections_regexp, bygroups(Text, Name.Class, Name.Class, Name.Constant), "multiline_descriptions"), (r'(\s+)(Scenarios|Examples)(:)(.*)$', bygroups(Text, Name.Class, Name.Class, Name.Constant), "scenario_table_description"), diff --git a/lexer.erb.py b/lexer.erb.py index 619cc2b..4adf811 100644 --- a/lexer.erb.py +++ b/lexer.erb.py @@ -8,7 +8,7 @@ class GherkinLexer(RegexLexer): aliases = ['Cucumber', 'cucumber', 'Gherkin', 'gherkin'] filenames = ['*.feature', '*.story'] - step_keywords_regexp = r'(<%= step_keywords %>)' + step_keywords_regexp = r'(\s+)(<%= step_keywords %>)' scenario_sections_regexp = r'(\s+)(<%= scenario_keywords %>)(:)(.*)$' tokens = { @@ -71,7 +71,7 @@ class GherkinLexer(RegexLexer): (r"'", String, "single_string"), include('table_vars'), (r'@[^@\s]+', Name.Namespace), - (step_keywords_regexp, Keyword), + (step_keywords_regexp, bygroups(Text, Keyword)), (r'^(Feature|Story)(:)(.*)$', bygroups(Name.Class, Name.Class, Name.Constant), 'narrative'), (scenario_sections_regexp, bygroups(Text, Name.Class, Name.Class, Name.Constant), "multiline_descriptions"), (r'(\s+)(Scenarios|Examples)(:)(.*)$', bygroups(Text, Name.Class, Name.Class, Name.Constant), "scenario_table_description"),