20 changes: 12 additions & 8 deletions TeXmacs/progs/prog/json-lang.scm
View file
Expand Up @@ -13,24 +13,28 @@

(texmacs-module (prog json-lang))

(tm-define (json-keywords)
`(keywords
(tm-define (parser-feature lan key)
(:require (and (== lan "json") (== key "keyword")))
`(,(string->symbol key)
(constant
"false" "true" "null")))

;; Ref: https://ecma-international.org/ecma-262/10.0/index.html#sec-update-expressions
(tm-define (json-operators)
`(operators
(tm-define (parser-feature lan key)
(:require (and (== lan "json") (== key "operator")))
`(,(string->symbol key)
(operator "+" "-" ":" ",")
(operator_openclose "{" "[" "(" ")" "]" "}")))

;; Ref: https://ecma-international.org/ecma-262/10.0/index.html#sec-literals-numeric-literals
(tm-define (json-numbers)
`(numbers
(tm-define (parser-feature lan key)
(:require (and (== lan "javascript") (== key "number")))
`(,(string->symbol key)
(bool_features
"sci_notation")))

(tm-define (json-string)
`(string
(tm-define (parser-feature lan key)
(:require (and (== lan "json") (== key "string")))
`(,(string->symbol key)
(bool_features )
(escape_sequences "\\" "/" "\"" "b" "f" "n" "r" "t")))
31 changes: 19 additions & 12 deletions TeXmacs/progs/prog/octave-lang.scm
View file
Expand Up @@ -11,10 +11,12 @@
;;
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

(texmacs-module (prog octave-lang))
(texmacs-module (prog octave-lang)
(:use (prog default-lang)))

(tm-define (octave-keywords)
`(keywords
(tm-define (parser-feature lan key)
(:require (and (== lan "octave") (== key "keyword")))
`(,(string->symbol key)
(constant
"false" "true")
(declare_type "function" "endfunction" "class")
Expand All @@ -25,8 +27,9 @@
(keyword_control
"catch" "try")))

(tm-define (octave-operators)
`(operators
(tm-define (parser-feature lan key)
(:require (and (== lan "octave") (== key "operator")))
`(,(string->symbol key)
(operator "," ";" ":" "=")
(operator_special "@")
(operator_field ".")
Expand All @@ -36,20 +39,24 @@
`(suffix
(imaginary "j" "J")))

(tm-define (octave-numbers)
`(numbers
(tm-define (parser-feature lan key)
(:require (and (== lan "octave") (== key "number")))
`(,(string->symbol key)
(bool_features
"prefix_0x" "prefix_0b"
"sci_notation")
(separator "_")
,(octave-number-suffix)))

(tm-define (octave-inline-comment-starts)
(list "#"))

(tm-define (octave-string)
`(string
(tm-define (parser-feature lan key)
(:require (and (== lan "octave") (== key "string")))
`(,(string->symbol key)
(bool_features
"hex_with_8_bits" "hex_with_16_bits"
"hex_with_32_bits" "octal_upto_3_digits")
(escape_sequences "\\" "\"" "'" "a" "b" "f" "n" "r" "t" "v")))

(tm-define (parser-feature lan key)
(:require (and (== lan "octave") (== key "comment")))
`(,(string->symbol key)
(inline "#")))
31 changes: 19 additions & 12 deletions TeXmacs/progs/prog/python-lang.scm
View file
Expand Up @@ -11,10 +11,12 @@
;;
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

(texmacs-module (prog python-lang))
(texmacs-module (prog python-lang)
(:use (prog default-lang)))

(tm-define (python-keywords)
`(keywords
(tm-define (parser-feature lan key)
(:require (and (== lan "python") (== key "keyword")))
`(,(string->symbol key)
(constant
"Ellipsis" "False" "None" "NotImplemented" "True" "__debug__" "__import__" "abs"
"all" "any" "apply" "ascii" "basestring" "bin" "bool" "buffer"
Expand Down Expand Up @@ -48,8 +50,9 @@
"assert" "except" "exec" "finally" "pass" "print" "raise" "return"
"try" "yield")))

(tm-define (python-operators)
`(operators
(tm-define (parser-feature lan key)
(:require (and (== lan "python") (== key "operator")))
`(,(string->symbol key)
(operator
"and" "not" "or"
"+" "-" "/" "*" "**" "//" "%" "|" "&" "^"
Expand All @@ -67,24 +70,28 @@
(imaginary "j" "J")))

;; https://docs.python.org/3.8/reference/lexical_analysis.html#numeric-literals
(tm-define (python-numbers)
`(numbers
(tm-define (parser-feature lan key)
(:require (and (== lan "python") (== key "number")))
`(,(string->symbol key)
(bool_features
"prefix_0x" "prefix_0b" "prefix_0o" "no_suffix_with_box"
"sci_notation")
,(python-number-suffix)
(separator "_")))

(tm-define (python-inline-comment-starts)
(list "#"))

(tm-define (python-string)
`(string
(tm-define (parser-feature lan key)
(:require (and (== lan "python") (== key "string")))
`(,(string->symbol key)
(bool_features
"hex_with_8_bits" "hex_with_16_bits"
"hex_with_32_bits" "octal_upto_3_digits")
(escape_sequences "\\" "\"" "'" "a" "b" "f" "n" "r" "t" "v" "newline")))

(tm-define (parser-feature lan key)
(:require (and (== lan "python") (== key "comment")))
`(,(string->symbol key)
(inline "#")))

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Preferences for syntax highlighting
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
Expand Down
26 changes: 14 additions & 12 deletions TeXmacs/progs/prog/scala-lang.scm
View file
Expand Up @@ -11,11 +11,13 @@
;;
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

(texmacs-module (prog scala-lang))
(texmacs-module (prog scala-lang)
(:use (prog default-lang)))

;; https://www.scala-lang.org/files/archive/spec/2.13/13-syntax-summary.html
(tm-define (scala-keywords)
`(keywords
(tm-define (parser-feature lan key)
(:require (and (== lan "scala") (== key "keyword")))
`(,(string->symbol key)
(constant
"false" "true" "null"
"Byte" "Short" "Int" "Long" "Char" "String" "Float" "Double" "Boolean"
Expand All @@ -40,8 +42,9 @@
(keyword_control
"throw" "catch" "finally" "return" "try" "yield")))

(tm-define (scala-operators)
`(operators
(tm-define (parser-feature lan key)
(:require (and (== lan "scala") (== key "operator")))
`(,(string->symbol key)
(operator
"+" "-" "/" "*" "%" ;; Arith
"|" "&" "^" ;; Bit
Expand All @@ -62,18 +65,17 @@
(double "d" "D")
(float "f" "F")))

(tm-define (scala-numbers)
`(numbers
(tm-define (parser-feature lan key)
(:require (and (== lan "scala") (== key "number")))
`(,(string->symbol key)
(bool_features
"prefix_0x" "prefix_0b"
"sci_notation")
,(scala-number-suffix)))

(tm-define (scala-inline-comment-starts)
(list "//"))

(tm-define (scala-string)
`(string
(tm-define (parser-feature lan key)
(:require (and (== lan "scala") (== key "string")))
`(,(string->symbol key)
(bool_features
"hex_with_8_bits" "hex_with_16_bits"
"hex_with_32_bits" "octal_upto_3_digits")
Expand Down
4 changes: 3 additions & 1 deletion src/Data/Parser/inline_comment_parser.cpp
View file
Expand Up @@ -17,13 +17,15 @@ inline_comment_parser_rep::inline_comment_parser_rep () {
}

void
inline_comment_parser_rep::set_starts(const array<string>& p_starts) {
inline_comment_parser_rep::set_starts (const array<string>& p_starts) {
m_starts= p_starts;
}

bool
inline_comment_parser_rep::can_parse (string s, int pos) {
if (pos >= N(s)) return false;
if (N(m_starts) == 0) return false;

int i=0;
while (i<N(m_starts)) {
string m_start= m_starts[i];
Expand Down
8 changes: 4 additions & 4 deletions src/Data/Parser/preprocessor_parser.cpp
View file
Expand Up @@ -34,8 +34,11 @@ preprocessor_parser_rep::set_directives (array<string> directives) {
bool
preprocessor_parser_rep::can_parse (string s, int pos) {
if (!parser_rep::can_parse (s, pos)) return false;

// A language with empty directives does not have preprocessors
if (N(m_directives) == 0) return false;

if (s[pos] != '#') return false;
if (s[pos] != m_start) return false;

int first_non_blank_pos= 0;
skip_spaces (s, first_non_blank_pos);
Expand All @@ -44,9 +47,6 @@ preprocessor_parser_rep::can_parse (string s, int pos) {

void
preprocessor_parser_rep::do_parse (string s, int& pos) {
// A language with empty directives does not have preprocessors
if (N(m_directives) == 0) return;

int opos= pos;

if (s[pos] != m_start) return;
Expand Down
2 changes: 2 additions & 0 deletions src/System/Language/impl_language.hpp
View file
Expand Up @@ -87,6 +87,8 @@ struct prog_language_rep: abstract_language_rep {
void customize_number (tree config);
void customize_string (tree config);
void customize_preprocessor (tree config);
void customize_comment (tree config);
tree get_parser_config (string lan, string key);
};

struct scheme_language_rep: language_rep {
Expand Down
68 changes: 37 additions & 31 deletions src/System/Language/prog_language.cpp
View file
Expand Up @@ -25,39 +25,29 @@ prog_language_rep::prog_language_rep (string name):
string use_modules= "(use-modules (prog " * name * "-lang))";
eval (use_modules);

// Load (<name>-keywords)
string get_the_keywords_tree= "(tm->tree (" * name * "-keywords))";
tree keyword_tree= as_tree (eval (get_the_keywords_tree));
customize_keyword (keyword_parser, keyword_tree);

// Load (<name>-operators)
string get_the_operators_tree= "(tm->tree (" * name * "-operators))";
tree operator_tree= as_tree (eval (get_the_operators_tree));
customize_operator (operator_tree);

// Load (<name>-numbers)
string get_the_numbers_tree= "(tm->tree (" * name * "-numbers))";
tree number_tree= as_tree (eval (get_the_numbers_tree));
customize_number (number_tree);

// Load (<name>-inline-comment-starts)
list<string> inline_comment_starts_list=
as_list_string (eval ("(" * name * "-inline-comment-starts)"));
array<string> inline_comment_starts;
for (int i=0; i<N(inline_comment_starts_list); i++) {
inline_comment_starts << inline_comment_starts_list[i];
}
inline_comment_parser.set_starts (inline_comment_starts);
tree keyword_config= get_parser_config (name, "keyword");
customize_keyword (keyword_parser, keyword_config);

tree operator_config= get_parser_config (name, "operator");
customize_operator (operator_config);

tree number_config= get_parser_config (name, "number");
customize_number (number_config);

tree string_config= get_parser_config (name, "string");
customize_string (string_config);

// Load (<name>-string)
string get_the_string_tree = "(tm->tree (" * name * "-string))";
tree string_tree= as_tree (eval (get_the_string_tree));
customize_string (string_tree);
tree comment_config= get_parser_config (name, "comment");
customize_comment (comment_config);

// Load (<name>-preprocessors)
string get_the_preprocessor_tree = "(tm->tree (" * name * "-preprocessors))";
tree preprocessor_tree= as_tree (eval (get_the_preprocessor_tree));
customize_preprocessor (preprocessor_tree);
tree preprocessor_config= get_parser_config (name, "preprocessor");
customize_preprocessor (preprocessor_config);
}

tree
prog_language_rep::get_parser_config(string lan, string key) {
string cmd = "(tm->tree (parser-feature " * raw_quote (lan) * " " * raw_quote (key) * "))";
return as_tree (eval (cmd));
}

void
Expand Down Expand Up @@ -132,6 +122,22 @@ prog_language_rep::customize_string (tree config) {
debug_packrat << string_parser.to_string();
}

void
prog_language_rep::customize_comment (tree config) {
for (int i=0; i<N(config); i++) {
tree feature= config[i];
string label= get_label (feature);
if (label == "inline") {
array<string> inline_comment_starts;
for (int i=0; i<N(feature); i++) {
inline_comment_starts << get_label (feature[i]);
}
inline_comment_parser.set_starts (inline_comment_starts);
}
}
}


void
prog_language_rep::customize_preprocessor (tree config) {
for (int i=0; i<N(config); i++) {
Expand Down