Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Fetching contributors…

Cannot retrieve contributors at this time

52 lines (41 sloc) 0.914 kB
%top{
#define TOK_DONE 0
#define TOK_NUMBER 1
#define TOK_WORD 2
#include "segment.h"
typedef struct lexinfo {
pos_t start;
pos_t end;
} lexinfo;
}
%option 8bit reentrant fast noyywrap
%option outfile="tokenizer.lex.c" header-file="tokenizer.lex.h"
%option extra-type="struct lexinfo*"
FIRSTWORDCHAR [^[:blank:][:punct:]<>\r\n]
INNERWORDCHAR [^[:blank:]<>\r\n]
LASTWORDCHAR [^[:blank:][:punct:]<>\r\n]
%%
\-?[[:digit:]]+(\.([[:digit:]]+)?)? {
yyextra->start = yyextra->end;
yyextra->end += (pos_t)yyleng;
return TOK_NUMBER;
}
{FIRSTWORDCHAR}{INNERWORDCHAR}*{LASTWORDCHAR} {
yyextra->start = yyextra->end;
yyextra->end += (pos_t)yyleng;
return TOK_WORD;
}
{FIRSTWORDCHAR}{LASTWORDCHAR}? {
yyextra->start = yyextra->end;
yyextra->end += (pos_t)yyleng;
return TOK_WORD;
}
[\r\n] {
yyextra->start++;
yyextra->end++;
}
. {
yyextra->start++;
yyextra->end++;
}
%%
Jump to Line
Something went wrong with that request. Please try again.