@@ -53,7 +53,7 @@ pub enum Token {
5353 Neq ,
5454 /// Less Than operator `<`
5555 Lt ,
56- /// Greater han operator `>`
56+ /// Greater Than operator `>`
5757 Gt ,
5858 /// Less Than Or Equals operator `<=`
5959 LtEq ,
@@ -101,6 +101,24 @@ pub enum Token {
101101 RBrace ,
102102 /// Right Arrow `=>`
103103 RArrow ,
104+ /// Sharp `#` use for PostgreSQL Bitwise XOR operator
105+ Sharp ,
106+ /// Tilde `~` use for PostgreSQL Bitwise NOT operator
107+ Tilde ,
108+ /// Bitwise left operator `<<` use for PostgreSQL
109+ ShiftLeft ,
110+ /// Bitwise right operator `>>` use for PostgreSQL
111+ ShiftRight ,
112+ /// Exclamation Mark `!` use for PostgreSQL factorial operator
113+ ExclamationMark ,
114+ /// Exclamation Mark `!!` use for PostgreSQL prefix factorial operator
115+ DoubleExclamationMark ,
116+ /// Ampersat `@` use for PostgreSQL abs operator
117+ Ampersat ,
118+ /// PostgreSQL square root math operator
119+ SquareRoot ,
120+ /// PostgreSQL cube root math operator
121+ CubeRoot ,
104122}
105123
106124impl fmt:: Display for Token {
@@ -142,6 +160,15 @@ impl fmt::Display for Token {
142160 Token :: LBrace => f. write_str ( "{" ) ,
143161 Token :: RBrace => f. write_str ( "}" ) ,
144162 Token :: RArrow => f. write_str ( "=>" ) ,
163+ Token :: Sharp => f. write_str ( "#" ) ,
164+ Token :: ExclamationMark => f. write_str ( "!" ) ,
165+ Token :: DoubleExclamationMark => f. write_str ( "!!" ) ,
166+ Token :: Tilde => f. write_str ( "~" ) ,
167+ Token :: Ampersat => f. write_str ( "@" ) ,
168+ Token :: ShiftLeft => f. write_str ( "<<" ) ,
169+ Token :: ShiftRight => f. write_str ( ">>" ) ,
170+ Token :: SquareRoot => f. write_str ( "|/" ) ,
171+ Token :: CubeRoot => f. write_str ( "||/" ) ,
145172 }
146173 }
147174}
@@ -398,7 +425,14 @@ impl<'a> Tokenizer<'a> {
398425 '|' => {
399426 chars. next ( ) ; // consume the '|'
400427 match chars. peek ( ) {
401- Some ( '|' ) => self . consume_and_return ( chars, Token :: StringConcat ) ,
428+ Some ( '/' ) => self . consume_and_return ( chars, Token :: SquareRoot ) ,
429+ Some ( '|' ) => {
430+ chars. next ( ) ; // consume the second '|'
431+ match chars. peek ( ) {
432+ Some ( '/' ) => self . consume_and_return ( chars, Token :: CubeRoot ) ,
433+ _ => Ok ( Some ( Token :: StringConcat ) ) ,
434+ }
435+ }
402436 // Bitshift '|' operator
403437 _ => Ok ( Some ( Token :: Pipe ) ) ,
404438 }
@@ -415,21 +449,24 @@ impl<'a> Tokenizer<'a> {
415449 chars. next ( ) ; // consume
416450 match chars. peek ( ) {
417451 Some ( '=' ) => self . consume_and_return ( chars, Token :: Neq ) ,
418- _ => self . tokenizer_error ( "Expected to see '=' after '!' character" ) ,
452+ Some ( '!' ) => self . consume_and_return ( chars, Token :: DoubleExclamationMark ) ,
453+ _ => Ok ( Some ( Token :: ExclamationMark ) ) ,
419454 }
420455 }
421456 '<' => {
422457 chars. next ( ) ; // consume
423458 match chars. peek ( ) {
424459 Some ( '=' ) => self . consume_and_return ( chars, Token :: LtEq ) ,
425460 Some ( '>' ) => self . consume_and_return ( chars, Token :: Neq ) ,
461+ Some ( '<' ) => self . consume_and_return ( chars, Token :: ShiftLeft ) ,
426462 _ => Ok ( Some ( Token :: Lt ) ) ,
427463 }
428464 }
429465 '>' => {
430466 chars. next ( ) ; // consume
431467 match chars. peek ( ) {
432468 Some ( '=' ) => self . consume_and_return ( chars, Token :: GtEq ) ,
469+ Some ( '>' ) => self . consume_and_return ( chars, Token :: ShiftRight ) ,
433470 _ => Ok ( Some ( Token :: Gt ) ) ,
434471 }
435472 }
@@ -448,6 +485,9 @@ impl<'a> Tokenizer<'a> {
448485 '^' => self . consume_and_return ( chars, Token :: Caret ) ,
449486 '{' => self . consume_and_return ( chars, Token :: LBrace ) ,
450487 '}' => self . consume_and_return ( chars, Token :: RBrace ) ,
488+ '~' => self . consume_and_return ( chars, Token :: Tilde ) ,
489+ '#' => self . consume_and_return ( chars, Token :: Sharp ) ,
490+ '@' => self . consume_and_return ( chars, Token :: Ampersat ) ,
451491 other => self . consume_and_return ( chars, Token :: Char ( other) ) ,
452492 } ,
453493 None => Ok ( None ) ,
@@ -560,6 +600,7 @@ mod tests {
560600 use super :: super :: dialect:: GenericDialect ;
561601 use super :: super :: dialect:: MsSqlDialect ;
562602 use super :: * ;
603+ use crate :: dialect:: PostgreSqlDialect ;
563604
564605 #[ test]
565606 fn tokenize_select_1 ( ) {
@@ -930,6 +971,65 @@ mod tests {
930971 compare ( expected, tokens) ;
931972 }
932973
974+ #[ test]
975+ fn tokenize_postgresql_bitwise_operations ( ) {
976+ let sql = String :: from ( "SELECT ~one << two # three >> four" ) ;
977+ let dialect = PostgreSqlDialect { } ;
978+ let mut tokenizer = Tokenizer :: new ( & dialect, & sql) ;
979+ let tokens = tokenizer. tokenize ( ) . unwrap ( ) ;
980+
981+ let expected = vec ! [
982+ Token :: make_keyword( "SELECT" ) ,
983+ Token :: Whitespace ( Whitespace :: Space ) ,
984+ Token :: Tilde ,
985+ Token :: make_word( "one" , None ) ,
986+ Token :: Whitespace ( Whitespace :: Space ) ,
987+ Token :: ShiftLeft ,
988+ Token :: Whitespace ( Whitespace :: Space ) ,
989+ Token :: make_word( "two" , None ) ,
990+ Token :: Whitespace ( Whitespace :: Space ) ,
991+ Token :: Sharp ,
992+ Token :: Whitespace ( Whitespace :: Space ) ,
993+ Token :: make_word( "three" , None ) ,
994+ Token :: Whitespace ( Whitespace :: Space ) ,
995+ Token :: ShiftRight ,
996+ Token :: Whitespace ( Whitespace :: Space ) ,
997+ Token :: make_word( "four" , None ) ,
998+ ] ;
999+
1000+ compare ( expected, tokens) ;
1001+ }
1002+
1003+ #[ test]
1004+ fn tokenize_postgresql_math_operations ( ) {
1005+ let sql = String :: from ( "SELECT !!5 5! @-6 |/4 ||/8" ) ;
1006+ let dialect = PostgreSqlDialect { } ;
1007+ let mut tokenizer = Tokenizer :: new ( & dialect, & sql) ;
1008+ let tokens = tokenizer. tokenize ( ) . unwrap ( ) ;
1009+
1010+ let expected = vec ! [
1011+ Token :: make_keyword( "SELECT" ) ,
1012+ Token :: Whitespace ( Whitespace :: Space ) ,
1013+ Token :: DoubleExclamationMark ,
1014+ Token :: Number ( "5" . to_string( ) ) ,
1015+ Token :: Whitespace ( Whitespace :: Space ) ,
1016+ Token :: Number ( "5" . to_string( ) ) ,
1017+ Token :: ExclamationMark ,
1018+ Token :: Whitespace ( Whitespace :: Space ) ,
1019+ Token :: Ampersat ,
1020+ Token :: Minus ,
1021+ Token :: Number ( "6" . to_string( ) ) ,
1022+ Token :: Whitespace ( Whitespace :: Space ) ,
1023+ Token :: SquareRoot ,
1024+ Token :: Number ( "4" . to_string( ) ) ,
1025+ Token :: Whitespace ( Whitespace :: Space ) ,
1026+ Token :: CubeRoot ,
1027+ Token :: Number ( "8" . to_string( ) ) ,
1028+ ] ;
1029+
1030+ compare ( expected, tokens) ;
1031+ }
1032+
9331033 fn compare ( expected : Vec < Token > , actual : Vec < Token > ) {
9341034 //println!("------------------------------");
9351035 //println!("tokens = {:?}", actual);
0 commit comments