No commit message

ByteArrayBinaryInputPort.cpp

@@ -49,7 +49,7 @@ ByteArrayBinaryInputPort::~ByteArrayBinaryInputPort()
 
 int ByteArrayBinaryInputPort::getU8()
 {
-    if (index_ >= size_) return -1;
+    if (index_ >= size_) return EOF;
     return buf_[index_++];
 }
 

ChangeLog

@@ -1,3 +1,7 @@
+2008-09-07  higepon  <higepon@users.sourceforge.jp>
+
+        * UTF8Codec.cpp (readWholeString): Added.
+
 2008-09-06  higepon  <higepon@users.sourceforge.jp>
 
         * Object-inl.h : Break all compilation dependencies.

Codec.h

@@ -46,6 +46,7 @@ class Codec EXTEND_GC
     virtual int out(BinaryOutputPort* port, ucs4char c) = 0;
     virtual int out(uint8_t* buf, ucs4char c) = 0;
     virtual ucs4char in(BinaryInputPort* port) = 0;
+    virtual ucs4string readWholeString(BinaryInputPort* port) = 0;
 };
 
 }; // namespace scheme

TextualByteVectorOutputPort.cpp

@@ -36,7 +36,7 @@
 
 using namespace scheme;
 
-TextualByteVectorOutputPort::TextualByteVectorOutputPort(Transcoder* transcoder) : transcoder_(transcoder), codec_(transcoder->getCodec())
+TextualByteVectorOutputPort::TextualByteVectorOutputPort(Transcoder* transcoder) : transcoder_(transcoder), codec_(transcoder->codec())
 {
 }
 

TextualInputPort.cpp

@@ -39,7 +39,7 @@
 using namespace scheme;
 
 
-TextualInputPort::TextualInputPort(BinaryInputPort* port, Transcoder* coder) : port_(port), codec_(coder->getCodec()), coder_(coder), line_(1)
+TextualInputPort::TextualInputPort(BinaryInputPort* port, Transcoder* coder) : port_(port), codec_(coder->codec()), coder_(coder), line_(1)
 {
 }
 
@@ -51,6 +51,12 @@ TextualInputPort::~TextualInputPort()
 {
 }
 
+int TextualInputPort::getU8()
+{
+    MOSH_ASSERT(port_);
+    return port_->getU8();
+}
+
 ucs4char TextualInputPort::getChar()
 {
     ucs4char c;
@@ -103,3 +109,13 @@ int TextualInputPort::close()
 {
     return port_->close();
 }
+
+Transcoder* TextualInputPort::transcoder() const
+{
+    return coder_;
+}
+
+Codec* TextualInputPort::codec() const
+{
+    return codec_;
+}

TextualInputPort.h

@@ -47,6 +47,7 @@ class TextualInputPort EXTEND_GC
     TextualInputPort();
     virtual ~TextualInputPort();
     virtual ucs4char getChar();
+    virtual int getU8();
     virtual int getLine() const;
     virtual void unGetChar(ucs4char c);
     virtual ucs4string toString();
@@ -55,6 +56,8 @@ class TextualInputPort EXTEND_GC
     virtual Object getDatum(bool& errorOccured);
     virtual Object getDatum2();
     virtual int close();
+    virtual Transcoder* transcoder() const;
+    virtual Codec* codec() const;
 
 private:
     BinaryInputPort* port_;

TextualOutputPort.cpp

@@ -53,7 +53,7 @@ TextualOutputPort::TextualOutputPort()
 }
 
 TextualOutputPort::TextualOutputPort(BinaryOutputPort* port, Transcoder* coder) : port_(port),
-                                                                                  codec_(coder->getCodec()),
+                                                                                  codec_(coder->codec()),
                                                                                   coder_(coder),
                                                                                   isErrorOccured_(false),
                                                                                   errorMessage_(Object::Nil),

Transcoder.h

@@ -66,7 +66,7 @@ class Transcoder EXTEND_GC
     }
 
 
-    Codec* getCodec() const { return codec_; }
+    Codec* codec() const { return codec_; }
 
 private:
     Codec* codec_;

UTF8Codec.cpp

@@ -146,3 +146,16 @@ ucs4char UTF8Codec::in(BinaryInputPort* port)
         exit(-1);
     }
 }
+
+ucs4string UTF8Codec::readWholeString(BinaryInputPort* port)
+{
+    ucs4string ret;
+    for (;;) {
+        const ucs4char ch = in(port);
+        if (EOF == ch) {
+            break;
+        }
+        ret += ch;
+    }
+    return ret;
+}

UTF8Codec.h

@@ -43,6 +43,7 @@ class UTF8Codec : public Codec
     int out(uint8_t* buf, ucs4char u);
     bool isUtf8Tail(uint8_t b);
     ucs4char in(BinaryInputPort* port);
+    ucs4string readWholeString(BinaryInputPort* port);
 };
 
 }; // namespace scheme

main.cpp

@@ -218,11 +218,7 @@ int main(int argc, char *argv[])
     for (Object p = in->getDatum2(); !p.isEof(); p = in->getDatum2()) {
         outPort.toTextualOutputPort()->putDatum(p);
     }
-//     outPort.toTextualOutputPort()->putDatum(in->getDatum2());
-//     outPort.toTextualOutputPort()->putDatum(in->getDatum2());
-//    yyin = fp;
     exit(-1);
-
 
     theVM->evaluate(compiler);
 

reader.h

@@ -43,7 +43,7 @@ typedef struct {
     union {
         bool  boolValue;
         int   intValue;
-        ucs4char* stringValue;
+        char* stringValue;
     };
     scheme::Object object;
 } YYSTYPE;

reader.lex

@@ -55,7 +55,10 @@ peculiar-identifier (([\+\-]|\.\.\.){subsequent}*)
 boolean {true}{false}
 true #[tT]
 false #[fF]
-character  (#\\.+|#\\{character-name}|#\\x{hex-scalar-value})
+
+single-char [^\n ]
+character-literal #\\{single-char}+
+character  ({character-literal}|#\\{character-name}|#\\x{hex-scalar-value})
 character-name  (nul|alarm|backspace|tab|linefeed|newline|vtab|page|return|esc|space|delete)
 string  \"{string-element}*\"
 
@@ -123,12 +126,14 @@ digit-16 {hex-digit}
 <COMMENT><<EOF>>
 <COMMENT>"|"+"#"         BEGIN(INITIAL);
 {identifier} {
-  //  yylval.string_value = yytext;
-  return IDENTIFIER; }
+  yylval.stringValue = yytext ;
+  return IDENTIFIER;
+}
 {string} {
   // remove dobule quotation.
   yytext[yyleng - 1] = '\0';
-  yylval.stringValue = reinterpret_cast<ucs4char*>(yytext + 1);
+  yylval.stringValue = yytext + 1;
+  yyleng -= 2;
   return STRING;
  }
 
@@ -168,7 +173,11 @@ digit-16 {hex-digit}
 "#,"                  { return ABBV_UNSYNTAX; }
 "#,@"                 { return ABBV_UNSYNTAXSPLICING; }
 
-{character} { printf("character, %s", yytext); }
+{character-literal} {
+    yylval.stringValue = yytext + 2;
+    yyleng -= 2;
+    return CHARACTER;
+}
 {lexme} { printf("lexme, %s", yytext); }
 
 

reader.y

@@ -9,23 +9,29 @@
 #include "Symbol.h"
 #include "SString.h"
 #include "ByteVector.h"
+#include "ByteArrayBinaryInputPort.h"
+#include "Codec.h"
 #include "reader.h"
 #define YYDEBUG 1
 using namespace scheme;
+extern Codec* parser_codec();
 extern int yylex();
 extern int yyerror(const char *);
 extern char* yytext;
+extern int yyleng;
 Object parsed;
 %}
 
-%token <Object> IDENTIFIER
+%token <stringValue> IDENTIFIER
 %token <boolValue> BOOLEAN
 %token <stringValue> STRING
+%token <stringValue> CHARACTER
 %token <intValue> NUMBER
 %token LEFT_PAREN RIGHT_PAREN END_OF_FILE VECTOR_START BYTE_VECTOR_START DOT
 %token ABBV_QUASIQUOTE ABBV_QUOTE ABBV_UNQUOTESPLICING ABBV_UNQUOTE
 %token ABBV_SYNTAX ABBV_QUASISYNTAX ABBV_UNSYNTAXSPLICING ABBV_UNSYNTAX
-%type <object> datum lexme_datum compound_datum list datum_list top_level vector bytevector abbreviation
+%type <object> datum lexme_datum compound_datum list datum_list top_level
+%type <object> symbol vector bytevector abbreviation
 %start top_level
 
 %%
@@ -35,11 +41,25 @@ datum : lexme_datum    { $$ = $1;}
       ;
 
 lexme_datum : BOOLEAN { $$ = $1 ? Object::True : Object::False; }
-            | STRING { $$ = $1; }
+            | STRING
+            {
+                $$ = parser_codec()->readWholeString(new ByteArrayBinaryInputPort((uint8_t*)$1, yyleng));
+            }
             | NUMBER { $$ = Object::makeInt($1); }
+            | symbol
+            | CHARACTER
+            {
+                $$ = Object::makeChar(parser_codec()->in(new ByteArrayBinaryInputPort((uint8_t*)$1, yyleng)));
+            }
             | END_OF_FILE { $$ = Object::Eof; }
             ;
-
+symbol : IDENTIFIER
+         {
+             ucs4string text = parser_codec()->readWholeString(new ByteArrayBinaryInputPort((uint8_t*)$1,
+                                                                                            yyleng));
+             $$ = Symbol::intern(text.c_str());
+         }
+       ;
 compound_datum : list { $$ = $1; }
                | vector { $$ = $1; }
                | bytevector { $$ = $1; }