parser.yxx

/* -*- mode: c++ -*- */
// Oblique grammar for parser (Bison C++ generator).

//- Options -------------------------------------------------------------------

%require "3.6"

%language "c++"
%define api.namespace {oblique::parser}
%define api.parser.class {Parser}
%define api.value.type variant // See: "C++ Variants"
%define api.token.constructor
%define parse.error detailed
%define parse.trace
%define api.token.prefix {TOK_}
// %verbose, have to specify the output file.
// %token-table // Has no effect on C++ code generation.

%defines
%locations
%define filename_type "const std::string"
// %initial-action does not work to set the filename since the location() is set
// on the symbol itself.
// This doesn't work either.
// %initial-action {
//   driver.WithLocation(@$);
// }

// Input state to parser.
%parse-param {::oblique::scanner::Scanner& lexer} {::oblique::parser::Driver& driver}

//- Code blocks ---------------------------------------------------------------

// Inserted in header, before anything else
%code requires {
#include "oblique/data.pb.h"
#include "oblique/data.h"

#include <cstdint>
#include <cstdio>
#include <fstream>
#include <list>
#include <memory>
#include <string>

#include "absl/strings/string_view.h"

namespace oblique {
namespace scanner { class Scanner; }
namespace parser { class Driver; }
typedef absl::string_view strview;

// Accumulator for term list.
struct TermList {
  // List of word text.
  std::list<std::string> words;

  // List of yet-to-be-resolved (type, ident) pairs.
  // Has ownershipo.
  std::list<proto::Ref*> unresolved_refs;
};

}  // namespace oblique
}  // requires

// Inserted in header, at the end.
%code provides {
namespace oblique {

// Parse contents from the given string.
// The 'line_offset' can be used in unit tests to mark the start of the test
// string in the parsed file.
std::unique_ptr<proto::Database> ParseString(const absl::string_view& input_string,
                                            const absl::string_view& filename,
                                            int line_offset = 0);

// Parse contents from a file.
std::unique_ptr<proto::Database> ParseFile(const absl::string_view& filename);

// Parse contents from stdin.
std::unique_ptr<proto::Database> ParseStdin();

}  // namespace oblique
}  // provides

//-----------------------------------------------------------------------------
// Inserted in implementation after including parser header.
%code top {
#include "oblique/scanner.h"  /* Generated by reflex. */

#include <iostream>
#include <list>
#include <string>
#include <unordered_map>
#include <utility>

#include "absl/container/flat_hash_map.h"
#include "absl/strings/str_cat.h"
#include "absl/strings/str_join.h"
#include "absl/strings/string_view.h"
#include "reflex/input.h"

using absl::string_view;
using std::cerr;
using std::cout;
using std::endl;
using std::list;
using std::pair;
using std::string;
using std::unique_ptr;

namespace oblique {
namespace parser {

// In-memory build state for graph of nodes. This driver accepts actions from
// the parser and simply builds up the parsed state. Since the input is not
// assumed to be ordered, resolving references occurs as a separate phase.
class Driver {
public:
  Driver();
  virtual ~Driver();

  // Set the location for errors to occur.
  // This is a stateful setter whose data get used to report errors.
  void WithLocation(const parser::location& location) {
    location_ = location;
  }

  // Add a new type.
  proto::Type* AddType(string_view type, string_view contents,
                       proto::TypeFlavor flavor);

  // Add a new instance of an object node.
  proto::Object* AddObject(string_view type,
                           string_view ident,
                           string_view contents);

  // Add a new instance of a lazy object node.
  // The identifer is allocated automatically.
  proto::Object* AddAutoObject(string_view type,
                               string_view contents) {
    return AddObject(type, "", contents);
  }

  // Add a new instance of an object with default type.
  // The identifer is allocated automatically.
  proto::Object* AddDefaultObject(string_view contents) {
    return AddAutoObject(data::kItemType, contents);
  }

  // Get the resulting database of objects and clear it.
  unique_ptr<proto::Database> GetClearDatabase() {
    return std::move(db_);
  }

private:
  // Allocate a new id.
  inline int32_t NewId() {
    return new_id_counter_++;
  }

  // A list of the objects, in the order they appeared in the input file.
  unique_ptr<proto::Database> db_;

  // Counter for producing unique identifiers for lazy references.
  // Note: Use Swiss table for heterogeneous lookup.
  int32_t new_id_counter_ = 0;

  // Last registered location.
  parser::location location_;
};

}  //  namespace parser
}  // namespace oblique
}  // top

// Inserted in implementation before including parser header
// (Within bison's parse() we should invoke lexer.lex(), not the global yylex().)
%code {
#undef yylex
#define yylex lexer.lex
}

//- Tokens --------------------------------------------------------------------
// Note: Token type names appear in the parser header; qualify fully.

/* Reserved keywords. */
%token <strview> TYPE
%token <strview> LAZYTYPE
%token <strview> IGNORE
%token MACRO
%token IMPORT

/* End of file. */
// %token EOF 0  // Does not have to  get declared explicitly.

/* Components of body. */
%token INDENT;
%token EOL;
%token <strview> COMMENT
%token <proto::Ref*> REF
%token <strview> AUTOREF
%token <strview> WORD

// %type <strview> term
%type <TermList*> termlist
%type <proto::Type*> type_declaration
%type <proto::Object*> item_definition
%type <proto::Object*> explicit_definition
%type <proto::Object*> lazy_definition

//- Grammar rules -------------------------------------------------------------
%%

declarations:
    declarations indented_declaration
    | empty

indented_declaration:
    indentlist declaration
    | global_declaration

global_declaration:
    type_declaration

indentlist:
    indentlist INDENT
    | empty

// All possible line declarations.
declaration:
    comment
    | explicit_definition
    | lazy_definition
    | item_definition


/* | command_type */
/* | command_lazytype */
/* | command_macro */

comment:
    COMMENT EOL
      { /* Ignore comments. */ }

termlist:
    termlist[terms] REF
      {
        $terms->words.push_back(data::MakeRefKey(*$REF));
        $terms->unresolved_refs.push_back($REF);  // Ownership transfer.
        $$ = $terms;
      }
    | termlist[terms] AUTOREF
      {
        $$ = $terms;
      }
    | termlist[terms] WORD
      {
        $terms->words.push_back(string($WORD));
        $$ = $terms;
      }
    | empty
      {
        // TODO(blais): Would unique<> work here?
        $$ = new TermList();  // {04aedc889a9d}
      }

// Definition of new types.
type_declaration:
    TYPE termlist EOL
      {
        driver.WithLocation(@$);
        $$ = driver.AddType($TYPE, absl::StrJoin($termlist->words, " "),
                            proto::TypeFlavor::STRICT);
      }
    | LAZYTYPE termlist EOL
      {
        driver.WithLocation(@$);
        $$ = driver.AddType($LAZYTYPE, absl::StrJoin($termlist->words, " "),
                            proto::TypeFlavor::LAZY);
      }
    | IGNORE termlist EOL
      {
        driver.WithLocation(@$);
        $$ = driver.AddType($IGNORE, absl::StrJoin($termlist->words, " "),
                            proto::TypeFlavor::IGNORE);
      }

// Definition of objects declared with type.
explicit_definition:
    REF termlist EOL
      {
        // Explicit object definition.
        driver.WithLocation(@$);
        proto::Object* obj = driver.AddObject(
          $REF->type(), $REF->ident(), absl::StrJoin($termlist->words, " "));
        for (auto ref : $termlist->unresolved_refs) {
          obj->mutable_unresolved_refs()->AddAllocated(ref);  // Ownership transfer.
        }
        delete $termlist; // {04aedc889a9d}
        delete $REF;
        $$ = obj;
      }

lazy_definition:
    AUTOREF termlist EOL
      {
        // Explicit object definition.
        driver.WithLocation(@$);
        proto::Object* obj = driver.AddAutoObject(
          $AUTOREF, absl::StrJoin($termlist->words, " "));
        for (auto ref : $termlist->unresolved_refs) {
          obj->mutable_unresolved_refs()->AddAllocated(ref);  // Ownership transfer.
        }
        delete $termlist; // {04aedc889a9d}
        $$ = obj;
      }

// Definition of items without an explicit type.
item_definition:
    WORD termlist EOL
      {
        // Item definition.
        driver.WithLocation(@$);
        $termlist->words.push_front(string($WORD));
        proto::Object* obj = driver.AddDefaultObject(absl::StrJoin($termlist->words, " "));
        for (auto ref : $termlist->unresolved_refs) {
          obj->mutable_unresolved_refs()->AddAllocated(ref);  // Ownership transfer.
        }
        delete $termlist; // {04aedc889a9d}
        $$ = obj;
      }

empty:

%%
//- User code ----------------------------------------------------------------

void oblique::parser::Parser::error(const location& loc, const string& msg)
{
  cerr << loc << ": " << msg << endl;
  if (loc.begin.line == loc.end.line && loc.begin.line == lexer.lineno())
  {
    cerr << lexer.matcher().line() << endl;
    for (size_t i = 0; i < loc.begin.column; ++i)
      cerr << " ";
    for (size_t i = loc.begin.column; i <= loc.end.column; ++i)
      cerr << "~";
    cerr << endl;
  }
  else
  {
    FILE* file = lexer.in().file(); // The current file being scanned.
    if (file != NULL)
    {
      oblique::scanner::Scanner::Matcher* m = lexer.new_matcher(file); // New matcher.
      lexer.push_matcher(m); // Save the current matcher.
      off_t pos = ftell(file); // Save current position in the file.
      fseek(file, 0, SEEK_SET); // Go to the start of the file.
      for (size_t i = 1; i < loc.begin.line; ++i)
        m->skip('\n'); // Skip to the next line.
      for (size_t i = loc.begin.line; i <= loc.end.line; ++i)
      {
        cerr << m->line() << endl; // Display the offending line.
        m->skip('\n'); // Next line.
      }
      fseek(file, pos, SEEK_SET); // Restore position in the file to continue scanning.
      lexer.pop_matcher(); // Restore matcher.
    }
  }
  if (lexer.size() == 0) // If token is unknown (no match).
    lexer.matcher().winput(); // Skip character.
}

namespace oblique {

namespace parser {

Driver::Driver() :
  db_(new proto::Database)
{
  // Add default base objects to the database.
  int saved_line = location_.begin.line;
  location_.begin.line = 0;
  AddType(data::kItemType, "Item type", proto::TypeFlavor::LAZY);
  location_.begin.line = saved_line;
}

Driver::~Driver() {
}

proto::Type* Driver::AddType(string_view type,
                             string_view contents,
                             proto::TypeFlavor flavor) {
  auto typ = db_->mutable_type()->Add();
  if (location_.begin.line > 0) {
    typ->set_lineno(location_.begin.line);
  }
  typ->set_type(type.data(), type.size());
  typ->set_contents(contents.data(), contents.size());
  if (flavor != proto::TypeFlavor::STRICT)
    typ->set_flavor(flavor);
  return typ;
}

proto::Object* Driver::AddObject(string_view type,
                                 string_view ident,
                                 string_view contents) {
  auto obj = db_->mutable_object()->Add();
  obj->set_lineno(location_.begin.line);
  auto* id = obj->mutable_id();
  id->set_type(type.data(), type.size());
  if (!ident.empty()) {
    id->set_ident(ident.data(), ident.size());
  }
  /// obj->set_sequence(NewId()); // TODO(blais): Remove
  obj->set_contents(contents.data(), contents.size());
  return obj;
}

}  //  namespace parser

namespace scanner {

// An extension to the generated scanner that also fetches the filename for the
// location.
class ScannerWithFile : public scanner::Scanner {
public:
  using scanner::Scanner::Scanner;

  // Set the filename for location tokens.
  void SetFilename(const string* filename) {
    filename_ = filename;
  }

  void SetLineOffset(const int line_offset) {
    line_offset_ = line_offset;
  }

  // Override the location to include the filename.
  oblique::parser::location location() override
  {
    /* yylloc.begin.line = static_cast<unsigned int>(matcher().lineno()); */
    /* yylloc.begin.column = static_cast<unsigned int>(matcher().columno()); */
    /* yylloc.end.line = static_cast<unsigned int>(matcher().lineno_end()); */
    /* yylloc.end.column = static_cast<unsigned int>(matcher().columno_end()); */
    auto yylloc = scanner::Scanner::location();
    if (filename_) {
      yylloc.begin.filename = filename_;
    }
    if (line_offset_ > 0) {
      yylloc.begin.line += line_offset_;
      yylloc.end.line += line_offset_;
    }
    return yylloc;
  }
private:
  const string* filename_ = nullptr;
  int line_offset_ = 0;
};

}  // namespace scanner

namespace parser {

// Parse the given input to a database.
std::unique_ptr<proto::Database> Parse(const reflex::Input& input,
                                       const string* filename = nullptr,
                                       int line_offset = 0) {
  // Create a scanner and driver.
  scanner::ScannerWithFile scanner(input, cout);
  if (filename != nullptr) {
    scanner.SetFilename(filename);
  }
  scanner.SetLineOffset(line_offset);
  parser::Driver driver;

  // Run the parser.
  oblique::parser::Parser parser(scanner, driver);
  int result = parser.parse();
  (void)result;

  auto db = driver.GetClearDatabase();
  return db;
}

}  // namespace parser

std::unique_ptr<proto::Database> ParseString(const string_view& input_string,
                                             const string_view& filename,
                                             int line_offset) {
  // Ensure the string is converted to UTF8 by using reflex::Input and
  // instantiate a scanner.
  reflex::Input input(input_string.data(), input_string.size());
  static const string filename_str(filename);
  return parser::Parse(input, &filename_str, line_offset);
}

std::unique_ptr<proto::Database> ParseFile(const string_view& filename) {
  std::ifstream in = std::ifstream(string(filename), std::ios::in);
  reflex::Input input(in);
  string filename_str(filename);
  return parser::Parse(input, &filename_str);
}

std::unique_ptr<proto::Database> ParseStdin() {
  reflex::Input input(stdin, reflex::Input::file_encoding::utf8);
  static const string filename = "<string>";
  auto db = parser::Parse(input, &filename);
  fclose(stdin);
  return db;
}

}  // namespace oblique