Skip to content
Permalink
Browse files

sketch out lexer and token interfaces.

Swift SVN r5
  • Loading branch information...
lattner committed Jul 18, 2010
1 parent afc81c1 commit 5e88a2175579b0b2ed3c4a15fe8c2382601ac321
Showing with 172 additions and 1 deletion.
  1. +35 −0 include/swift/Lex/Lexer.h
  2. +116 −0 include/swift/Lex/Token.h
  3. +21 −1 lib/Lex/Lexer.cpp
@@ -0,0 +1,35 @@
//===--- Lexer.h - Swift Language Lexer -------------------------*- C++ -*-===//
//
// This source file is part of the Swift.org open source project
//
// Copyright (c) 2014 - 2015 Apple Inc. and the Swift project authors

This comment has been minimized.

Copy link
@wrutkowski

wrutkowski Dec 10, 2015

Why do we have future copyright (2014 - 2015) in commit from back 2010?

// Licensed under Apache License v2.0 with Runtime Library Exception
//
// See http://swift.org/LICENSE.txt for license information
// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
//
//===----------------------------------------------------------------------===//
//
// This file defines the Lexer interface.
//
//===----------------------------------------------------------------------===//

#ifndef SWIFT_LEXER_H
#define SWIFT_LEXER_H

#include "swift/Lex/Token.h"

namespace swift {

class Lexer {
Lexer(const Lexer&); // DO NOT IMPLEMENT
void operator=(const Lexer&); // DO NOT IMPLEMENT
public:

void Lex(Token &Result);
};


} // end namespace swift

#endif
@@ -0,0 +1,116 @@
//===--- Token.h - Token interface ------------------------------*- C++ -*-===//
//
// This source file is part of the Swift.org open source project
//
// Copyright (c) 2014 - 2015 Apple Inc. and the Swift project authors
// Licensed under Apache License v2.0 with Runtime Library Exception
//
// See http://swift.org/LICENSE.txt for license information
// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
//
//===----------------------------------------------------------------------===//
//
// This file defines the Token interface.
//
//===----------------------------------------------------------------------===//

#ifndef SWIFT_TOKEN_H
#define SWIFT_TOKEN_H

#include "llvm/Support/SMLoc.h"
#include "llvm/ADT/StringRef.h"

namespace swift {

namespace tok {
enum TokenKind {
unknown = 0,
eof,

#define KEYWORD(X) kw_ ## X,
// Types.
KEYWORD(int)
// KEYWORD(uint)
// KEYWORD(int8)
// KEYWORD(uint8)
// KEYWORD(int16)
// KEYWORD(uint16)
// KEYWORD(int64)
// KEYWORD(uint64)
// KEYWORD(float)

KEYWORD(var)

// KEYWORD(foreach)
#undef KEYWORD

#define PUNCTUATOR(X, Y) X,
PUNCTUATOR(comma, ",")
PUNCTUATOR(colon, ":")
PUNCTUATOR(semi, ";")
PUNCTUATOR(equal, "=")
PUNCTUATOR(l_paren, "(")
PUNCTUATOR(r_paren, ")")

#undef PUNCTUATOR

NUM_TOKENS
};
} // end namespace tok.

/// Token - This structure provides full information about a lexed token.
/// It is not intended to be space efficient, it is intended to return as much
/// information as possible about each returned token. This is expected to be
/// compressed into a smaller form if memory footprint is important.
///
class Token {
/// Kind - The actual flavor of token this is.
///
tok::TokenKind Kind;

/// The location of the token.
llvm::SMLoc Loc;

/// Text - The actual string covered by the token in the source buffer.
llvm::StringRef Text;

public:

tok::TokenKind getKind() const { return Kind; }
void setKind(tok::TokenKind K) { Kind = K; }

/// is/isNot - Predicates to check if this token is a specific kind, as in
/// "if (Tok.is(tok::l_brace)) {...}".
bool is(tok::TokenKind K) const { return Kind == K; }
bool isNot(tok::TokenKind K) const { return Kind != K; }

/// getLocation - Return a source location identifier for the specified
/// offset in the current file.
llvm::SMLoc getLocation() const { return Loc; }
void setLocation(llvm::SMLoc L) { Loc = L; }

llvm::StringRef getText() const { return Text; }
void setText(llvm::StringRef T) { Text = T; }

unsigned getLength() const { return Text.size(); }

/// startToken - Reset all flags to cleared.
///
void startToken() {
Kind = tok::unknown;
Loc = llvm::SMLoc();
Text = llvm::StringRef();
}
};

} // end namespace swift


#if 0
namespace llvm {
template <>
struct isPodLike<swift::Token> { static const bool value = true; };
} // end namespace llvm
#endif

#endif
@@ -1,2 +1,22 @@
int X;
//===--- Lexer.cpp - Swift Language Lexer ---------------------------------===//
//
// This source file is part of the Swift.org open source project
//
// Copyright (c) 2014 - 2015 Apple Inc. and the Swift project authors
// Licensed under Apache License v2.0 with Runtime Library Exception
//
// See http://swift.org/LICENSE.txt for license information
// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
//
//===----------------------------------------------------------------------===//
//
// This file implements the Lexer and Token interfaces.
//
//===----------------------------------------------------------------------===//

#include "swift/Lex/Lexer.h"
using namespace swift;



int X;

2 comments on commit 5e88a21

@Rogaven

This comment has been minimized.

Copy link

replied Dec 4, 2015

@lattner Why didn't you use any lexer generator (lex, ragel, ..) for sketch of your compiler?

@lattner

This comment has been minimized.

Copy link
Collaborator Author

replied Dec 6, 2015

Please sign in to comment.
You can’t perform that action at this time.