Add and use cpp_token_string for unexposed entities

Author: foonathan

include/cppast/code_generator.hpp

@@ -500,12 +500,17 @@ namespace cppast
     /// \exclude
     class cpp_template_argument;
 
+    /// \exclude
+    class cpp_token_string;
+
     /// \exclude
     namespace detail
     {
         void write_template_arguments(
             code_generator::output&                                                output,
             type_safe::optional<type_safe::array_ref<const cpp_template_argument>> arguments);
+
+        void write_token_string(code_generator::output& output, const cpp_token_string& tokens);
     } // namespace detail
 } // namespace cppast
 

include/cppast/cpp_entity.hpp

@@ -11,6 +11,7 @@
 #include <type_safe/optional_ref.hpp>
 
 #include <cppast/detail/intrusive_list.hpp>
+#include <cppast/cpp_token.hpp>
 
 namespace cppast
 {
@@ -141,9 +142,7 @@ namespace cppast
 
     protected:
         /// \effects Creates it giving it the the name.
-        cpp_entity(std::string name) : name_(std::move(name)), user_data_(nullptr)
-        {
-        }
+        cpp_entity(std::string name) : name_(std::move(name)), user_data_(nullptr) {}
 
     private:
         /// \returns The kind of the entity.
@@ -182,27 +181,27 @@ namespace cppast
         /// \returns A newly built and registered unexposed entity.
         /// \notes It will be registered as a declaration.
         static std::unique_ptr<cpp_entity> build(const cpp_entity_index& index, cpp_entity_id id,
-                                                 std::string name, std::string spelling);
+                                                 std::string name, cpp_token_string spelling);
 
         /// \returns A newly built unnamed unexposed entity.
         /// It will not be registered.
-        static std::unique_ptr<cpp_entity> build(std::string spelling);
+        static std::unique_ptr<cpp_entity> build(cpp_token_string spelling);
 
         /// \returns The spelling of that entity.
-        const std::string& spelling() const noexcept
+        const cpp_token_string& spelling() const noexcept
         {
             return spelling_;
         }
 
     private:
-        cpp_unexposed_entity(std::string name, std::string spelling)
+        cpp_unexposed_entity(std::string name, cpp_token_string spelling)
         : cpp_entity(std::move(name)), spelling_(std::move(spelling))
         {
         }
 
         cpp_entity_kind do_get_entity_kind() const noexcept override;
 
-        std::string spelling_;
+        cpp_token_string spelling_;
     };
 
     /// \returns Whether or not the entity is templated.

include/cppast/cpp_expression.hpp

@@ -8,6 +8,7 @@
 #include <atomic>
 #include <memory>
 
+#include <cppast/cpp_token.hpp>
 #include <cppast/cpp_type.hpp>
 
 namespace cppast
@@ -82,20 +83,20 @@ namespace cppast
     public:
         /// \returns A newly created unexposed expression.
         static std::unique_ptr<cpp_unexposed_expression> build(std::unique_ptr<cpp_type> type,
-                                                               std::string               str)
+                                                               cpp_token_string          str)
         {
             return std::unique_ptr<cpp_unexposed_expression>(
                 new cpp_unexposed_expression(std::move(type), std::move(str)));
         }
 
         /// \returns The expression as a string.
-        const std::string& expression() const noexcept
+        const cpp_token_string& expression() const noexcept
         {
             return str_;
         }
 
     private:
-        cpp_unexposed_expression(std::unique_ptr<cpp_type> type, std::string str)
+        cpp_unexposed_expression(std::unique_ptr<cpp_type> type, cpp_token_string str)
         : cpp_expression(std::move(type)), str_(std::move(str))
         {
         }
@@ -105,7 +106,7 @@ namespace cppast
             return cpp_expression_kind::unexposed_t;
         }
 
-        std::string str_;
+        cpp_token_string str_;
     };
 
     /// A [cppast::cpp_expression]() that is a literal.

include/cppast/cpp_template.hpp

@@ -11,6 +11,7 @@
 
 #include <cppast/cpp_entity.hpp>
 #include <cppast/cpp_entity_container.hpp>
+#include <cppast/cpp_token.hpp>
 #include <cppast/cpp_template_parameter.hpp>
 
 namespace cppast
@@ -179,7 +180,7 @@ namespace cppast
         }
 
         type_safe::variant<std::vector<cpp_template_argument>, std::string> arguments_;
-        cpp_template_ref templ_;
+        cpp_template_ref                                                    templ_;
     };
 
     /// Base class for all entities modelling a C++ template specialization.
@@ -214,9 +215,9 @@ namespace cppast
         /// \requires The arguments are not exposed, i.e. `arguments_exposed()` returns `false`.
         /// \notes For function template specializations it can be empty,
         /// meaning that the arguments are not explictly given but deduced from the signature.
-        const std::string& unexposed_arguments() const noexcept
+        const cpp_token_string& unexposed_arguments() const noexcept
         {
-            return arguments_.value(type_safe::variant_type<std::string>{});
+            return arguments_.value(type_safe::variant_type<cpp_token_string>{});
         }
 
         /// \returns Whether or not the specialization is a full specialization.
@@ -252,7 +253,7 @@ namespace cppast
             }
 
             /// \effects Adds unexposed arguments as string.
-            void add_unexposed_arguments(std::string arg)
+            void add_unexposed_arguments(cpp_token_string arg)
             {
                 auto& specialization =
                     static_cast<cpp_template_specialization&>(*this->template_entity);
@@ -276,8 +277,8 @@ namespace cppast
         }
 
     private:
-        type_safe::variant<std::vector<cpp_template_argument>, std::string> arguments_;
-        cpp_entity_id templ_;
+        type_safe::variant<std::vector<cpp_template_argument>, cpp_token_string> arguments_;
+        cpp_entity_id                                                            templ_;
     };
 } // namespace cppast
 

include/cppast/cpp_token.hpp

@@ -0,0 +1,137 @@
+// Copyright (C) 2017 Jonathan Müller <jonathanmueller.dev@gmail.com>
+// This file is subject to the license terms in the LICENSE file
+// found in the top-level directory of this distribution.
+
+#ifndef CPPAST_CPP_TOKEN_HPP_INCLUDED
+#define CPPAST_CPP_TOKEN_HPP_INCLUDED
+
+#include <string>
+#include <vector>
+
+#include <type_safe/reference.hpp>
+
+namespace cppast
+{
+    /// The kinds of C++ tokens.
+    enum class cpp_token_kind
+    {
+        identifier,  //< Any identifier.
+        keyword,     //< Any keyword.
+        literal,     //< Any literal.
+        punctuation, //< Any other punctuation.
+
+        unknown, //< An unknown token.
+    };
+
+    /// A C++ token.
+    struct cpp_token
+    {
+        std::string    spelling;
+        cpp_token_kind kind;
+
+        cpp_token(cpp_token_kind kind, std::string spelling)
+        : spelling(std::move(spelling)), kind(kind)
+        {
+        }
+
+        friend bool operator==(const cpp_token& lhs, const cpp_token& rhs) noexcept
+        {
+            return lhs.spelling == rhs.spelling;
+        }
+
+        friend bool operator!=(const cpp_token& lhs, const cpp_token& rhs) noexcept
+        {
+            return !(rhs == lhs);
+        }
+    };
+
+    /// A combination of multiple C++ tokens.
+    class cpp_token_string
+    {
+    public:
+        /// Builds a token string.
+        class builder
+        {
+        public:
+            builder() = default;
+
+            /// \effects Adds a token.
+            void add_token(cpp_token tok)
+            {
+                tokens_.push_back(std::move(tok));
+            }
+
+            /// \effects Converts a trailing `>>` to `>` token.
+            void unmunch();
+
+            /// \returns The finished string.
+            cpp_token_string finish()
+            {
+                return cpp_token_string(std::move(tokens_));
+            }
+
+        private:
+            std::vector<cpp_token> tokens_;
+        };
+
+        /// \effects Creates it from a sequence of tokens.
+        cpp_token_string(std::vector<cpp_token> tokens) : tokens_(std::move(tokens)) {}
+
+        /// \effects Creates from a string.
+        /// \notes This does not do tokenization, it will only store a single, unknown token!
+        static cpp_token_string from_string(std::string str)
+        {
+            return cpp_token_string({cpp_token(cpp_token_kind::unknown, std::move(str))});
+        }
+
+        /// \exclude target
+        using iterator = std::vector<cpp_token>::const_iterator;
+
+        /// \returns An iterator to the first token.
+        iterator begin() const noexcept
+        {
+            return tokens_.begin();
+        }
+
+        /// \returns An iterator one past the last token.
+        iterator end() const noexcept
+        {
+            return tokens_.end();
+        }
+
+        /// \returns Whether or not the string is empty.
+        bool empty() const noexcept
+        {
+            return tokens_.empty();
+        }
+
+        /// \returns A reference to the first token.
+        const cpp_token& front() const noexcept
+        {
+            return tokens_.front();
+        }
+
+        /// \returns A reference to the last token.
+        const cpp_token& back() const noexcept
+        {
+            return tokens_.back();
+        }
+
+        /// \returns The string representation of the tokens, without any whitespace.
+        std::string as_string() const;
+
+    private:
+        std::vector<cpp_token> tokens_;
+
+        friend bool operator==(const cpp_token_string& lhs, const cpp_token_string& rhs);
+    };
+
+    bool operator==(const cpp_token_string& lhs, const cpp_token_string& rhs);
+
+    inline bool operator!=(const cpp_token_string& lhs, const cpp_token_string& rhs)
+    {
+        return !(lhs == rhs);
+    }
+} // namespace cppast
+
+#endif // CPPAST_CPP_TOKEN_HPP_INCLUDED

src/CMakeLists.txt

@@ -36,6 +36,7 @@ set(header
     ../include/cppast/cpp_storage_class_specifiers.hpp
     ../include/cppast/cpp_template.hpp
     ../include/cppast/cpp_template_parameter.hpp
+    ../include/cppast/cpp_token.hpp
     ../include/cppast/cpp_type.hpp
     ../include/cppast/cpp_type_alias.hpp
     ../include/cppast/cpp_variable.hpp
@@ -68,6 +69,7 @@ set(source
         cpp_preprocessor.cpp
         cpp_static_assert.cpp
         cpp_template_parameter.cpp
+        cpp_token.cpp
         cpp_type.cpp
         cpp_type_alias.cpp
         cpp_variable.cpp