Reparse: Generate colourful -ast-dump HTML in the browser

This patch introduces the reparse capability which allows arbitrary
serverside code to run an AST generation of Clang, using purely the file
contents and build configuration stored in the database.

To see the results of the parse, the command-line output obtainable via
`clang -Xclang -ast-dump` is available as a coloured HTML over the API.

doc/deps.md

@@ -12,10 +12,11 @@ packages are necessary for building CodeCompass:
 - **`g++`**: For compiling CodeCompass. A version which supports C++14 features
   is required. (Alternatively, you can compile with Clang.)
 - **`libboost-all-dev`**: Boost can be used during the development.
-- **`llvm-6.0`**, **`clang-6.0`**: For compiling CodeCompass with Clang
-  instead of G++.
-- **`llvm-6.0-dev`**, **`libclang-6.0-dev`**: C++ parser uses LLVM/Clang for
-  parsing the source code. Version 6.0 or newer is required.
+- **llvm-7.0**, **clang-7.0**: For compiling CodeCompass with Clang instead of
+  G++.
+- **llvm-7.0-dev**, **libclang-7.0-dev**: C++ parser uses LLVM/Clang for
+  parsing the source code. Version 7.0 or newer is required. Clang 7 is not yet
+  released, so the project must be compiled manually from source.
   ***See [Known issues](#known-issues)!***
 - **`odb`**, **`libodb-dev`**, **`libodb-sqlite-dev`**, **`libodb-pgsql-dev`**:
   For persistence ODB can be used which is an Object Relation Mapping (ORM)
@@ -96,35 +97,41 @@ In Ubuntu 16.04 LTS the LLVM/Clang has some packaging issues, i.e. some libs
 are searched in `/usr/lib` however the package has these in
 `/usr/lib/llvm-3.8/lib` (see
 [http://stackoverflow.com/questions/38171543/error-when-using-cmake-with-llvm](http://stackoverflow.com/questions/38171543/error-when-using-cmake-with-llvm)
-. Also, LLVM version 6.0 is not available as a package.
-This problem causes an error when emitting `cmake` command during CodeCompass
+. This problem causes an error when emitting `cmake` command during CodeCompass
 build. A solution would be to download a prebuilt package from the LLVM/Clang
-webpage but another issue is that the prebuilt packages don't use RTTI which
-is needed for CodeCompass compilation. So Clang needs to be compiled with
-RTTI manually:
+webpage but another issue is that the prebuilt packages don't use runtime type
+informations (RTTI) which is needed for CodeCompass. Clang needs to be compiled
+with RTTI manually.
+
+Additionally, LLVM 7.0 has not yet been made into a full release, it is only
+available as the current in-development version. However, certain CodeCompass
+features depend on these features.
 
 ```bash
+sudo apt-get install unzip
+
 # If you want Clang's diagnostic output to have colours, install the following.
 sudo apt-get install libtinfo-dev
 
-wget http://github.com/llvm-mirror/llvm/archive/release_60.zip
-unzip release_60.zip
-rm release_60.zip
-cd llvm-release_60/tools
-wget http://github.com/llvm-mirror/clang/archive/release_60.zip
-unzip release_60.zip
-mv clang-release_60 clang
-rm release_60.zip
+wget https://github.com/llvm-mirror/llvm/archive/d79c539c3b03f5e05ff3a528a8e4d9bfce121d69.zip -O llvm.zip
+unzip llvm.zip
+rm llvm.zip
+mv llvm-* llvm
+cd llvm/tools
+wget https://github.com/llvm-mirror/clang/archive/e2fbe37780ca1bad55fbdb18a8c448d7156a932d.zip -O clang.zip
+unzip clang.zip
+rm clang.zip
+mv clang-* clang
 cd ../..
 
 mkdir build
 cd build
 export REQUIRES_RTTI=1
 cmake -G "Unix Makefiles" \
-  -DLLVM_ENABLE_RTTI=ON \
   -DCMAKE_BUILD_TYPE=Release \
+  -DLLVM_ENABLE_RTTI=ON \
   -DCMAKE_INSTALL_PREFIX=<clang_install_dir> \
-  ../llvm-release_60
+  ../llvm
 # This make step takes a while. If you have more CPUs then you can compile on
 # several threads with -j<number_of_threads> flag.
 make install

model/include/model/filecontent.h

@@ -36,6 +36,15 @@ struct FileContentIds
   std::string hash;
 };
 
+#pragma db view object(FileContent)
+struct FileContentLength
+{
+  std::string hash;
+
+  #pragma db column("length(" + FileContent::content + ")")
+  std::size_t size;
+};
+
 } // model
 } // cc
 

plugins/cpp_reparse/CMakeLists.txt

@@ -0,0 +1,13 @@
+# The C++ reparser plugin depends on the C++ plugin's components to be built.
+
+if ("${cpp_PLUGIN_DIR}" STREQUAL "")
+  # Use SEND_ERROR here so a build file is not generated at the end.
+  # CodeCompass might use a lot of plugins and produce a lengthy build, so
+  # a warning at configure time would easily be missed by the users.
+  message(SEND_ERROR
+    "C++ Reparse plugin found without C++ plugin in the plugins directory.")
+endif()
+
+add_subdirectory(service)
+
+install_webplugin(webgui)

plugins/cpp_reparse/service/CMakeLists.txt

@@ -0,0 +1,81 @@
+find_package(Clang REQUIRED CONFIG)
+
+llvm_map_components_to_libnames(llvm_libs support core irreader)
+
+include_directories(
+  include
+  gen-cpp
+  ${PROJECT_SOURCE_DIR}/model/include
+  ${PROJECT_SOURCE_DIR}/util/include
+  ${PROJECT_SOURCE_DIR}/webserver/include
+  ${PROJECT_SOURCE_DIR}/service/language/gen-cpp
+  ${PROJECT_SOURCE_DIR}/service/project/gen-cpp
+  ${PROJECT_SOURCE_DIR}/service/project/include
+  ${cpp_PLUGIN_DIR}/model/include
+  ${cpp_PLUGIN_DIR}/parser/include)
+
+include_directories(SYSTEM
+  ${THRIFT_LIBTHRIFT_INCLUDE_DIRS}
+  ${LLVM_INCLUDE_DIRS}
+  ${CLANG_INCLUDE_DIRS})
+
+link_directories(${LLVM_LIBRARY_DIRS})
+
+add_definitions(${LLVM_DEFINITIONS})
+
+add_custom_command(
+  OUTPUT
+    ${CMAKE_CURRENT_SOURCE_DIR}/gen-cpp/cppreparse_constants.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/gen-cpp/cppreparse_constants.h
+    ${CMAKE_CURRENT_SOURCE_DIR}/gen-cpp/cppreparse_types.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/gen-cpp/cppreparse_types.h
+    ${CMAKE_CURRENT_SOURCE_DIR}/gen-cpp/CppReparseService.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/gen-cpp/CppReparseService.h
+    ${CMAKE_CURRENT_SOURCE_DIR}/gen-cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/gen-js
+  COMMAND
+    thrift --gen cpp --gen js:jquery
+      -o ${CMAKE_CURRENT_SOURCE_DIR}
+      -I ${PROJECT_SOURCE_DIR}/service
+      ${CMAKE_CURRENT_SOURCE_DIR}/cppreparse.thrift
+  DEPENDS
+    ${CMAKE_CURRENT_SOURCE_DIR}/cppreparse.thrift
+  COMMENT
+    "Generating Thrift for cppreparse.thrift")
+
+add_library(cppreparsethrift STATIC
+  gen-cpp/cppreparse_constants.cpp
+  gen-cpp/cppreparse_types.cpp
+  gen-cpp/CppReparseService.cpp)
+
+target_compile_options(cppreparsethrift PUBLIC -fPIC)
+
+add_dependencies(cppreparsethrift commonthrift projectthrift)
+
+add_library(cppreparseservice SHARED
+  src/plugin.cpp
+  src/cppreparseservice.cpp
+  src/astcache.cpp
+  src/asthtml.cpp
+  src/databasefilesystem.cpp
+  src/reparser.cpp)
+
+target_compile_options(cppreparseservice PUBLIC -Wno-unknown-pragmas)
+
+target_link_libraries(cppreparseservice
+  util
+  model
+  cppmodel
+  mongoose
+  projectservice
+  ${THRIFT_LIBTHRIFT_LIBRARIES}
+  cppreparsethrift
+  clangTooling
+  clangFrontend
+  clangBasic
+  clangAST
+  clang
+  ${llvm_libs})
+
+install(TARGETS cppreparseservice DESTINATION ${INSTALL_SERVICE_DIR})
+install_js_thrift(gen-js/)

plugins/cpp_reparse/service/cppreparse.thrift

@@ -0,0 +1,48 @@
+include "project/common.thrift"
+include "project/project.thrift"
+
+namespace cpp cc.service.language
+
+// TODO: Throw exception if file ID or ASTNode ID is invalid.
+
+// TODO: Do something about header files and AST Nodes in headers. Currently
+// there is no build action for the headers and thus the whole reparse can't
+// work for them.
+
+/**
+ * To make sure the InfoTree AST view is as lazy-loading as possible, the
+ * information contained therein is split into two types. This one is the basic
+ * smaller type that is used to render a node without any unnecessary extra
+ * details.
+ */
+struct ASTNodeBasic
+{
+  1: i64    visitId /** The visit-order of the node, relative to the root node of the tree being traversed. */
+  2: string type /** The type name of the AST node (e.g. FunctionDecl). */
+  3: bool   hasChildren /** Whether the node has further nodes as children. */
+}
+
+/**
+ * Contains all the other extra information that is usually loaded when the
+ * details of an InfoTree AST is expanded.
+ */
+struct ASTNodeDetail
+{
+  // TODO: Expand 1 to be more detailed, typed, etc. Currently this sends only a dump.
+  1: string otherStuff /** The other things the AST node can be described with. */
+  2: list<ASTNodeBasic> children /** Basic details about the children nodes. */
+}
+
+service CppReparseService
+{
+  /**
+  * Returns false if the server was started with a command-line argument that
+  * disables the reparse capabilities.
+  */
+  bool isEnabled();
+
+  /**
+   * Returns the Abstract Syntax Tree (AST) for the given file as HTML string.
+   */
+  string getAsHTML(1: common.FileId fileId);
+}

plugins/cpp_reparse/service/include/service/cppreparseservice.h

@@ -0,0 +1,62 @@
+#ifndef CC_SERVICE_CPPREPARSESERVICE_H
+#define CC_SERVICE_CPPREPARSESERVICE_H
+
+#include <string>
+#include <memory>
+
+#include <boost/program_options/variables_map.hpp>
+
+#include <odb/database.hxx>
+
+#include <util/odbtransaction.h>
+#include <webserver/servercontext.h>
+
+#include <CppReparseService.h>
+
+namespace cc
+{
+
+namespace service
+{
+
+namespace reparse
+{
+
+class ASTCache;
+class CppReparser;
+
+} // namespace reparse
+
+namespace language
+{
+
+class CppReparseServiceHandler: virtual public CppReparseServiceIf
+{
+public:
+  CppReparseServiceHandler(
+    std::shared_ptr<odb::database> db_,
+    std::shared_ptr<std::string> /* datadir_ */,
+    const cc::webserver::ServerContext& context_);
+
+  ~CppReparseServiceHandler();
+
+  virtual bool isEnabled() override;
+
+  virtual void getAsHTML(
+    std::string& return_,
+    const core::FileId& fileId_) override;
+
+private:
+  std::shared_ptr<odb::database> _db;
+  util::OdbTransaction _transaction;
+  const boost::program_options::variables_map& _config;
+
+  std::shared_ptr<reparse::ASTCache> _astCache;
+  std::unique_ptr<reparse::CppReparser> _reparser;
+};
+
+} // namespace language
+} // namespace service
+} // namespace cc
+
+#endif // CC_SERVICE_CPPREPARSESERVICE_H

plugins/cpp_reparse/service/include/service/reparser.h

@@ -0,0 +1,76 @@
+#ifndef CC_SERVICE_CPPREPARSESERVICE_REPARSER_H
+#define CC_SERVICE_CPPREPARSESERVICE_REPARSER_H
+
+#include <memory>
+
+#include <boost/variant.hpp>
+
+#include <odb/database.hxx>
+#include <util/odbtransaction.h>
+
+// Required for the Thrift objects, such as core::FileId.
+#include "cppreparse_types.h"
+
+namespace clang
+{
+class ASTUnit;
+
+namespace tooling
+{
+class FixedCompilationDatabase;
+} // namespace tooling
+} // namespace clang
+
+namespace cc
+{
+namespace service
+{
+namespace reparse
+{
+
+class ASTCache;
+
+class CppReparser
+{
+public:
+  CppReparser(
+    std::shared_ptr<odb::database> db_,
+    std::shared_ptr<ASTCache> astCache_);
+  CppReparser(const CppReparser&) = delete;
+  CppReparser& operator=(const CppReparser&) = delete;
+  ~CppReparser() = default;
+
+  /**
+   * Obtain the compilation command that used the given file as source file.
+   * @param fileId_ The file ID of the file to query.
+   * @return Either a pointer to the compilation database containing the compile
+   * command for the File ID, if no errors happen, or a string explaining the
+   * error happened.
+   */
+  boost::variant<std::unique_ptr<clang::tooling::FixedCompilationDatabase>,
+                 std::string>
+  getCompilationCommandForFile(const core::FileId& fileId_);
+
+  /**
+   * Returns the ASTUnit instance for the given source file
+   * @param fileId_ The file ID of the file to build the AST for.
+   * @return An ASTUnit pointer, on which ASTConsumers can be executed. If an
+   * error happened and the AST could not be obtained, a string explaining the
+   * error is returned.
+   */
+  boost::variant<std::shared_ptr<clang::ASTUnit>, std::string>
+  getASTForTranslationUnitFile(const core::FileId& fileId_);
+
+private:
+  std::shared_ptr<odb::database> _db;
+  util::OdbTransaction _transaction;
+  std::shared_ptr<ASTCache> _astCache;
+
+  std::string getFilenameForId(const core::FileId& fileId_);
+};
+
+} // namespace reparse
+} // namespace service
+} // namespace cc
+
+#endif // CC_SERVICE_CPPREPARSESERVICE_REPARSER_H