Merge pull request #32239 from HassanElDesouky/GSoC-DiagEngineRefactor

[GSoC] Refactor DiagnosticEngine to use `YAML` files

Author: xedin

include/swift/AST/CMakeLists.txt

@@ -0,0 +1,3 @@
+swift_install_in_component(DIRECTORY diagnostics
+                           DESTINATION "share/swift/"
+                           COMPONENT compiler)

include/swift/AST/DiagnosticEngine.h

@@ -58,6 +58,11 @@ namespace swift {
     DiagID ID;
   };
 
+  struct DiagnosticNode {
+    DiagID id;
+    std::string msg;
+  };
+
   namespace detail {
     /// Describes how to pass a diagnostic argument of the given type.
     ///
@@ -629,7 +634,26 @@ namespace swift {
     DiagnosticState(DiagnosticState &&) = default;
     DiagnosticState &operator=(DiagnosticState &&) = default;
   };
-    
+
+  class LocalizationProducer {
+  public:
+    /// If the  message isn't available/localized in the current `yaml` file,
+    /// return the fallback default message.
+    virtual std::string getMessageOr(DiagID id,
+                                     std::string defaultMessage) const {
+      return defaultMessage;
+    }
+
+    virtual ~LocalizationProducer() {}
+  };
+
+  class YAMLLocalizationProducer final : public LocalizationProducer {
+  public:
+    std::vector<std::string> diagnostics;
+    explicit YAMLLocalizationProducer(std::string locale, std::string path);
+    std::string getMessageOr(DiagID id,
+                             std::string defaultMessage) const override;
+  };
   /// Class responsible for formatting diagnostics and presenting them
   /// to the user.
   class DiagnosticEngine {
@@ -663,6 +687,10 @@ namespace swift {
     /// but rather stored until all transactions complete.
     llvm::StringSet<llvm::BumpPtrAllocator &> TransactionStrings;
 
+    /// Diagnostic producer to handle the logic behind retriving a localized
+    /// diagnostic message.
+    std::unique_ptr<LocalizationProducer> localization;
+
     /// The number of open diagnostic transactions. Diagnostics are only
     /// emitted once all transactions have closed.
     unsigned TransactionCount = 0;
@@ -734,6 +762,11 @@ namespace swift {
       return diagnosticDocumentationPath;
     }
 
+    void setLocalization(std::string locale, std::string path) {
+      if (!locale.empty() && !path.empty())
+        localization = std::make_unique<YAMLLocalizationProducer>(locale, path);
+    }
+
     void ignoreDiagnostic(DiagID id) {
       state.setDiagnosticBehavior(id, DiagnosticState::Behavior::Ignore);
     }
@@ -955,8 +988,7 @@ namespace swift {
     void emitTentativeDiagnostics();
 
   public:
-    static const char *diagnosticStringFor(const DiagID id,
-                                           bool printDiagnosticName);
+    const char *diagnosticStringFor(const DiagID id, bool printDiagnosticName);
 
     /// If there is no clear .dia file for a diagnostic, put it in the one
     /// corresponding to the SourceLoc given here.

include/swift/AST/DiagnosticMessageFormat.h

@@ -0,0 +1,42 @@
+//===--- DiagnosticMessageFormat.h - YAML format for Diagnostic Messages ---*-
+//C++ -*-===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the YAML format for diagnostic messages.
+//
+//===----------------------------------------------------------------------===//
+
+#include "DiagnosticList.cpp"
+#include "llvm/Support/YAMLTraits.h"
+
+namespace llvm {
+namespace yaml {
+
+template <> struct ScalarEnumerationTraits<swift::DiagID> {
+  static void enumeration(IO &io, swift::DiagID &value) {
+#define DIAG(KIND, ID, Options, Text, Signature)                               \
+  io.enumCase(value, #ID, swift::DiagID::ID);
+#include "swift/AST/DiagnosticsAll.def"
+  }
+};
+
+template <> struct MappingTraits<swift::DiagnosticNode> {
+  static void mapping(IO &io, swift::DiagnosticNode &node) {
+    io.mapRequired("id", node.id);
+    io.mapRequired("msg", node.msg);
+  }
+};
+
+} // namespace yaml
+} // namespace llvm
+
+LLVM_YAML_IS_SEQUENCE_VECTOR(swift::DiagnosticNode)

include/swift/AST/Diagnostics/fr.yaml

@@ -0,0 +1,19 @@
+#===--- fr.yaml - Localized diagnostic messages for French ---*- YAML -*-===#
+#
+# This source file is part of the Swift.org open source project
+#
+# Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors
+# Licensed under Apache License v2.0 with Runtime Library Exception
+#
+# See https://swift.org/LICENSE.txt for license information
+# See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+#
+#===----------------------------------------------------------------------===#
+#
+# This file defines the diagnostic messages for French language.
+# Each diagnostic is described in the following format:
+# - id: <diagnostic-id>
+#   msg: <diagnostic-message>
+# 
+#===----------------------------------------------------------------------===#
+

include/swift/AST/diagnostics/en.yaml

@@ -0,0 +1,19 @@
+#===--- en.yaml - Localized diagnostic messages for English ---*- YAML -*-===#
+#
+# This source file is part of the Swift.org open source project
+#
+# Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors
+# Licensed under Apache License v2.0 with Runtime Library Exception
+#
+# See https://swift.org/LICENSE.txt for license information
+# See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+#
+#===----------------------------------------------------------------------===#
+#
+# This file defines the diagnostic messages for English language.
+# Each diagnostic is described in the following format:
+# - id: <diagnostic-id>
+#   msg: <diagnostic-message>
+# 
+#===----------------------------------------------------------------------===#
+

lib/AST/DiagnosticEngine.cpp

@@ -20,6 +20,7 @@
 #include "swift/AST/ASTPrinter.h"
 #include "swift/AST/Decl.h"
 #include "swift/AST/DiagnosticSuppression.h"
+#include "swift/AST/DiagnosticMessageFormat.h"
 #include "swift/AST/Module.h"
 #include "swift/AST/Pattern.h"
 #include "swift/AST/PrintOptions.h"
@@ -32,6 +33,8 @@
 #include "llvm/ADT/Twine.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Format.h"
+#include "llvm/Support/YAMLParser.h"
+#include "llvm/Support/YAMLTraits.h"
 #include "llvm/Support/raw_ostream.h"
 
 using namespace swift;
@@ -140,6 +143,52 @@ struct EducationalNotes {
 static constexpr EducationalNotes<LocalDiagID::NumDiags> _EducationalNotes = EducationalNotes<LocalDiagID::NumDiags>();
 static constexpr auto educationalNotes = _EducationalNotes.value;
 
+class LocalizationInput : public llvm::yaml::Input {
+  using Input::Input;
+
+  /// Read diagnostics in the YAML file iteratively
+  template <typename T, typename Context>
+  static typename std::enable_if<llvm::yaml::has_SequenceTraits<T>::value,
+                                 void>::type
+  readYAML(IO &io, T &Seq, bool, Context &Ctx) {
+    unsigned count = io.beginSequence();
+
+    // Resize Diags from YAML file to be the same size
+    // as diagnosticStrings from def files.
+    Seq.resize(LocalDiagID::NumDiags);
+    for (unsigned i = 0; i < count; ++i) {
+      void *SaveInfo;
+      if (io.preflightElement(i, SaveInfo)) {
+        DiagnosticNode current;
+        yamlize(io, current, true, Ctx);
+        io.postflightElement(SaveInfo);
+        // YAML file isn't guaranteed to have diagnostics in order of their
+        // declaration in `.def` files, to accommodate that we need to leave
+        // holes in diagnostic array for diagnostics which haven't yet been
+        // localized and for the ones that have `DiagnosticNode::id` 
+        // indicates their position.
+        Seq[static_cast<unsigned>(current.id)] = std::move(current.msg);
+      }
+    }
+    io.endSequence();
+  }
+
+  template <typename T>
+  inline friend
+      typename std::enable_if<llvm::yaml::has_SequenceTraits<T>::value,
+                              LocalizationInput &>::type
+      operator>>(LocalizationInput &yin, T &diagnostics) {
+    llvm::yaml::EmptyContext Ctx;
+    if (yin.setCurrentDocument()) {
+      // If YAML file's format doesn't match the current format in
+      // DiagnosticMessageFormat, will throw an error.
+      readYAML(yin, diagnostics, true, Ctx);
+    }
+
+    return yin;
+  }
+};
+
 DiagnosticState::DiagnosticState() {
   // Initialize our per-diagnostic state to default
   perDiagnosticBehavior.resize(LocalDiagID::NumDiags, Behavior::Unspecified);
@@ -311,6 +360,28 @@ void Diagnostic::addChildNote(Diagnostic &&D) {
   ChildNotes.push_back(std::move(D));
 }
 
+YAMLLocalizationProducer::YAMLLocalizationProducer(std::string locale,
+                                                   std::string path) {
+  llvm::SmallString<128> DiagnosticsFilePath(path);
+  llvm::sys::path::append(DiagnosticsFilePath, locale);
+  llvm::sys::path::replace_extension(DiagnosticsFilePath, ".yaml");
+  auto FileBufOrErr = llvm::MemoryBuffer::getFileOrSTDIN(DiagnosticsFilePath);
+  if (!FileBufOrErr)
+    llvm_unreachable("Failed to read yaml file");
+  llvm::MemoryBuffer *document = FileBufOrErr->get();
+  LocalizationInput yin(document->getBuffer());
+  yin >> diagnostics;
+}
+
+std::string
+YAMLLocalizationProducer::getMessageOr(DiagID id,
+                                       std::string defaultMessage) const {
+  std::string diagnosticMessage = diagnostics[(unsigned)id];
+  if (diagnosticMessage.empty())
+    return defaultMessage;
+  return diagnosticMessage;
+}
+
 bool DiagnosticEngine::isDiagnosticPointsToFirstBadToken(DiagID ID) const {
   return storedDiagnosticInfos[(unsigned) ID].pointsToFirstBadToken;
 }
@@ -1011,10 +1082,17 @@ void DiagnosticEngine::emitDiagnostic(const Diagnostic &diagnostic) {
 
 const char *DiagnosticEngine::diagnosticStringFor(const DiagID id,
                                                   bool printDiagnosticName) {
+  // TODO: Print diagnostic names from `localization`.
   if (printDiagnosticName) {
     return debugDiagnosticStrings[(unsigned)id];
   }
-  return diagnosticStrings[(unsigned)id];
+  auto defaultMessage = diagnosticStrings[(unsigned)id];
+  if (localization) {
+    std::string localizedMessage =
+        localization.get()->getMessageOr(id, defaultMessage);
+    return localizedMessage.c_str();
+  }
+  return defaultMessage;
 }
 
 const char *InFlightDiagnostic::fixItStringFor(const FixItID id) {