ref: compilation module

Author: kkharji

Cargo.toml

@@ -4,17 +4,17 @@ version = "0.1.0"
 edition = "2021"
 
 [features]
-default    = [ ]
-xcodegen   = [ "async", "dirs" ]
-daemon     = [ "serial", "completion", "logging", "xcode", "async", "watcher", "proc", "xcodegen" ]
-lua        = [ "mlua" ]
-serial     = [ "serde", "serde_json", "serde_yaml" ]
-completion = [ "serial", "lazy_static", "regex" ]
-logging    = [ "tracing", "tracing-appender", "tracing-subscriber" ]
-xcode      = [ "serial" ]
-async      = [ "tokio", "async-trait" ]
-proc       = [ "libproc" ]
-watcher    = [ "dirs", "notify", "wax" ]
+default     = [ ]
+xcodegen    = [ "async", "dirs" ]
+daemon      = [ "serial", "compilation", "logging", "xcode", "async", "watcher", "proc", "xcodegen" ]
+lua         = [ "mlua" ]
+serial      = [ "serde", "serde_json", "serde_yaml" ]
+compilation = [ "serial", "lazy_static", "regex", "shell-words" ]
+logging     = [ "tracing", "tracing-appender", "tracing-subscriber" ]
+xcode       = [ ]
+async       = [ "tokio", "async-trait" ]
+proc        = [ "libproc" ]
+watcher     = [ "dirs", "notify", "wax" ]
 
 [lib]
 name = "xcodebase"
@@ -34,32 +34,36 @@ required-features = ["daemon"]
 
 # Error Handling
 anyhow             = "^1.0.42"
+# suffix-position pipeline
+tap = "1.0.1"
 
 # Serialization Feature
-serde              = { version = "1.0",    features = ["derive"], optional = true }
-serde_json         = { version = "1.0.79", optional = true }
-serde_yaml         = { version = "0.8.23", optional = true }
+serde                 = { version = "1.0",    features = ["derive"], optional = true }
+serde_json            = { version = "1.0.79", optional = true }
+serde_yaml            = { version = "0.8.23", optional = true }
 
 # Async Runtime Feature
-tokio              = { version = "1.17.0", features = ["full"], optional = true }
-async-trait        = { version = "0.1.52", optional = true }
+tokio                 = { version = "1.17.0", features = ["full"], optional = true }
+tokio-util            = { version = "0.7.1", features = ["codec"], optional = true }
+async-trait           = { version = "0.1.52", optional = true }
 
 # Logging Feature
-tracing            = { version = "0.1.32", optional = true }
-tracing-subscriber = { version = "0.3.9",  features = ["env-filter"], optional = true}
-tracing-appender   = { version = "0.2.1",  optional = true }
+tracing               = { version = "0.1.32", optional = true }
+tracing-subscriber    = { version = "0.3.9",  features = ["env-filter"], optional = true}
+tracing-appender      = { version = "0.2.1",  optional = true }
 
 # Filesystem watcher
-notify             = { version = "5.0.0-pre.13", optional = true }
-dirs               = { version = "4.0",   optional = true }
-wax                = { version = "0.4.0", optional = true }
+notify                = { version = "5.0.0-pre.13", optional = true }
+dirs                  = { version = "4.0",   optional = true }
+wax                   = { version = "0.4.0", optional = true }
+
+# Build Logs/Command Compilation feature
+regex                 = { version = "1.5",   optional = true }
+lazy_static           = { version = "1.4.0", optional = true }
+shell-words           = { version  = "1.1.0", optional = true }
+mlua                  = { version = "0.7.4", optional = true } 
 
-# Build logs Completion feature
-regex              = { version = "1.5",   optional = true }
-lazy_static        = { version = "1.4.0", optional = true }
-shell-words        = { version  = "1.1.0" }
-mlua               = { version = "0.7.4", optional = true } 
 
 # Proc feature: processes, processors, disks, components and networks
 # sysinfo = "0.23.5"
-libproc            = { version = "0.10.0", optional = true }
+libproc               = { version = "0.10.0", optional = true }

src/compile.rs

@@ -0,0 +1,73 @@
+mod command;
+pub use command::CompileCommand;
+
+use crate::util::regex::matches_compile_swift_sources;
+use anyhow::{Context, Result};
+use serde::Deserialize;
+use std::path::PathBuf;
+use tap::Pipe;
+
+// TODO: Support compiling commands for objective-c files
+// TODO: Test multiple module command compile
+
+#[derive(Debug, Deserialize)]
+pub struct CompileCommands(Vec<CompileCommand>);
+
+impl CompileCommands {
+    pub fn from_logs(lines: Vec<String>) -> Self {
+        // TODO: support index store
+        let mut _index_store_path = Vec::default();
+        let mut commands = vec![];
+        let mut cursor = 0;
+
+        for line in lines.iter() {
+            cursor += 1;
+            if !line.starts_with("===") {
+                continue;
+            }
+
+            if matches_compile_swift_sources(line) {
+                if let Some(command) = CompileCommand::swift_module(&lines, cursor) {
+                    if let Some(ref index_store_path) = command.index_store_path {
+                        _index_store_path.push(index_store_path.clone());
+                    }
+                    commands.push(command);
+                }
+            }
+        }
+
+        Self(commands)
+    }
+
+    pub fn from_file(path: &PathBuf) -> Result<Self> {
+        std::fs::read_to_string(path)?
+            .pipe_ref(|s| serde_json::from_str(s))
+            .context("Deserialize .compile")
+    }
+
+    /// Generate and write compile commands from build logs to directory
+    #[cfg(feature = "async")]
+    pub async fn update(dir: &PathBuf, build_log: Vec<String>) -> Result<()> {
+        tracing::info!("Updating .compile in {:?}", dir);
+        Self::from_logs(build_log)
+            .pipe(|cmd| serde_json::to_vec_pretty(&cmd.0))?
+            .pipe(|json| tokio::fs::write(dir.join(".compile"), json))
+            .await
+            .context("Write CompileCommands")
+    }
+}
+
+#[test]
+fn test() {
+    tokio::runtime::Runtime::new().unwrap().block_on(async {
+        use tap::Pipe;
+        tokio::fs::read_to_string("/Users/tami5/repos/swift/wordle/build.log")
+            .await
+            .unwrap()
+            .split("\n")
+            .map(|l| l.to_string())
+            .collect::<Vec<_>>()
+            .pipe(CompileCommands::from_logs)
+            .pipe(|v| println!("{:#?}", v));
+    });
+}

src/compile/command.rs

@@ -0,0 +1,113 @@
+// See https://clang.llvm.org/docs/JSONCompilationDatabase.html
+// See https://github.com/apple/sourcekit-lsp/blob/main/Sources/SKCore/CompilationDatabase.swift
+
+use serde::{Deserialize, Serialize};
+
+use crate::util::regex::matches_compile_swift_sources;
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct CompileCommand {
+    /// Module name
+    /// NOTE: not sure if this required
+    #[serde(
+        rename(serialize = "module_name"),
+        skip_serializing_if = "String::is_empty"
+    )]
+    pub name: String,
+
+    /// The path of the main file for the compilation, which may be relative to `directory`.
+    #[serde(skip_serializing_if = "String::is_empty")]
+    pub file: String,
+
+    /// The wroking directory for the compilation
+    pub directory: String,
+
+    /// The compile command, this is alias with commandLine or split form of command
+    pub command: String,
+
+    /// Source code files.
+    #[serde(rename(serialize = "fileLists"), skip_serializing_if = "Vec::is_empty")]
+    pub files: Vec<String>,
+
+    /// For SwiftFileList
+    #[serde(rename(serialize = "fileLists"))]
+    pub file_lists: Vec<String>,
+
+    /// The name of the build output
+    #[serde(skip_serializing_if = "String::is_empty")]
+    pub output: String,
+
+    /// Index store path. Kept for the caller to further process.
+    #[serde(skip)]
+    pub index_store_path: Option<String>,
+}
+
+impl CompileCommand {
+    pub fn can_parse(line: &String) -> bool {
+        matches_compile_swift_sources(line)
+    }
+
+    /// Parse starting from current line as swift module
+    /// Matching r"^CompileSwiftSources\s*"
+    pub fn swift_module(lines: &Vec<String>, cursor: usize) -> Option<CompileCommand> {
+        let directory = match lines.get(cursor) {
+            Some(s) => s.trim().replace("cd ", ""),
+            None => {
+                tracing::error!("Found COMPILE_SWIFT_MODULE_PATERN but no more lines");
+                return None;
+            }
+        };
+
+        let command = match lines.get(cursor) {
+            Some(s) => s.trim().to_string(),
+            None => {
+                tracing::error!("Found COMPILE_SWIFT_MODULE_PATERN but couldn't extract command");
+                return None;
+            }
+        };
+
+        let arguments = match shell_words::split(&command) {
+            Ok(l) => l,
+            Err(e) => {
+                tracing::error!("Fail to create swift module command {e}");
+                return None;
+            }
+        };
+
+        // NOTE: This is never changed
+        let file = String::default();
+        let output = String::default();
+        let mut name = String::default();
+        let mut files = Vec::default();
+        let mut file_lists = Vec::default();
+        let mut index_store_path = None;
+
+        for i in 0..arguments.len() {
+            let val = &arguments[i];
+            if val == "-module-name" {
+                name = arguments[i + 1].to_owned();
+            } else if val == "-index-store-path" {
+                index_store_path = Some(arguments[i + 1].to_owned());
+            } else if val.ends_with(".swift") {
+                files.push(val.to_owned());
+            } else if val.ends_with(".SwiftFileList") {
+                file_lists.push(val.replace("@", "").to_owned());
+            }
+        }
+
+        let command = Self {
+            directory,
+            command: arguments.join(" "),
+            name,
+            file,
+            output,
+            files,
+            file_lists,
+            index_store_path,
+        };
+
+        tracing::debug!("Got Swift commands for {}", command.directory);
+        tracing::trace!("{:#?}", command);
+        Some(command)
+    }
+}

src/lib.rs

@@ -9,6 +9,9 @@ pub use state::*;
 #[cfg(feature = "xcode")]
 pub mod xcode;
 
+#[cfg(feature = "compilation")]
+pub mod compile;
+
 #[cfg(feature = "xcodegen")]
 pub mod xcodegen;
 
@@ -24,3 +27,9 @@ pub use util::tracing::install_tracing;
 
 #[cfg(feature = "watcher")]
 pub use util::watch;
+
+#[cfg(feature = "server")]
+pub mod server;
+
+#[cfg(feature = "server")]
+pub use server::*;

src/util/mod.rs

@@ -2,6 +2,8 @@
 pub mod mlua;
 #[cfg(feature = "proc")]
 pub mod proc;
+#[cfg(feature = "regex")]
+pub mod regex;
 #[cfg(feature = "logging")]
 pub mod tracing;
 #[cfg(feature = "watcher")]

src/util/regex.rs

@@ -0,0 +1,18 @@
+use lazy_static::lazy_static;
+use regex::Regex;
+
+#[cfg(feature = "compilation")]
+lazy_static! {
+    static ref COMPILE_SWIFT_SOURCES: Regex = Regex::new(r"^CompileSwiftSources\s*").unwrap();
+    static ref COMPILE_SWIFT: Regex = Regex::new(r"^CompileSwift\s+ \w+\s+ \w+\s+ (.+)$").unwrap();
+}
+
+#[cfg(feature = "compilation")]
+pub fn matches_compile_swift_sources(text: &str) -> bool {
+    COMPILE_SWIFT_SOURCES.is_match(text)
+}
+
+#[cfg(feature = "compilation")]
+pub fn matches_compile_swift(text: &str) -> bool {
+    COMPILE_SWIFT.is_match(text)
+}

src/xcode/actions.rs renamed to src/xcode.rs

@@ -1,14 +1,12 @@
 use anyhow::Result;
+use serde_json::json;
 use std::ffi;
 use std::fmt::Debug;
 use std::path::{Path, PathBuf};
 use std::process::{ExitStatus, Stdio};
-use tokio::process::Command;
-
-use super::compilation::Compiliation;
-use serde_json::json;
 use tokio::fs;
 use tokio::io::AsyncWriteExt;
+use tokio::process::Command;
 
 // https://github.com/Gordon-F/cargo-xcodebuild
 /// run xcodebuild build with extra arguments
@@ -65,14 +63,6 @@ where
         .await
 }
 
-#[cfg(feature = "serial")]
-pub async fn update_compiled_commands(root: &PathBuf, build_log: Vec<String>) -> Result<()> {
-    let path = root.join(".compile");
-    tracing::info!("Updating {:?}", path);
-    fs::write(path, Compiliation::new(build_log).to_json()?).await?;
-    Ok(())
-}
-
 pub async fn ensure_server_config_file(root: &PathBuf) -> Result<()> {
     let path = root.join("buildServer.json");
     if fs::File::open(&path).await.is_ok() {
@@ -85,7 +75,7 @@ pub async fn ensure_server_config_file(root: &PathBuf) -> Result<()> {
     let config = json! ({
         "name": "XcodeBase Server",
         // FIXME: Point to user xcode-build-server
-        "argv": ["/Users/tami5/repos/neovim/XcodeBase.nvim/target/debug/xcodebase-build-server"],
+        "argv": ["/Users/tami5/repos/neovim/XcodeBase.nvim/target/debug/xcodebase-server"],
         "version": "0.1",
         "bspVersion": "0.2",
         "languages": [
@@ -94,7 +84,7 @@ pub async fn ensure_server_config_file(root: &PathBuf) -> Result<()> {
             "objective-cpp",
             "c",
             "cpp"
-        ],
+        ]
     });
 
     file.write_all(config.to_string().as_ref()).await?;