Improve resource sync performance (#8674)

* Improve resource sync performance Fixes #8595 * Bring back the misspelling
defold · Mar 25, 2024 · 2d6a359 · 2d6a359
1 parent f4ead9d
commit 2d6a359
Show file tree

Hide file tree

Showing 2 changed files with 43 additions and 22 deletions.
diff --git a/editor/src/clj/editor/resource.clj b/editor/src/clj/editor/resource.clj
@@ -108,29 +108,36 @@
   (atom {}))
 
 ;; The same logic implemented in Project.java.
-;; If you change something here, plese change it there as well
+;; If you change something here, please change it there as well
 ;; Search for excluedFilesAndFoldersEntries.
 ;; root -> pred if project path (string starting with /) is ignored
-(defn- defignore-pred [^File root]
+(defn defignore-pred [^File root]
   (let [defignore-file (io/file root ".defignore")
         defignore-path (.getCanonicalPath defignore-file)
         latest-mtime (.lastModified defignore-file)
         {:keys [mtime pred]} (get @defignore-cache defignore-path)]
     (if (= mtime latest-mtime)
       pred
       (let [pred (if (.isFile defignore-file)
-                   (let [prefixes (into
-                                    #{}
-                                    (filter #(string/starts-with? % "/"))
-                                    (string/split-lines (slurp defignore-file)))]
+                   (let [prefixes (into []
+                                        (comp
+                                          (filter #(string/starts-with? % "/"))
+                                          (distinct))
+                                        (string/split-lines (slurp defignore-file)))]
                      (fn ignored-path? [path]
                        (boolean (some #(string/starts-with? path %) prefixes))))
                    (constantly false))]
         (swap! defignore-cache assoc defignore-path {:mtime latest-mtime :pred pred})
         pred))))
 
+(def ^:dynamic *defignore-pred* nil)
+
+(defmacro with-defignore-pred [root-expr & body]
+  `(binding [*defignore-pred* (defignore-pred ~root-expr)]
+     ~@body))
+
 (defn ignored-project-path? [^File root proj-path]
-  ((defignore-pred root) proj-path))
+  ((or *defignore-pred* (defignore-pred root)) proj-path))
 
 ;; Note! Used to keep a file here instead of path parts, but on
 ;; Windows (File. "test") equals (File. "Test") which broke

diff --git a/editor/src/clj/editor/resource_watch.clj b/editor/src/clj/editor/resource_watch.clj
@@ -57,11 +57,12 @@
         {resources :tree crc :crc} (load-library-zip workspace file)
         flat-resources (resource/resource-list-seq resources)]
     {:resources resources
-     :status-map (into {} (map (fn [resource]
-                                 (let [path (resource/proj-path resource)
-                                       version (str zip-file-version ":" (crc path))]
-                                   [path {:version version :source :library :library uri-string}]))
-                               flat-resources))}))
+     :status-map (into {}
+                       (map (fn [resource]
+                              (let [path (resource/proj-path resource)
+                                    version (str zip-file-version ":" (crc path))]
+                                [path {:version version :source :library :library uri-string}])))
+                       flat-resources)}))
 
 (defn- update-library-snapshot-cache
   [library-snapshot-cache workspace lib-states]
@@ -93,7 +94,7 @@
         resources (:tree (resource/load-zip-resources workspace builtins-zip-file))
         flat-resources (resource/resource-list-seq resources)]
     {:resources resources
-     :status-map (into {} (map (juxt resource/proj-path (constantly {:version :constant :source :builtins})) flat-resources))}))
+     :status-map (into {} (map (juxt resource/proj-path (constantly {:version :constant :source :builtins}))) flat-resources)}))
 
 (def reserved-proj-paths #{"/builtins" "/build" "/.internal" "/.git"})
 
@@ -152,10 +153,22 @@
    :status-map nil
    :errors nil})
 
+(defn map-intersection
+  "Given 2 maps, return a vector of keys present in both maps"
+  [m1 m2]
+  (if (< (count m2) (count m1))
+    (recur m2 m1)
+    (persistent!
+      (reduce-kv
+        (fn [acc k _]
+          (cond-> acc (contains? m2 k) (conj! k)))
+        (transient [])
+        m1))))
+
 (defn- combine-snapshots [snapshots]
   (reduce
    (fn [result snapshot]
-     (if-let [collisions (seq (clojure.set/intersection (resource-paths result) (resource-paths snapshot)))]
+     (if-let [collisions (not-empty (map-intersection (:status-map result) (:status-map snapshot)))]
        (update result :errors conj {:type :collision :collisions (select-keys (:status-map snapshot) collisions)})
        (-> result
            (update :resources concat (:resources snapshot))
@@ -182,16 +195,17 @@
   (update snapshot :status-map into file-resource-status-map-entries))
 
 (defn make-snapshot-info [workspace project-directory library-uris snapshot-cache]
-  (let [lib-states (library/current-library-state project-directory library-uris)
-        new-library-snapshot-cache (update-library-snapshot-cache snapshot-cache workspace lib-states)]
-    {:snapshot (combine-snapshots (list* (make-builtins-snapshot workspace)
-                                         (make-directory-snapshot workspace project-directory)
-                                         (make-debugger-snapshot workspace)
-                                         (make-library-snapshots new-library-snapshot-cache lib-states)))
-     :snapshot-cache new-library-snapshot-cache}))
+  (resource/with-defignore-pred project-directory
+    (let [lib-states (library/current-library-state project-directory library-uris)
+          new-library-snapshot-cache (update-library-snapshot-cache snapshot-cache workspace lib-states)]
+      {:snapshot (combine-snapshots (list* (make-builtins-snapshot workspace)
+                                           (make-directory-snapshot workspace project-directory)
+                                           (make-debugger-snapshot workspace)
+                                           (make-library-snapshots new-library-snapshot-cache lib-states)))
+       :snapshot-cache new-library-snapshot-cache})))
 
 (defn make-resource-map [snapshot]
-  (into {} (map (juxt resource/proj-path identity) (resource/resource-list-seq (:resources snapshot)))))
+  (into {} (map (juxt resource/proj-path identity)) (resource/resource-list-seq (:resources snapshot))))
 
 (defn- resource-status [snapshot path]
   (get-in snapshot [:status-map path]))