-
-
Notifications
You must be signed in to change notification settings - Fork 300
/
PathRef.scala
203 lines (180 loc) · 7.1 KB
/
PathRef.scala
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
package mill.api
import java.nio.{file => jnio}
import java.security.{DigestOutputStream, MessageDigest}
import java.util.concurrent.ConcurrentHashMap
import scala.util.{DynamicVariable, Using}
import upickle.default.{ReadWriter => RW}
/**
* A wrapper around `os.Path` that calculates it's hashcode based
* on the contents of the filesystem underneath it. Used to ensure filesystem
* changes can bust caches which are keyed off hashcodes.
*/
case class PathRef private (
path: os.Path,
quick: Boolean,
sig: Int,
revalidate: PathRef.Revalidate
) {
def recomputeSig(): Int = PathRef.apply(path, quick).sig
def validate(): Boolean = recomputeSig() == sig
/* Hide case class specific copy method. */
private def copy(
path: os.Path = path,
quick: Boolean = quick,
sig: Int = sig,
revalidate: PathRef.Revalidate = revalidate
): PathRef = PathRef(path, quick, sig, revalidate)
def withRevalidate(revalidate: PathRef.Revalidate): PathRef = copy(revalidate = revalidate)
def withRevalidateOnce: PathRef = copy(revalidate = PathRef.Revalidate.Once)
override def toString: String = {
val quick = if (this.quick) "qref:" else "ref:"
val valid = revalidate match {
case PathRef.Revalidate.Never => "v0:"
case PathRef.Revalidate.Once => "v1:"
case PathRef.Revalidate.Always => "vn:"
}
val sig = String.format("%08x", this.sig: Integer)
quick + valid + sig + ":" + path.toString()
}
}
object PathRef {
/**
* This class maintains a cache of already validated paths.
* It is threadsafe and meant to be shared between threads, e.g. in a ThreadLocal.
*/
class ValidatedPaths() {
private val map = new ConcurrentHashMap[Int, PathRef]()
/**
* Revalidates the given [[PathRef]], if required.
* It will only revalidate a [[PathRef]] if it's value for [[PathRef.revalidate]] says so and also considers the previously revalidated paths.
* @throws PathRefValidationException If a the [[PathRef]] needs revalidation which fails
*/
def revalidateIfNeededOrThrow(pathRef: PathRef): Unit = {
def mapKey(pr: PathRef): Int = (pr.path, pr.quick, pr.sig).hashCode()
pathRef.revalidate match {
case Revalidate.Never => // ok
case Revalidate.Once if map.contains(mapKey(pathRef)) => // ok
case Revalidate.Once | Revalidate.Always =>
val changedSig = PathRef.apply(pathRef.path, pathRef.quick).sig
if (pathRef.sig != changedSig) {
throw new PathRefValidationException(pathRef)
}
map.put(mapKey(pathRef), pathRef)
}
}
def clear(): Unit = map.clear()
}
private[mill] val validatedPaths: DynamicVariable[ValidatedPaths] =
new DynamicVariable[ValidatedPaths](new ValidatedPaths())
class PathRefValidationException(val pathRef: PathRef)
extends RuntimeException(s"Invalid path signature detected: ${pathRef}")
sealed trait Revalidate
object Revalidate {
case object Never extends Revalidate
case object Once extends Revalidate
case object Always extends Revalidate
}
def apply(path: os.Path, quick: Boolean, sig: Int, revalidate: Revalidate): PathRef =
new PathRef(path, quick, sig, revalidate)
/**
* Create a [[PathRef]] by recursively digesting the content of a given `path`.
*
* @param path The digested path.
* @param quick If `true` the digest is only based to some file attributes (like mtime and size).
* If `false` the digest is created of the files content.
* @return
*/
def apply(
path: os.Path,
quick: Boolean = false,
revalidate: Revalidate = Revalidate.Never
): PathRef = {
val basePath = path
val sig = {
val isPosix = path.wrapped.getFileSystem.supportedFileAttributeViews().contains("posix")
val digest = MessageDigest.getInstance("MD5")
val digestOut = new DigestOutputStream(DummyOutputStream, digest)
def updateWithInt(value: Int): Unit = {
digest.update((value >>> 24).toByte)
digest.update((value >>> 16).toByte)
digest.update((value >>> 8).toByte)
digest.update(value.toByte)
}
if (os.exists(path)) {
for (
(path, attrs) <-
os.walk.attrs(path, includeTarget = true, followLinks = true).sortBy(_._1.toString)
) {
val sub = path.subRelativeTo(basePath)
digest.update(sub.toString().getBytes())
if (!attrs.isDir) {
try {
if (isPosix) {
updateWithInt(os.perms(path, followLinks = false).value)
}
if (quick) {
val value = (attrs.mtime, attrs.size).hashCode()
updateWithInt(value)
} else if (jnio.Files.isReadable(path.toNIO)) {
val is =
try Some(os.read.inputStream(path))
catch {
case _: jnio.FileSystemException =>
// This is known to happen, when we try to digest a socket file.
// We ignore the content of this file for now, as we would do,
// when the file isn't readable.
// See https://github.com/com-lihaoyi/mill/issues/1875
None
}
is.foreach {
Using.resource(_) { is =>
StreamSupport.stream(is, digestOut)
}
}
}
} catch {
case e: java.nio.file.NoSuchFileException =>
// If file was deleted after we listed the folder but before we operate on it,
// `os.perms` or `os.read.inputStream` will crash. In that case, just do nothing,
// so next time we calculate the `PathRef` we'll get a different hash signature
// (either with the file missing, or with the file present) and invalidate any
// caches
}
}
}
}
java.util.Arrays.hashCode(digest.digest())
}
new PathRef(path, quick, sig, revalidate)
}
/**
* Default JSON formatter for [[PathRef]].
*/
implicit def jsonFormatter: RW[PathRef] = upickle.default.readwriter[String].bimap[PathRef](
p => p.toString(),
s => {
val Array(prefix, valid0, hex, pathString) = s.split(":", 4)
val path = os.Path(pathString)
val quick = prefix match {
case "qref" => true
case "ref" => false
}
val validOrig = valid0 match {
case "v0" => Revalidate.Never
case "v1" => Revalidate.Once
case "vn" => Revalidate.Always
}
// Parsing to a long and casting to an int is the only way to make
// round-trip handling of negative numbers work =(
val sig = java.lang.Long.parseLong(hex, 16).toInt
val pr = PathRef(path, quick, sig, revalidate = validOrig)
validatedPaths.value.revalidateIfNeededOrThrow(pr)
pr
}
)
// scalafix:off; we want to hide the unapply method
private def unapply(pathRef: PathRef): Option[(os.Path, Boolean, Int, Revalidate)] = {
Some((pathRef.path, pathRef.quick, pathRef.sig, pathRef.revalidate))
}
// scalalfix:on
}