Skip to content
Permalink
Browse files

Add SHA1 hashing to FactsParse

Summary:
Watchman provides a field called `"content.sha1hex"`. When a file changes, Watchman will tell us the new SHA1 hash of that file without us having to open it and hash it ourselves. We can use this SHA1 hash as our key in Memcache, allowing servers to share information about the files they parsed.

But on a cache miss, we need to hash the file ourselves. The only way to be sure that our hash matches the facts we read from the file is to hash the same string we parsed, which means FactsParse has to perform the hashing and return the hash as part of its results.

Reviewed By: kmeht

Differential Revision: D13631325

fbshipit-source-id: f949a15bae20f959d29cd53da6ded2f2914dbb35
  • Loading branch information...
jthemphill authored and hhvm-bot committed Jan 15, 2019
1 parent 8cc0da0 commit 12aacb09c44c72374a87f96187f220b05470f8fd
@@ -5,5 +5,6 @@
core_kernel
hhbc
parser
parser_syntax)
parser_syntax
utils_hash)
(preprocess (pps ppx_deriving.std)))
@@ -92,11 +92,12 @@ let type_facts_to_json name tf =
members in
J.JSON_Object members

let facts_to_json md5 facts =
let facts_to_json ~md5 ~sha1 facts =
let md5sum0 =
"md5sum0", hex_number_to_json (String.sub md5 0 16) in
let md5sum1 =
"md5sum1", hex_number_to_json (String.sub md5 16 16) in
let sha1sum = ("sha1sum", J.JSON_String sha1) in
let type_facts_json =
let elements =
InvSMap.fold (fun name v acc -> (type_facts_to_json name v) :: acc)
@@ -111,6 +112,7 @@ let facts_to_json md5 facts =
J.JSON_Object [
md5sum0;
md5sum1;
sha1sum;
type_facts_json;
functions_json;
constants_json;
@@ -36,4 +36,4 @@ type facts = {

val empty: facts

val facts_to_json: string -> facts -> Hh_json.json
val facts_to_json: md5:string -> sha1:string -> facts -> Hh_json.json
@@ -282,4 +282,5 @@ let extract_as_json ~php5_compat_mode ~hhvm_compat_mode ~force_hh ~enable_xhp te
from_text php5_compat_mode hhvm_compat_mode force_hh enable_xhp text
|> Option.map ~f:(fun facts ->
let md5 = OpaqueDigest.to_hex @@ OpaqueDigest.string text in
facts_to_json md5 facts)
let sha1 = Sha1.digest text in
facts_to_json ~md5 ~sha1 facts)
@@ -1,10 +1,12 @@
array(3) {
[%s]=>
array(6) {
array(7) {
["md5sum0"]=>
int(%i)
["md5sum1"]=>
int(%i)
["sha1sum"]=>
string(40) "%s"
["types"]=>
array(13) {
[0]=>
@@ -266,11 +268,13 @@ array(3) {
}
}
[%s]=>
array(6) {
array(7) {
["md5sum0"]=>
int(%i)
["md5sum1"]=>
int(%i)
["sha1sum"]=>
string(40) "%s"
["types"]=>
array(13) {
[0]=>
@@ -524,11 +528,13 @@ array(3) {
}
}
[%s]=>
array(6) {
array(7) {
["md5sum0"]=>
int(%i)
["md5sum1"]=>
int(%i)
["sha1sum"]=>
string(40) "%s"
["types"]=>
array(13) {
[0]=>
@@ -1,10 +1,12 @@
dict(3) {
[%s]=>
dict(6) {
dict(7) {
["md5sum0"]=>
int(%i)
["md5sum1"]=>
int(%i)
["sha1sum"]=>
string(40) "%s"
["types"]=>
vec(13) {
dict(4) {
@@ -221,11 +223,13 @@ dict(3) {
}
}
[%s]=>
dict(6) {
dict(7) {
["md5sum0"]=>
int(%i)
["md5sum1"]=>
int(%i)
["sha1sum"]=>
string(40) "%s"
["types"]=>
vec(13) {
dict(4) {
@@ -438,11 +442,13 @@ dict(3) {
}
}
[%s]=>
dict(6) {
dict(7) {
["md5sum0"]=>
int(%i)
["md5sum1"]=>
int(%i)
["sha1sum"]=>
string(40) "%s"
["types"]=>
vec(13) {
dict(4) {
@@ -1,10 +1,12 @@
array(3) {
[%s]=>
array(6) {
array(7) {
["md5sum0"]=>
int(%i)
["md5sum1"]=>
int(%i)
["sha1sum"]=>
string(40) "%s"
["types"]=>
array(13) {
[0]=>
@@ -266,11 +268,13 @@ array(3) {
}
}
[%s]=>
array(6) {
array(7) {
["md5sum0"]=>
int(%i)
["md5sum1"]=>
int(%i)
["sha1sum"]=>
string(40) "%s"
["types"]=>
array(13) {
[0]=>
@@ -524,11 +528,13 @@ array(3) {
}
}
[%s]=>
array(6) {
array(7) {
["md5sum0"]=>
int(%i)
["md5sum1"]=>
int(%i)
["sha1sum"]=>
string(40) "%s"
["types"]=>
array(13) {
[0]=>

0 comments on commit 12aacb0

Please sign in to comment.
You can’t perform that action at this time.
You signed in with another tab or window. Reload to refresh your session. You signed out in another tab or window. Reload to refresh your session.