Skip to content

Commit 85ab374

Browse files
nkorostecopybara-github
authored andcommitted
Fix AarResourcesExtractor action to produce consistent zips
The `aar_native_libs_zip_creator.py` script was producing inconsistent output zips for the same input files due to changing the last modified date of the input files. This resulted in inconsistent cache keys and therefore poor build results. Closes #11970. PiperOrigin-RevId: 345541072
1 parent 39ca727 commit 85ab374

File tree

2 files changed

+72
-2
lines changed

2 files changed

+72
-2
lines changed

tools/android/aar_native_libs_zip_creator.py

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,16 @@ def CreateNativeLibsZip(aar, cpu, native_libs_zip):
6363
# Only replaces the first instance of jni, in case the AAR contains
6464
# something like /jni/x86/jni.so.
6565
new_filename = lib.replace("jni", "lib", 1)
66-
native_libs_zip.writestr(new_filename, aar.read(lib))
66+
# To guarantee reproducible zips we must specify a new zipinfo.
67+
# From writestr docs: "If its a name, the date and time is set to the
68+
# current date and time." which will break the HASH calculation and result
69+
# in a cache miss.
70+
old_zipinfo = aar.getinfo(lib)
71+
new_zipinfo = zipfile.ZipInfo(filename=new_filename)
72+
new_zipinfo.date_time = old_zipinfo.date_time
73+
new_zipinfo.compress_type = old_zipinfo.compress_type
74+
75+
native_libs_zip.writestr(new_zipinfo, aar.read(lib))
6776

6877

6978
def Main(input_aar_path, output_zip_path, cpu, input_aar_path_for_error_msg):

tools/android/aar_native_libs_zip_creator_test.py

Lines changed: 62 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,21 +14,38 @@
1414

1515
"""Tests for aar_native_libs_zip_creator."""
1616

17+
import hashlib
1718
import io
19+
import tempfile
20+
import time
1821
import unittest
1922
import zipfile
2023

2124
from tools.android import aar_native_libs_zip_creator
2225

2326

27+
def md5(buf):
28+
hash_md5 = hashlib.md5()
29+
hash_md5.update(buf)
30+
return hash_md5.hexdigest()
31+
32+
33+
def md5_from_file(fname):
34+
hash_md5 = hashlib.md5()
35+
with open(fname, "rb") as f:
36+
for chunk in iter(lambda: f.read(4096), b""):
37+
hash_md5.update(chunk)
38+
return hash_md5.hexdigest()
39+
40+
2441
class AarNativeLibsZipCreatorTest(unittest.TestCase):
2542
"""Unit tests for aar_native_libs_zip_creator.py."""
2643

2744
def testAarWithNoLibs(self):
2845
aar = zipfile.ZipFile(io.BytesIO(), "w")
2946
outzip = zipfile.ZipFile(io.BytesIO(), "w")
3047
aar_native_libs_zip_creator.CreateNativeLibsZip(aar, "x86", outzip)
31-
self.assertEquals([], outzip.namelist())
48+
self.assertEqual([], outzip.namelist())
3249

3350
def testAarWithMissingLibs(self):
3451
aar = zipfile.ZipFile(io.BytesIO(), "w")
@@ -48,6 +65,50 @@ def testAarWithAllLibs(self):
4865
self.assertIn("lib/x86/foo.so", outzip.namelist())
4966
self.assertNotIn("lib/armeabi/foo.so", outzip.namelist())
5067

68+
def testMultipleInvocationConsistency(self):
69+
input_aar = tempfile.NamedTemporaryFile(delete=False)
70+
aar = zipfile.ZipFile(input_aar.name, "w")
71+
aar.writestr(zipfile.ZipInfo(filename="jni/x86/foo.so"), "foo")
72+
aar.writestr(zipfile.ZipInfo(filename="jni/x86/bar.so"), "bar")
73+
aar.close()
74+
input_aar.close()
75+
# CreateNativeLibsZip expects a readonly file, this is not required but
76+
# more correct
77+
readonly_aar = zipfile.ZipFile(input_aar.name, "r")
78+
79+
outfile1 = tempfile.NamedTemporaryFile(delete=False)
80+
outzip1 = zipfile.ZipFile(outfile1.name, "w")
81+
aar_native_libs_zip_creator.CreateNativeLibsZip(readonly_aar, "x86",
82+
outzip1)
83+
outfile1.close()
84+
85+
# Must be more than 1 second because last modified date changes on second
86+
# basis
87+
time.sleep(2)
88+
89+
outfile2 = tempfile.NamedTemporaryFile(delete=False)
90+
outzip2 = zipfile.ZipFile(outfile2.name, "w")
91+
aar_native_libs_zip_creator.CreateNativeLibsZip(readonly_aar, "x86",
92+
outzip2)
93+
outfile2.close()
94+
95+
self.assertIn("lib/x86/foo.so", outzip1.namelist())
96+
self.assertIn("lib/x86/bar.so", outzip1.namelist())
97+
self.assertNotEqual(
98+
md5(outzip1.read("lib/x86/foo.so")),
99+
md5(outzip1.read("lib/x86/bar.so")))
100+
101+
self.assertIn("lib/x86/foo.so", outzip2.namelist())
102+
self.assertIn("lib/x86/bar.so", outzip2.namelist())
103+
self.assertNotEqual(
104+
md5(outzip1.read("lib/x86/foo.so")),
105+
md5(outzip1.read("lib/x86/bar.so")))
106+
107+
# The hash for the output zips must always match if the inputs match.
108+
# Otherwise, there will be a cache miss which will produce poort build
109+
# times.
110+
self.assertEqual(md5_from_file(outfile1.name), md5_from_file(outfile2.name))
111+
51112

52113
if __name__ == "__main__":
53114
unittest.main()

0 commit comments

Comments
 (0)