From 50fca519a40035830830b04aa6ea94230bb7eef6 Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Thu, 14 Apr 2022 18:03:33 -0400 Subject: [PATCH 01/19] Remove the patch to enable Snappy support --- cmake/external/firestore.cmake | 63 +-- cmake/external/firestore_patch.py | 119 +++++ cmake/external/firestore_patch_test.py | 162 ++++++ cmake/external/firestore_snappy.patch.txt | 619 ---------------------- 4 files changed, 296 insertions(+), 667 deletions(-) create mode 100644 cmake/external/firestore_patch.py create mode 100644 cmake/external/firestore_patch_test.py delete mode 100644 cmake/external/firestore_snappy.patch.txt diff --git a/cmake/external/firestore.cmake b/cmake/external/firestore.cmake index d94c6681e3..af34fe9a0e 100644 --- a/cmake/external/firestore.cmake +++ b/cmake/external/firestore.cmake @@ -13,59 +13,26 @@ # limitations under the License. include(ExternalProject) +include(FindPythonInterp) if(TARGET firestore) return() endif() -function(GetReleasedDep version) - message("Getting released firebase-ios-sdk @ ${version}") - ExternalProject_Add( - firestore +ExternalProject_Add( + firestore - DOWNLOAD_DIR ${FIREBASE_DOWNLOAD_DIR} - URL https://github.com/firebase/firebase-ios-sdk/archive/${version}.tar.gz + DOWNLOAD_DIR ${FIREBASE_DOWNLOAD_DIR} + GIT_REPOSITORY https://github.com/firebase/firebase-ios-sdk + GIT_TAG 582b384c99a5dd24331161d436fdb6fd088fa833 + GIT_SHALLOW ON - PREFIX ${PROJECT_BINARY_DIR} - - CONFIGURE_COMMAND "" - BUILD_COMMAND "" - INSTALL_COMMAND "" - TEST_COMMAND "" - PATCH_COMMAND patch -Np1 -i ${CMAKE_CURRENT_LIST_DIR}/firestore_snappy.patch.txt - HTTP_HEADER "${EXTERNAL_PROJECT_HTTP_HEADER}" - ) -endfunction() - -function(GetTag t) - message("Getting firebase-ios-sdk from ${t}") - ExternalProject_Add( - firestore - - DOWNLOAD_DIR ${FIREBASE_DOWNLOAD_DIR} - GIT_REPOSITORY "https://github.com/firebase/firebase-ios-sdk.git" - GIT_TAG ${t} - GIT_SHALLOW "ON" - - PREFIX ${PROJECT_BINARY_DIR} - - CONFIGURE_COMMAND "" - BUILD_COMMAND "" - INSTALL_COMMAND "" - TEST_COMMAND "" - PATCH_COMMAND patch -Np1 -i ${CMAKE_CURRENT_LIST_DIR}/firestore_snappy.patch.txt - HTTP_HEADER "${EXTERNAL_PROJECT_HTTP_HEADER}" - ) -endfunction() - -if((NOT FIRESTORE_DEP_SOURCE) OR (FIRESTORE_DEP_SOURCE STREQUAL "RELEASED")) - # Get from released dependency by default - GetReleasedDep("CocoaPods-8.12.1") -else() - if(FIRESTORE_DEP_SOURCE STREQUAL "TIP") - GetTag("origin/master") - else() - GetTag(${FIRESTORE_DEP_SOURCE}) - endif() -endif() + PREFIX ${PROJECT_BINARY_DIR} + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + TEST_COMMAND "" + PATCH_COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_LIST_DIR}/firestore_patch.py --leveldb-version-from ${CMAKE_CURRENT_LIST_DIR}/leveldb.cmake + HTTP_HEADER "${EXTERNAL_PROJECT_HTTP_HEADER}" +) diff --git a/cmake/external/firestore_patch.py b/cmake/external/firestore_patch.py new file mode 100644 index 0000000000..f25e46e1da --- /dev/null +++ b/cmake/external/firestore_patch.py @@ -0,0 +1,119 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Modify the version in leveldb.cmake from the Firebase iOS SDK to match the +version from this C++ SDK. +""" + +import argparse +import dataclasses +import os +import pathlib +import re +from typing import Iterable, Sequence + + +VERSION_PATTERN = r"\s*set\s*\(\s*version\s+([^)\s]+)\s*\)\s*" +VERSION_EXPR = re.compile(VERSION_PATTERN, re.IGNORECASE) + + +def main() -> None: + args = parse_args() + leveldb_version = load_leveldb_version(args.leveldb_cmake_src_file) + set_leveldb_version(args.leveldb_cmake_dest_file, leveldb_version) + + +@dataclasses.dataclass(frozen=True) +class ParsedArgs: + leveldb_cmake_src_file: pathlib.Path + leveldb_cmake_dest_file: pathlib.Path + + +def parse_args() -> ParsedArgs: + arg_parser = argparse.ArgumentParser() + arg_parser.add_argument("--leveldb-version-from", required=True) + arg_parser.add_argument("--leveldb-version-to") + + parsed_args = arg_parser.parse_args() + + leveldb_cmake_src_file = pathlib.Path(parsed_args.leveldb_version_from) + + if parsed_args.leveldb_version_to: + leveldb_cmake_dest_file = pathlib.Path(parsed_args.leveldb_version_to) + else: + leveldb_cmake_dest_file = pathlib.Path.cwd() \ + / "cmake" / "external" / "leveldb.cmake" + + return ParsedArgs( + leveldb_cmake_src_file=leveldb_cmake_src_file, + leveldb_cmake_dest_file=leveldb_cmake_dest_file, + ) + + +def load_leveldb_version(cmake_file: pathlib.Path) -> str: + version = None + version_line = None + version_line_number = None + + with cmake_file.open("rt", encoding="utf8") as f: + for (line_number, line) in enumerate(f, start=1): + match = VERSION_EXPR.fullmatch(line) + if match: + if version is not None: + raise LevelDbVersionLineError( + f"Multiple lines matching regex {VERSION_EXPR.pattern} found in " + f"{cmake_file}: line {version_line_number}, {version_line.strip()} " + f"and line {line_number}, {line.strip()}") + version = match.group(1).strip() + version_line = line + version_line_number = line_number + + if version is None: + raise LevelDbVersionLineError( + f"No line matching regex {VERSION_EXPR.pattern} found in {cmake_file}") + + return version + + +def set_leveldb_version(cmake_file: pathlib.Path, version: str) -> str: + with cmake_file.open("rt", encoding="utf8") as f: + lines = list(f) + + version_lines_found = [] + for (i, line) in enumerate(lines): + match = VERSION_EXPR.fullmatch(line) + if match: + lines[i] = line[:match.start(1)] + version + line[match.end(1):] + version_lines_found.append(i) + + if len(version_lines_found) == 0: + raise LevelDbVersionLineError( + f"No line matching regex {VERSION_EXPR.pattern} found in {cmake_file}") + elif len(version_lines_found) > 1: + raise LevelDbVersionLineError( + f"Multiple lines matching regex {VERSION_EXPR.pattern} found in " + f"{cmake_file}: {', '.join(str(i+1) for i in version_lines_found)}") + + with cmake_file.open("wt", encoding="utf8") as f: + f.writelines(lines) + + +class LevelDbVersionLineError(Exception): + pass + + + +if __name__ == "__main__": + main() diff --git a/cmake/external/firestore_patch_test.py b/cmake/external/firestore_patch_test.py new file mode 100644 index 0000000000..3b289259f3 --- /dev/null +++ b/cmake/external/firestore_patch_test.py @@ -0,0 +1,162 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import firestore_patch +import pathlib +import shutil +import tempfile +import unittest + + +class LoadLevelDbVersionTest(unittest.TestCase): + + def setUp(self): + super().setUp() + temp_dir = pathlib.Path(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, temp_dir) + self.temp_file = temp_dir / "temp_file.txt" + + def test_loads_correct_version(self): + with self.temp_file.open("wt", encoding="utf8") as f: + print("blah1", file=f) + print("set(version 1.23)", file=f) + print("blah2", file=f) + + version = firestore_patch.load_leveldb_version(self.temp_file) + + self.assertEqual(version, "1.23") + + def test_ignores_whitespace(self): + with self.temp_file.open("wt", encoding="utf8") as f: + print(" set ( version 1.23 ) ", file=f) + + version = firestore_patch.load_leveldb_version(self.temp_file) + + self.assertEqual(version, "1.23") + + def test_case_insensitive(self): + with self.temp_file.open("wt", encoding="utf8") as f: + print("SeT(vErSiOn 1.23)", file=f) + + version = firestore_patch.load_leveldb_version(self.temp_file) + + self.assertEqual(version, "1.23") + + def test_version_not_found_raises_error(self): + with self.temp_file.open("wt", encoding="utf8") as f: + print("aaa", file=f) + print("bbb", file=f) + + with self.assertRaises(firestore_patch.LevelDbVersionLineError) as cm: + firestore_patch.load_leveldb_version(self.temp_file) + + self.assertIn("no line matching", str(cm.exception).lower()) + self.assertIn(str(self.temp_file), str(cm.exception)) + + def test_multiple_version_lines_found_raises_error(self): + with self.temp_file.open("wt", encoding="utf8") as f: + for i in range(100): + print(f"line {i+1}", file=f) + print("set(version aaa)", file=f) + print("set(version bbb)", file=f) + + with self.assertRaises(firestore_patch.LevelDbVersionLineError) as cm: + firestore_patch.load_leveldb_version(self.temp_file) + + self.assertIn("multiple lines matching", str(cm.exception).lower()) + self.assertIn(str(self.temp_file), str(cm.exception)) + self.assertIn("line 101", str(cm.exception)) + self.assertIn("line 102", str(cm.exception)) + self.assertIn("aaa", str(cm.exception)) + self.assertIn("bbb", str(cm.exception)) + + +class SetLevelDbVersionTest(unittest.TestCase): + + def setUp(self): + super().setUp() + temp_dir = pathlib.Path(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, temp_dir) + self.temp_file = temp_dir / "temp_file.txt" + + def test_saves_correct_version(self): + with self.temp_file.open("wt", encoding="utf8") as f: + print("set(version asdfasdf)", file=f) + + firestore_patch.set_leveldb_version(self.temp_file, "1.2.3") + + new_version = firestore_patch.load_leveldb_version(self.temp_file) + self.assertEqual(new_version, "1.2.3") + + def test_case_insensitivity(self): + with self.temp_file.open("wt", encoding="utf8") as f: + print("sEt(vErSiOn asdfasdf)", file=f) + + firestore_patch.set_leveldb_version(self.temp_file, "1.2.3") + + new_version = firestore_patch.load_leveldb_version(self.temp_file) + self.assertEqual(new_version, "1.2.3") + + def test_leaves_whitespace_alone(self): + with self.temp_file.open("wt", encoding="utf8") as f: + print(" set ( version 1.2.3.4 ) ", file=f) + temp_file_contents = self.temp_file.read_text(encoding="utf8") + + firestore_patch.set_leveldb_version(self.temp_file, "a.b.c.d") + + temp_file_contents_after = self.temp_file.read_text(encoding="utf8") + expected_temp_file_contents = temp_file_contents.replace("1.2.3.4", "a.b.c.d") + self.assertEqual(temp_file_contents_after, expected_temp_file_contents) + + def test_does_not_touch_other_lines(self): + with self.temp_file.open("wt", encoding="utf8") as f: + print("blah1", file=f) + print("set(version 1.2.3.4)", file=f) + print("blah2", file=f) + temp_file_contents = self.temp_file.read_text(encoding="utf8") + + firestore_patch.set_leveldb_version(self.temp_file, "a.b.c.d") + + temp_file_contents_after = self.temp_file.read_text(encoding="utf8") + expected_temp_file_contents = temp_file_contents.replace("1.2.3.4", "a.b.c.d") + self.assertEqual(temp_file_contents_after, expected_temp_file_contents) + + def test_version_not_found_raises_error(self): + with self.temp_file.open("wt", encoding="utf8") as f: + print("aaa", file=f) + print("bbb", file=f) + + with self.assertRaises(firestore_patch.LevelDbVersionLineError) as cm: + firestore_patch.set_leveldb_version(self.temp_file, "a.b.c") + + self.assertIn("no line matching", str(cm.exception).lower()) + self.assertIn(str(self.temp_file), str(cm.exception)) + + def test_multiple_version_lines_found_raises_error(self): + with self.temp_file.open("wt", encoding="utf8") as f: + for i in range(100): + print(f"line {i+1}", file=f) + print("set(version aaa)", file=f) + print("set(version bbb)", file=f) + + with self.assertRaises(firestore_patch.LevelDbVersionLineError) as cm: + firestore_patch.set_leveldb_version(self.temp_file, "a.b.c") + + self.assertIn("multiple lines matching", str(cm.exception).lower()) + self.assertIn(str(self.temp_file), str(cm.exception)) + self.assertIn("101, 102", str(cm.exception)) + + +if __name__ == "__main__": + unittest.main() diff --git a/cmake/external/firestore_snappy.patch.txt b/cmake/external/firestore_snappy.patch.txt deleted file mode 100644 index 5a50404302..0000000000 --- a/cmake/external/firestore_snappy.patch.txt +++ /dev/null @@ -1,619 +0,0 @@ -diff --git a/CMakeLists.txt b/CMakeLists.txt -index 29458bf13..7be37691d 100644 ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -227,6 +227,12 @@ if(NOT ZLIB_FOUND) - endif() - - -+# Snappy -+set(SNAPPY_BUILD_TESTS OFF CACHE BOOL "Firestore disabled") -+set(SNAPPY_BUILD_BENCHMARKS OFF CACHE BOOL "Firestore disabled") -+add_external_subdirectory(snappy) -+firebase_ios_add_alias(Snappy::Snappy snappy) -+ - # LevelDB - set(LEVELDB_BUILD_TESTS OFF CACHE BOOL "Firestore disabled") - set(LEVELDB_BUILD_BENCHMARKS OFF CACHE BOOL "Firestore disabled") -diff --git a/cmake/external/CMakeLists.txt b/cmake/external/CMakeLists.txt -index 2179633a8..c1de37b6d 100644 ---- a/cmake/external/CMakeLists.txt -+++ b/cmake/external/CMakeLists.txt -@@ -30,6 +30,7 @@ include(c-ares) - include(googletest) - include(GoogleUtilities) - include(grpc) -+include(snappy) - include(leveldb) - include(libfuzzer) - include(nanopb) -diff --git a/cmake/external/leveldb.cmake b/cmake/external/leveldb.cmake -index b71a77535..2556d7041 100644 ---- a/cmake/external/leveldb.cmake -+++ b/cmake/external/leveldb.cmake -@@ -13,20 +13,27 @@ - # limitations under the License. - - include(ExternalProject) -+include(FindPythonInterp) - - if(TARGET leveldb) - return() - endif() - --set(version 1.22) -+set(version 1.23) -+ -+ExternalProject_Get_property(snappy SOURCE_DIR) -+set(snappy_source_dir "${SOURCE_DIR}") -+ExternalProject_Get_property(snappy BINARY_DIR) -+set(snappy_binary_dir "${BINARY_DIR}") - - ExternalProject_Add( - leveldb - -+ DEPENDS snappy -+ - DOWNLOAD_DIR ${FIREBASE_DOWNLOAD_DIR} - DOWNLOAD_NAME leveldb-${version}.tar.gz - URL https://github.com/google/leveldb/archive/${version}.tar.gz -- URL_HASH SHA256=55423cac9e3306f4a9502c738a001e4a339d1a38ffbee7572d4a07d5d63949b2 - - PREFIX ${PROJECT_BINARY_DIR} - -@@ -34,6 +41,7 @@ ExternalProject_Add( - BUILD_COMMAND "" - INSTALL_COMMAND "" - TEST_COMMAND "" -+ PATCH_COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_LIST_DIR}/leveldb_patch.py --snappy-source-dir ${snappy_source_dir} --snappy-binary-dir ${snappy_binary_dir} - - HTTP_HEADER "${EXTERNAL_PROJECT_HTTP_HEADER}" - ) -diff --git a/cmake/external/leveldb_patch.py b/cmake/external/leveldb_patch.py -new file mode 100644 -index 000000000..51a62d54a ---- /dev/null -+++ b/cmake/external/leveldb_patch.py -@@ -0,0 +1,144 @@ -+# Copyright 2022 Google LLC -+# -+# Licensed under the Apache License, Version 2.0 (the "License"); -+# you may not use this file except in compliance with the License. -+# You may obtain a copy of the License at -+# -+# http://www.apache.org/licenses/LICENSE-2.0 -+# -+# Unless required by applicable law or agreed to in writing, software -+# distributed under the License is distributed on an "AS IS" BASIS, -+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -+# See the License for the specific language governing permissions and -+# limitations under the License. -+ -+""" -+Modify the CMakeLists.txt from LevelDb to staticly link Snappy compression -+support. -+""" -+ -+import argparse -+import dataclasses -+import os -+import pathlib -+from typing import Iterable, Sequence -+ -+ -+def main() -> None: -+ arg_parser = argparse.ArgumentParser() -+ arg_parser.add_argument("--snappy-source-dir", required=True) -+ arg_parser.add_argument("--snappy-binary-dir", required=True) -+ parsed_args = arg_parser.parse_args() -+ del arg_parser -+ snappy_source_dir = pathlib.Path(parsed_args.snappy_source_dir) -+ snappy_binary_dir = pathlib.Path(parsed_args.snappy_binary_dir) -+ del parsed_args -+ -+ cmakelists_txt_file = pathlib.Path("CMakeLists.txt") -+ with cmakelists_txt_file.open("rt", encoding="utf8") as f: -+ lines = tuple(f) -+ -+ patcher = CMakeListsPatcher( -+ lines, -+ os.path.abspath(__file__), -+ snappy_source_dir, -+ snappy_binary_dir, -+ ) -+ -+ patched_lines = tuple(patcher.patch()) -+ -+ with cmakelists_txt_file.open("wt", encoding="utf8") as f: -+ f.writelines(patched_lines) -+ -+ -+@dataclasses.dataclass(frozen=True) -+class LineComponents: -+ full: str -+ indent: str -+ line: str -+ eol: str -+ -+ -+class CMakeListsPatcher: -+ -+ SNAPPY_DETECT_LINE = \ -+ """check_library_exists(snappy snappy_compress "" HAVE_SNAPPY)""" -+ SNAPPY_INCLUDE_LINE = \ -+ "target_include_directories(leveldb" -+ SNAPPY_LINK_LINE = \ -+ "target_link_libraries(leveldb snappy)" -+ -+ def __init__( -+ self, -+ lines: Sequence[str], -+ script_name: str, -+ snappy_source_dir: pathlib.Path, -+ snappy_binary_dir: pathlib.Path) -> None: -+ self.i = 0 -+ self.lines = lines -+ self.script_name = script_name -+ self.snappy_source_dir_str = snappy_source_dir.as_posix() -+ self.snappy_binary_dir_str = snappy_binary_dir.as_posix() -+ -+ def patch(self) -> Iterable[str]: -+ while self.i < len(self.lines): -+ full_line = self.lines[self.i] -+ line = self._split_line(full_line) -+ self.i += 1 -+ -+ if line.line == self.SNAPPY_DETECT_LINE: -+ yield from self._on_snappy_detect_line(line) -+ elif line.line == self.SNAPPY_INCLUDE_LINE: -+ yield full_line -+ yield from self._on_leveldb_include_start() -+ elif line.line == self.SNAPPY_LINK_LINE: -+ yield from self._on_leveldb_snappy_link_line(line) -+ else: -+ yield full_line -+ -+ def _begin_mod_line(self, mod_name: str) -> str: -+ return f"# BEGIN: {mod_name} modification by {self.script_name}" -+ -+ def _end_mod_line(self, mod_name: str) -> str: -+ return f"# END: {mod_name} modification by {self.script_name}" -+ -+ def _on_snappy_detect_line(self, line: LineComponents) -> Iterable[str]: -+ yield self._begin_mod_line("snappy_detect_line") + line.eol -+ yield line.indent + "# " + line.line + line.eol -+ yield line.indent + """set(HAVE_SNAPPY ON CACHE BOOL "")""" + line.eol -+ yield self._end_mod_line("snappy_detect_line") + line.eol -+ -+ def _on_leveldb_include_start(self) -> Iterable[str]: -+ line1 = self._split_line(self.lines[self.i]) -+ line2 = self._split_line(self.lines[self.i + 1]) -+ begin_mod_line = self._begin_mod_line("leveldb_include_start") -+ -+ if line1.line == begin_mod_line: -+ return -+ -+ yield begin_mod_line + line1.eol -+ yield line1.indent + "PRIVATE" + line1.eol -+ yield line2.indent + self.snappy_source_dir_str + line2.eol -+ yield line2.indent + self.snappy_binary_dir_str + line2.eol -+ yield self._end_mod_line("leveldb_include_start") + line1.eol -+ -+ def _on_leveldb_snappy_link_line(self, line: LineComponents) -> Iterable[str]: -+ yield self._begin_mod_line("leveldb_snappy_link_line") + line.eol -+ yield line.indent + "# " + line.line + line.eol -+ yield line.indent + f"target_link_libraries(leveldb Snappy::Snappy)" + line.eol -+ yield self._end_mod_line("leveldb_snappy_link_line") + line.eol -+ -+ def _split_line(self, line: str) -> LineComponents: -+ line_rstripped = line.rstrip() -+ eol = line[len(line_rstripped):] -+ line_stripped = line_rstripped.strip() -+ indent = line_rstripped[:len(line_rstripped) - len(line_stripped)] -+ return LineComponents(full=line, indent=indent, line=line_stripped, eol=eol) -+ -+ -+class LeveDbPatchException(Exception): -+ pass -+ -+ -+if __name__ == "__main__": -+ main() -diff --git a/cmake/external/leveldb_patch_test.py b/cmake/external/leveldb_patch_test.py -new file mode 100644 -index 000000000..b1d62526b ---- /dev/null -+++ b/cmake/external/leveldb_patch_test.py -@@ -0,0 +1,328 @@ -+# Copyright 2022 Google LLC -+# -+# Licensed under the Apache License, Version 2.0 (the "License"); -+# you may not use this file except in compliance with the License. -+# You may obtain a copy of the License at -+# -+# http://www.apache.org/licenses/LICENSE-2.0 -+# -+# Unless required by applicable law or agreed to in writing, software -+# distributed under the License is distributed on an "AS IS" BASIS, -+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -+# See the License for the specific language governing permissions and -+# limitations under the License. -+ -+import leveldb_patch -+import pathlib -+import unittest -+ -+ -+class CMakeListsPatcherTest(unittest.TestCase): -+ -+ def setUp(self): -+ super().setUp() -+ self.sample_snappy_source_dir = pathlib.Path("a/b/snappy_source_dir") -+ self.sample_snappy_binary_dir = pathlib.Path("a/b/snappy_binary_dir") -+ -+ def test_snappy_detect_line_is_commented_and_replaced(self): -+ lines = ( -+ """check_library_exists(snappy snappy_compress "" HAVE_SNAPPY)""", -+ ) -+ patcher = leveldb_patch.CMakeListsPatcher( -+ lines, -+ "MyCoolScript", -+ self.sample_snappy_source_dir, -+ self.sample_snappy_binary_dir, -+ ) -+ -+ patched_lines = tuple(patcher.patch()) -+ -+ self.assertSequenceEqual(patched_lines, [ -+ "# BEGIN: snappy_detect_line modification by MyCoolScript", -+ """# check_library_exists(snappy snappy_compress "" HAVE_SNAPPY)""", -+ """set(HAVE_SNAPPY ON CACHE BOOL "")""", -+ "# END: snappy_detect_line modification by MyCoolScript", -+ ]) -+ -+ def test_snappy_detect_line_has_indent_and_eol_preserved(self): -+ lines = ( -+ """ check_library_exists(snappy snappy_compress "" HAVE_SNAPPY) \n""", -+ ) -+ patcher = leveldb_patch.CMakeListsPatcher( -+ lines, -+ "MyCoolScript", -+ self.sample_snappy_source_dir, -+ self.sample_snappy_binary_dir, -+ ) -+ -+ patched_lines = tuple(patcher.patch()) -+ -+ self.assertSequenceEqual(patched_lines, [ -+ "# BEGIN: snappy_detect_line modification by MyCoolScript \n", -+ """ # check_library_exists(snappy snappy_compress "" HAVE_SNAPPY) \n""", -+ """ set(HAVE_SNAPPY ON CACHE BOOL "") \n""", -+ "# END: snappy_detect_line modification by MyCoolScript \n", -+ ]) -+ -+ def test_snappy_detect_line_does_not_affect_surrounding_lines(self): -+ lines = ( -+ "aaa", -+ "bbb", -+ """check_library_exists(snappy snappy_compress "" HAVE_SNAPPY)""", -+ "ccc", -+ "ddd", -+ ) -+ patcher = leveldb_patch.CMakeListsPatcher( -+ lines, -+ "MyCoolScript", -+ self.sample_snappy_source_dir, -+ self.sample_snappy_binary_dir, -+ ) -+ -+ patched_lines = tuple(patcher.patch()) -+ -+ self.assertSequenceEqual(patched_lines, [ -+ "aaa", -+ "bbb", -+ "# BEGIN: snappy_detect_line modification by MyCoolScript", -+ """# check_library_exists(snappy snappy_compress "" HAVE_SNAPPY)""", -+ """set(HAVE_SNAPPY ON CACHE BOOL "")""", -+ "# END: snappy_detect_line modification by MyCoolScript", -+ "ccc", -+ "ddd", -+ ]) -+ -+ def test_snappy_include_is_amended(self): -+ lines = ( -+ "target_include_directories(leveldb", -+ "PUBLIC", -+ "path1", -+ "path2", -+ ")", -+ ) -+ patcher = leveldb_patch.CMakeListsPatcher( -+ lines, -+ script_name="MyCoolSript", -+ snappy_source_dir=pathlib.Path("a/b"), -+ snappy_binary_dir=pathlib.Path("c/d"), -+ ) -+ -+ patched_lines = tuple(patcher.patch()) -+ -+ self.assertSequenceEqual(patched_lines, [ -+ "target_include_directories(leveldb", -+ "# BEGIN: leveldb_include_start modification by MyCoolSript", -+ "PRIVATE", -+ "a/b", -+ "c/d", -+ "# END: leveldb_include_start modification by MyCoolSript", -+ "PUBLIC", -+ "path1", -+ "path2", -+ ")", -+ ]) -+ -+ def test_snappy_include_lines_adopt_indenting_and_eol_convention(self): -+ lines = ( -+ "target_include_directories(leveldb\n", -+ " PUBLIC \n", -+ " path1 \n", -+ " path2 \n", -+ ")\n", -+ ) -+ patcher = leveldb_patch.CMakeListsPatcher( -+ lines, -+ script_name="MyCoolSript", -+ snappy_source_dir=pathlib.Path("a/b"), -+ snappy_binary_dir=pathlib.Path("c/d"), -+ ) -+ -+ patched_lines = tuple(patcher.patch()) -+ -+ self.assertSequenceEqual(patched_lines, [ -+ "target_include_directories(leveldb\n", -+ "# BEGIN: leveldb_include_start modification by MyCoolSript \n", -+ " PRIVATE \n", -+ " a/b \n", -+ " c/d \n", -+ "# END: leveldb_include_start modification by MyCoolSript \n", -+ " PUBLIC \n", -+ " path1 \n", -+ " path2 \n", -+ ")\n", -+ ]) -+ -+ def test_snappy_include_line_does_not_affect_surrounding_lines(self): -+ lines = ( -+ "aaa", -+ "bbb", -+ "target_include_directories(leveldb", -+ "PUBLIC", -+ "path1", -+ "path2", -+ ")", -+ "ccc", -+ "ddd", -+ ) -+ patcher = leveldb_patch.CMakeListsPatcher( -+ lines, -+ script_name="MyCoolSript", -+ snappy_source_dir=pathlib.Path("a/b"), -+ snappy_binary_dir=pathlib.Path("c/d"), -+ ) -+ -+ patched_lines = tuple(patcher.patch()) -+ -+ self.assertSequenceEqual(patched_lines, [ -+ "aaa", -+ "bbb", -+ "target_include_directories(leveldb", -+ "# BEGIN: leveldb_include_start modification by MyCoolSript", -+ "PRIVATE", -+ "a/b", -+ "c/d", -+ "# END: leveldb_include_start modification by MyCoolSript", -+ "PUBLIC", -+ "path1", -+ "path2", -+ ")", -+ "ccc", -+ "ddd", -+ ]) -+ -+ def test_leveldb_snappy_link_line_is_commented_and_replaced(self): -+ lines = ( -+ "target_link_libraries(leveldb snappy)", -+ ) -+ patcher = leveldb_patch.CMakeListsPatcher( -+ lines, -+ script_name="MyCoolSript", -+ snappy_source_dir=pathlib.Path("a/b"), -+ snappy_binary_dir=pathlib.Path("c/d"), -+ ) -+ -+ patched_lines = tuple(patcher.patch()) -+ -+ self.assertSequenceEqual(patched_lines, [ -+ "# BEGIN: leveldb_snappy_link_line modification by MyCoolSript", -+ "# target_link_libraries(leveldb snappy)", -+ "target_link_libraries(leveldb Snappy::Snappy)", -+ "# END: leveldb_snappy_link_line modification by MyCoolSript", -+ ]) -+ -+ def test_leveldb_snappy_link_line_has_indent_and_eol_preserved(self): -+ lines = ( -+ " target_link_libraries(leveldb snappy) \n", -+ ) -+ patcher = leveldb_patch.CMakeListsPatcher( -+ lines, -+ script_name="MyCoolSript", -+ snappy_source_dir=pathlib.Path("a/b"), -+ snappy_binary_dir=pathlib.Path("c/d"), -+ ) -+ -+ patched_lines = tuple(patcher.patch()) -+ -+ self.assertSequenceEqual(patched_lines, [ -+ "# BEGIN: leveldb_snappy_link_line modification by MyCoolSript \n", -+ " # target_link_libraries(leveldb snappy) \n", -+ " target_link_libraries(leveldb Snappy::Snappy) \n", -+ "# END: leveldb_snappy_link_line modification by MyCoolSript \n", -+ ]) -+ -+ def test_leveldb_snappy_link_line_does_not_affect_surrounding_lines(self): -+ lines = ( -+ "aaa", -+ "bbb", -+ "target_link_libraries(leveldb snappy)", -+ "ccc", -+ "ddd", -+ ) -+ patcher = leveldb_patch.CMakeListsPatcher( -+ lines, -+ script_name="MyCoolSript", -+ snappy_source_dir=pathlib.Path("a/b"), -+ snappy_binary_dir=pathlib.Path("c/d"), -+ ) -+ -+ patched_lines = tuple(patcher.patch()) -+ -+ self.assertSequenceEqual(patched_lines, [ -+ "aaa", -+ "bbb", -+ "# BEGIN: leveldb_snappy_link_line modification by MyCoolSript", -+ "# target_link_libraries(leveldb snappy)", -+ "target_link_libraries(leveldb Snappy::Snappy)", -+ "# END: leveldb_snappy_link_line modification by MyCoolSript", -+ "ccc", -+ "ddd", -+ ]) -+ -+ def test_all_patches_combined(self): -+ lines = ( -+ """check_library_exists(snappy snappy_compress "" HAVE_SNAPPY)""", -+ "target_include_directories(leveldb", -+ "PUBLIC", -+ "path1", -+ ")", -+ "target_link_libraries(leveldb snappy)", -+ ) -+ -+ patcher = leveldb_patch.CMakeListsPatcher( -+ lines, -+ script_name="MyCoolSript", -+ snappy_source_dir=pathlib.Path("a/b"), -+ snappy_binary_dir=pathlib.Path("c/d"), -+ ) -+ patched_lines = tuple(patcher.patch()) -+ -+ self.assertSequenceEqual(patched_lines, [ -+ "# BEGIN: snappy_detect_line modification by MyCoolSript", -+ """# check_library_exists(snappy snappy_compress "" HAVE_SNAPPY)""", -+ """set(HAVE_SNAPPY ON CACHE BOOL "")""", -+ "# END: snappy_detect_line modification by MyCoolSript", -+ "target_include_directories(leveldb", -+ "# BEGIN: leveldb_include_start modification by MyCoolSript", -+ "PRIVATE", -+ "a/b", -+ "c/d", -+ "# END: leveldb_include_start modification by MyCoolSript", -+ "PUBLIC", -+ "path1", -+ ")", -+ "# BEGIN: leveldb_snappy_link_line modification by MyCoolSript", -+ "# target_link_libraries(leveldb snappy)", -+ "target_link_libraries(leveldb Snappy::Snappy)", -+ "# END: leveldb_snappy_link_line modification by MyCoolSript", -+ ]) -+ -+ def test_idempotence(self): -+ lines = ( -+ """check_library_exists(snappy snappy_compress "" HAVE_SNAPPY)\n""", -+ "target_include_directories(leveldb", -+ "PUBLIC", -+ "path1", -+ ")", -+ "target_link_libraries(leveldb snappy)", -+ ) -+ -+ patcher1 = leveldb_patch.CMakeListsPatcher( -+ lines, -+ script_name="MyCoolSript", -+ snappy_source_dir=pathlib.Path("a/b"), -+ snappy_binary_dir=pathlib.Path("c/d"), -+ ) -+ patched_lines1 = tuple(patcher1.patch()) -+ patcher2 = leveldb_patch.CMakeListsPatcher( -+ patched_lines1, -+ script_name="MyCoolSript", -+ snappy_source_dir=pathlib.Path("a/b"), -+ snappy_binary_dir=pathlib.Path("c/d"), -+ ) -+ patched_lines2 = tuple(patcher2.patch()) -+ -+ self.assertSequenceEqual(patched_lines1, patched_lines2) -+ -+ -+if __name__ == "__main__": -+ unittest.main() -diff --git a/cmake/external/snappy.cmake b/cmake/external/snappy.cmake -new file mode 100644 -index 000000000..9f25c03d0 ---- /dev/null -+++ b/cmake/external/snappy.cmake -@@ -0,0 +1,40 @@ -+# Copyright 2022 Google LLC -+# -+# Licensed under the Apache License, Version 2.0 (the "License"); -+# you may not use this file except in compliance with the License. -+# You may obtain a copy of the License at -+# -+# http://www.apache.org/licenses/LICENSE-2.0 -+# -+# Unless required by applicable law or agreed to in writing, software -+# distributed under the License is distributed on an "AS IS" BASIS, -+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -+# See the License for the specific language governing permissions and -+# limitations under the License. -+ -+include(ExternalProject) -+ -+if(TARGET snappy) -+ return() -+endif() -+ -+set(version 1.1.9) -+ -+ExternalProject_Add( -+ snappy -+ -+ DOWNLOAD_DIR ${FIREBASE_DOWNLOAD_DIR} -+ DOWNLOAD_NAME snappy-${version}.tar.gz -+ URL https://github.com/google/snappy/archive/refs/tags/${version}.tar.gz -+ URL_HASH SHA256=75c1fbb3d618dd3a0483bff0e26d0a92b495bbe5059c8b4f1c962b478b6e06e7 -+ -+ PREFIX ${PROJECT_BINARY_DIR} -+ -+ CONFIGURE_COMMAND "" -+ BUILD_COMMAND "" -+ INSTALL_COMMAND "" -+ TEST_COMMAND "" -+ PATCH_COMMAND patch -Np1 -i ${CMAKE_CURRENT_LIST_DIR}/snappy.patch -+ -+ HTTP_HEADER "${EXTERNAL_PROJECT_HTTP_HEADER}" -+) -diff --git a/cmake/external/snappy.patch b/cmake/external/snappy.patch -new file mode 100644 -index 000000000..28bfb0837 ---- /dev/null -+++ b/cmake/external/snappy.patch -@@ -0,0 +1,12 @@ -+diff -Naur snappy/snappy.cc snappy_patched/snappy.cc -+--- snappy/snappy.cc 2022-04-12 20:44:55.000000000 -0400 -++++ snappy_patched/snappy.cc 2022-04-12 20:47:05.000000000 -0400 -+@@ -1014,7 +1014,7 @@ -+ } -+ -+ SNAPPY_ATTRIBUTE_ALWAYS_INLINE -+-size_t AdvanceToNextTag(const uint8_t** ip_p, size_t* tag) { -++inline size_t AdvanceToNextTag(const uint8_t** ip_p, size_t* tag) { -+ const uint8_t*& ip = *ip_p; -+ // This section is crucial for the throughput of the decompression loop. -+ // The latency of an iteration is fundamentally constrained by the From 13222be982a8a5aa48246523d8877b9f945e4a6c Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Thu, 14 Apr 2022 20:20:35 -0400 Subject: [PATCH 02/19] firestore_patch.py: also copy the URL_HASH --- cmake/external/firestore_patch.py | 117 +++++----- cmake/external/firestore_patch_test.py | 297 +++++++++++++++++-------- cmake/external/leveldb.cmake | 1 + 3 files changed, 261 insertions(+), 154 deletions(-) diff --git a/cmake/external/firestore_patch.py b/cmake/external/firestore_patch.py index f25e46e1da..b5b50a258e 100644 --- a/cmake/external/firestore_patch.py +++ b/cmake/external/firestore_patch.py @@ -18,30 +18,29 @@ """ import argparse -import dataclasses import os import pathlib import re -from typing import Iterable, Sequence +from typing import List, Tuple VERSION_PATTERN = r"\s*set\s*\(\s*version\s+([^)\s]+)\s*\)\s*" VERSION_EXPR = re.compile(VERSION_PATTERN, re.IGNORECASE) +URL_HASH_PATTERN = r"\s*URL_HASH\s+([0-9a-zA-Z=]+)\s*" +URL_HASH_EXPR = re.compile(URL_HASH_PATTERN, re.IGNORECASE) def main() -> None: - args = parse_args() - leveldb_version = load_leveldb_version(args.leveldb_cmake_src_file) - set_leveldb_version(args.leveldb_cmake_dest_file, leveldb_version) + (src_file, dest_file) = parse_args() + leveldb_version = load_value(src_file, VERSION_EXPR) + url_hash = load_value(src_file, URL_HASH_EXPR) -@dataclasses.dataclass(frozen=True) -class ParsedArgs: - leveldb_cmake_src_file: pathlib.Path - leveldb_cmake_dest_file: pathlib.Path + set_value(dest_file, VERSION_EXPR, leveldb_version) + set_value(dest_file, URL_HASH_EXPR, url_hash) -def parse_args() -> ParsedArgs: +def parse_args() -> Tuple[pathlib.Path, pathlib.Path]: arg_parser = argparse.ArgumentParser() arg_parser.add_argument("--leveldb-version-from", required=True) arg_parser.add_argument("--leveldb-version-to") @@ -56,64 +55,68 @@ def parse_args() -> ParsedArgs: leveldb_cmake_dest_file = pathlib.Path.cwd() \ / "cmake" / "external" / "leveldb.cmake" - return ParsedArgs( - leveldb_cmake_src_file=leveldb_cmake_src_file, - leveldb_cmake_dest_file=leveldb_cmake_dest_file, - ) + return (leveldb_cmake_src_file, leveldb_cmake_dest_file) -def load_leveldb_version(cmake_file: pathlib.Path) -> str: - version = None - version_line = None - version_line_number = None +def load_value(file_: pathlib.Path, expr: re.Pattern) -> str: + value = None + value_line = None + value_line_number = None - with cmake_file.open("rt", encoding="utf8") as f: + with file_.open("rt", encoding="utf8") as f: for (line_number, line) in enumerate(f, start=1): - match = VERSION_EXPR.fullmatch(line) - if match: - if version is not None: - raise LevelDbVersionLineError( - f"Multiple lines matching regex {VERSION_EXPR.pattern} found in " - f"{cmake_file}: line {version_line_number}, {version_line.strip()} " - f"and line {line_number}, {line.strip()}") - version = match.group(1).strip() - version_line = line - version_line_number = line_number - - if version is None: - raise LevelDbVersionLineError( - f"No line matching regex {VERSION_EXPR.pattern} found in {cmake_file}") - - return version - - -def set_leveldb_version(cmake_file: pathlib.Path, version: str) -> str: - with cmake_file.open("rt", encoding="utf8") as f: + match = expr.fullmatch(line) + if not match: + continue + elif value is not None: + raise RegexMatchError( + f"Multiple lines matching regex {expr.pattern} found in " + f"{file_}: line {value_line_number}, {value_line.strip()} " + f"and line {line_number}, {line.strip()}") + + value = match.group(1).strip() + value_line = line + value_line_number = line_number + + if value is None: + raise RegexMatchError( + f"No line matching regex {expr.pattern} found in {file_}") + + return value + + +def set_value(file_: pathlib.Path, expr: re.Pattern, version: str) -> None: + with file_.open("rt", encoding="utf8") as f: lines = list(f) - version_lines_found = [] - for (i, line) in enumerate(lines): - match = VERSION_EXPR.fullmatch(line) - if match: - lines[i] = line[:match.start(1)] + version + line[match.end(1):] - version_lines_found.append(i) - - if len(version_lines_found) == 0: - raise LevelDbVersionLineError( - f"No line matching regex {VERSION_EXPR.pattern} found in {cmake_file}") - elif len(version_lines_found) > 1: - raise LevelDbVersionLineError( - f"Multiple lines matching regex {VERSION_EXPR.pattern} found in " - f"{cmake_file}: {', '.join(str(i+1) for i in version_lines_found)}") - - with cmake_file.open("wt", encoding="utf8") as f: + matching_line = None + matching_line_number = None + + for (line_number, line) in enumerate(lines, start=1): + match = expr.fullmatch(line) + if not match: + continue + elif matching_line is not None: + raise RegexMatchError( + f"Multiple lines matching regex {expr.pattern} found in " + f"{file_}: line {matching_line_number}, {matching_line.strip()} " + f"and line {line_number}, {line.strip()}") + + lines[line_number - 1] = line[:match.start(1)] + version + line[match.end(1):] + matching_line = line + matching_line_number = line_number + + if matching_line is None: + raise RegexMatchError( + f"No line matching regex {expr.pattern} found in {file_}") + + with file_.open("wt", encoding="utf8") as f: f.writelines(lines) -class LevelDbVersionLineError(Exception): +class RegexMatchError(Exception): pass - if __name__ == "__main__": main() diff --git a/cmake/external/firestore_patch_test.py b/cmake/external/firestore_patch_test.py index 3b289259f3..bf739be555 100644 --- a/cmake/external/firestore_patch_test.py +++ b/cmake/external/firestore_patch_test.py @@ -12,150 +12,253 @@ # See the License for the specific language governing permissions and # limitations under the License. -import firestore_patch +import os import pathlib +import re import shutil import tempfile +from typing import Iterable import unittest +import unittest.mock + +import firestore_patch + + +class TestUtilsMixin: + + def create_temp_file_with_lines(self, lines: Iterable[str]) -> pathlib.Path: + (handle, path_str) = tempfile.mkstemp() + os.close(handle) + path = pathlib.Path(path_str) + self.addCleanup(path.unlink, missing_ok=True) # pytype: disable=attribute-error + + with path.open("wt", encoding="utf8") as f: + for line in lines: + print(line, file=f) + + return path + + def create_temp_file_with_line(self, line: str) -> pathlib.Path: + return self.create_temp_file_with_lines([line]) + +class MainTest(TestUtilsMixin, unittest.TestCase): -class LoadLevelDbVersionTest(unittest.TestCase): + def test(self): + src_file = self.create_temp_file_with_lines([ + "aaa", + "bbb", + "set(version 1.2.3)", + "URL_HASH SHA256=abcdef", + "ccc", + "ddd", + ]) + dest_file = self.create_temp_file_with_lines([ + "aaa", + "bbb", + "set(version 4.5.6)", + "URL_HASH SHA256=ghijkl", + "ccc", + "ddd", + ]) + dest_file_contents_before = dest_file.read_text(encoding="utf8") + patcher = unittest.mock.patch.object( + firestore_patch, + "parse_args", + spec_set=True, + autospec=True, + return_value=(src_file, dest_file) + ) - def setUp(self): - super().setUp() - temp_dir = pathlib.Path(tempfile.mkdtemp()) - self.addCleanup(shutil.rmtree, temp_dir) - self.temp_file = temp_dir / "temp_file.txt" + with patcher: + firestore_patch.main() - def test_loads_correct_version(self): - with self.temp_file.open("wt", encoding="utf8") as f: - print("blah1", file=f) - print("set(version 1.23)", file=f) - print("blah2", file=f) + dest_file_contents_after = dest_file.read_text(encoding="utf8") + self.assertEqual( + dest_file_contents_after, + dest_file_contents_before + .replace("4.5.6", "1.2.3") + .replace("ghijkl", "abcdef") + ) - version = firestore_patch.load_leveldb_version(self.temp_file) - self.assertEqual(version, "1.23") +class VersionExprTest(TestUtilsMixin, unittest.TestCase): + + def test_matches_semantic_version(self): + path = self.create_temp_file_with_line("set(version 1.2.3)") + + value = firestore_patch.load_value(path, firestore_patch.VERSION_EXPR) + + self.assertEqual(value, "1.2.3") + + def test_matches_sha1(self): + path = self.create_temp_file_with_line("set(version fd054fa01)") + + value = firestore_patch.load_value(path, firestore_patch.VERSION_EXPR) + + self.assertEqual(value, "fd054fa01") def test_ignores_whitespace(self): - with self.temp_file.open("wt", encoding="utf8") as f: - print(" set ( version 1.23 ) ", file=f) + path = self.create_temp_file_with_line(" set ( version 1.2.3 ) ") - version = firestore_patch.load_leveldb_version(self.temp_file) + value = firestore_patch.load_value(path, firestore_patch.VERSION_EXPR) - self.assertEqual(version, "1.23") + self.assertEqual(value, "1.2.3") def test_case_insensitive(self): - with self.temp_file.open("wt", encoding="utf8") as f: - print("SeT(vErSiOn 1.23)", file=f) + path = self.create_temp_file_with_line("sEt(vErSiOn 1.2.3)") - version = firestore_patch.load_leveldb_version(self.temp_file) + value = firestore_patch.load_value(path, firestore_patch.VERSION_EXPR) - self.assertEqual(version, "1.23") + self.assertEqual(value, "1.2.3") - def test_version_not_found_raises_error(self): - with self.temp_file.open("wt", encoding="utf8") as f: - print("aaa", file=f) - print("bbb", file=f) - with self.assertRaises(firestore_patch.LevelDbVersionLineError) as cm: - firestore_patch.load_leveldb_version(self.temp_file) +class UrlHashExprTest(TestUtilsMixin, unittest.TestCase): - self.assertIn("no line matching", str(cm.exception).lower()) - self.assertIn(str(self.temp_file), str(cm.exception)) + def test_matches_sha256(self): + path = self.create_temp_file_with_line("URL_HASH SHA256=abc123def456") - def test_multiple_version_lines_found_raises_error(self): - with self.temp_file.open("wt", encoding="utf8") as f: - for i in range(100): - print(f"line {i+1}", file=f) - print("set(version aaa)", file=f) - print("set(version bbb)", file=f) + value = firestore_patch.load_value(path, firestore_patch.URL_HASH_EXPR) - with self.assertRaises(firestore_patch.LevelDbVersionLineError) as cm: - firestore_patch.load_leveldb_version(self.temp_file) + self.assertEqual(value, "SHA256=abc123def456") - self.assertIn("multiple lines matching", str(cm.exception).lower()) - self.assertIn(str(self.temp_file), str(cm.exception)) - self.assertIn("line 101", str(cm.exception)) - self.assertIn("line 102", str(cm.exception)) - self.assertIn("aaa", str(cm.exception)) - self.assertIn("bbb", str(cm.exception)) + def test_ignores_whitespace(self): + path = self.create_temp_file_with_line(" URL_HASH abc123def456 ") + + value = firestore_patch.load_value(path, firestore_patch.URL_HASH_EXPR) + + self.assertEqual(value, "abc123def456") + + def test_case_insensitive(self): + path = self.create_temp_file_with_line("UrL_hAsH Sha256=abc123def456") + + value = firestore_patch.load_value(path, firestore_patch.URL_HASH_EXPR) + + self.assertEqual(value, "Sha256=abc123def456") + + +class LoadValueTest(TestUtilsMixin, unittest.TestCase): + + def test_loads_the_value(self): + path = self.create_temp_file_with_line("aaa1234ccc") + expr = re.compile(r"\D+(\d+)\D+") + value = firestore_patch.load_value(path, expr) -class SetLevelDbVersionTest(unittest.TestCase): + self.assertEqual(value, "1234") - def setUp(self): - super().setUp() - temp_dir = pathlib.Path(tempfile.mkdtemp()) - self.addCleanup(shutil.rmtree, temp_dir) - self.temp_file = temp_dir / "temp_file.txt" + def test_loads_the_value_ignoring_non_matching_lines(self): + path = self.create_temp_file_with_lines([ + "blah", + "blah", + "aaa1234cccc", + "blah", + "blah", + ]) + expr = re.compile(r"\D+(\d+)\D+") - def test_saves_correct_version(self): - with self.temp_file.open("wt", encoding="utf8") as f: - print("set(version asdfasdf)", file=f) + value = firestore_patch.load_value(path, expr) - firestore_patch.set_leveldb_version(self.temp_file, "1.2.3") + self.assertEqual(value, "1234") - new_version = firestore_patch.load_leveldb_version(self.temp_file) - self.assertEqual(new_version, "1.2.3") + def test_raises_error_if_no_matching_line_found(self): + path = self.create_temp_file_with_lines([ + "blah", + "blah", + "blah", + "blah", + ]) + expr = re.compile(r"\D+(\d+)\D+") - def test_case_insensitivity(self): - with self.temp_file.open("wt", encoding="utf8") as f: - print("sEt(vErSiOn asdfasdf)", file=f) + with self.assertRaises(firestore_patch.RegexMatchError) as cm: + firestore_patch.load_value(path, expr) + + self.assertIn("no line matching", str(cm.exception).lower()) + self.assertIn(expr.pattern, str(cm.exception)) + self.assertIn(str(path), str(cm.exception)) + + def test_raises_error_if_multiple_matching_lines_found(self): + lines = ["blah"] * 100 + lines.append("aaa123bbb") + lines.append("ccc456ddd") + path = self.create_temp_file_with_lines(lines) + expr = re.compile(r"\D+(\d+)\D+") + + with self.assertRaises(firestore_patch.RegexMatchError) as cm: + firestore_patch.load_value(path, expr) + + self.assertIn("multiple lines matching", str(cm.exception).lower()) + self.assertIn(str(path), str(cm.exception)) + self.assertIn(expr.pattern, str(cm.exception)) + self.assertIn("line 101", str(cm.exception)) + self.assertIn("line 102", str(cm.exception)) + self.assertIn("123", str(cm.exception)) + self.assertIn("456", str(cm.exception)) - firestore_patch.set_leveldb_version(self.temp_file, "1.2.3") - new_version = firestore_patch.load_leveldb_version(self.temp_file) - self.assertEqual(new_version, "1.2.3") +class SaveValueTest(TestUtilsMixin, unittest.TestCase): - def test_leaves_whitespace_alone(self): - with self.temp_file.open("wt", encoding="utf8") as f: - print(" set ( version 1.2.3.4 ) ", file=f) - temp_file_contents = self.temp_file.read_text(encoding="utf8") + def test_saves_the_value(self): + path = self.create_temp_file_with_line("aaa1234ccc") + expr = re.compile(r"\D+(\d+)\D+") - firestore_patch.set_leveldb_version(self.temp_file, "a.b.c.d") + firestore_patch.set_value(path, expr, "9876") - temp_file_contents_after = self.temp_file.read_text(encoding="utf8") - expected_temp_file_contents = temp_file_contents.replace("1.2.3.4", "a.b.c.d") - self.assertEqual(temp_file_contents_after, expected_temp_file_contents) + new_value = firestore_patch.load_value(path, expr) + self.assertEqual(new_value, "9876") - def test_does_not_touch_other_lines(self): - with self.temp_file.open("wt", encoding="utf8") as f: - print("blah1", file=f) - print("set(version 1.2.3.4)", file=f) - print("blah2", file=f) - temp_file_contents = self.temp_file.read_text(encoding="utf8") + def test_saves_the_value_ignoring_non_matching_lines(self): + path = self.create_temp_file_with_lines([ + "blah", + "blah", + "aaa1234cccc", + "blah", + "blah", + ]) + expr = re.compile(r"\D+(\d+)\D+") + file_contents_before = path.read_text(encoding="utf8") - firestore_patch.set_leveldb_version(self.temp_file, "a.b.c.d") + firestore_patch.set_value(path, expr, "9876") - temp_file_contents_after = self.temp_file.read_text(encoding="utf8") - expected_temp_file_contents = temp_file_contents.replace("1.2.3.4", "a.b.c.d") - self.assertEqual(temp_file_contents_after, expected_temp_file_contents) + file_contents_after = path.read_text(encoding="utf8") + self.assertEqual( + file_contents_after, + file_contents_before.replace("1234", "9876"), + ) - def test_version_not_found_raises_error(self): - with self.temp_file.open("wt", encoding="utf8") as f: - print("aaa", file=f) - print("bbb", file=f) + def test_raises_error_if_no_matching_line_found(self): + path = self.create_temp_file_with_lines([ + "blah", + "blah", + "blah", + "blah", + ]) + expr = re.compile(r"\D+(\d+)\D+") - with self.assertRaises(firestore_patch.LevelDbVersionLineError) as cm: - firestore_patch.set_leveldb_version(self.temp_file, "a.b.c") + with self.assertRaises(firestore_patch.RegexMatchError) as cm: + firestore_patch.set_value(path, expr, "") self.assertIn("no line matching", str(cm.exception).lower()) - self.assertIn(str(self.temp_file), str(cm.exception)) + self.assertIn(expr.pattern, str(cm.exception)) + self.assertIn(str(path), str(cm.exception)) - def test_multiple_version_lines_found_raises_error(self): - with self.temp_file.open("wt", encoding="utf8") as f: - for i in range(100): - print(f"line {i+1}", file=f) - print("set(version aaa)", file=f) - print("set(version bbb)", file=f) + def test_raises_error_if_multiple_matching_lines_found(self): + lines = ["blah"] * 100 + lines.append("aaa123bbb") + lines.append("ccc456ddd") + path = self.create_temp_file_with_lines(lines) + expr = re.compile(r"\D+(\d+)\D+") - with self.assertRaises(firestore_patch.LevelDbVersionLineError) as cm: - firestore_patch.set_leveldb_version(self.temp_file, "a.b.c") + with self.assertRaises(firestore_patch.RegexMatchError) as cm: + firestore_patch.set_value(path, expr, "") self.assertIn("multiple lines matching", str(cm.exception).lower()) - self.assertIn(str(self.temp_file), str(cm.exception)) - self.assertIn("101, 102", str(cm.exception)) + self.assertIn(str(path), str(cm.exception)) + self.assertIn(expr.pattern, str(cm.exception)) + self.assertIn("line 101", str(cm.exception)) + self.assertIn("line 102", str(cm.exception)) + self.assertIn("123", str(cm.exception)) + self.assertIn("456", str(cm.exception)) if __name__ == "__main__": diff --git a/cmake/external/leveldb.cmake b/cmake/external/leveldb.cmake index 18970659a4..a8a154bb4e 100644 --- a/cmake/external/leveldb.cmake +++ b/cmake/external/leveldb.cmake @@ -26,6 +26,7 @@ ExternalProject_Add( DOWNLOAD_DIR ${FIREBASE_DOWNLOAD_DIR} DOWNLOAD_NAME leveldb-${version}.tar.gz URL https://github.com/google/leveldb/archive/${version}.tar.gz + URL_HASH SHA256=9a37f8a6174f09bd622bc723b55881dc541cd50747cbd08831c2a82d620f6d76 PREFIX ${PROJECT_BINARY_DIR} From 47a256270f19aab8d666b32acdc5efeae3bba7d7 Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Thu, 14 Apr 2022 20:38:27 -0400 Subject: [PATCH 03/19] query_main.h: fix build error due to Filter being renamed to FieldFilter. --- firestore/src/main/query_main.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/firestore/src/main/query_main.h b/firestore/src/main/query_main.h index bc430aae9b..9993777c8a 100644 --- a/firestore/src/main/query_main.h +++ b/firestore/src/main/query_main.h @@ -182,7 +182,7 @@ class QueryInternal { kEndAt, }; - using Operator = core::Filter::Operator; + using Operator = core::FieldFilter::Operator; Query Where(const FieldPath& field, Operator op, From fa99004bb56f2d0d9780902d23df0bc864135b46 Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Thu, 14 Apr 2022 21:08:40 -0400 Subject: [PATCH 04/19] user_data_converter_main.cc: fix build error when calling NullValue(), which now just returns a google_firestore_v1_Value instead of a Message --- firestore/src/main/user_data_converter_main.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/firestore/src/main/user_data_converter_main.cc b/firestore/src/main/user_data_converter_main.cc index a87ae70c42..0de008dcce 100644 --- a/firestore/src/main/user_data_converter_main.cc +++ b/firestore/src/main/user_data_converter_main.cc @@ -315,7 +315,7 @@ nanopb::Message UserDataConverter::ParseArray( for (size_t i = 0; i != input.size(); ++i) { auto parsed = ParseData(input[i], context.ChildContext(i)); if (!parsed) { - parsed = NullValue(); + parsed = Message(NullValue()); } result->array_value.values[i] = *parsed->release(); } From 555a406b8da2bae6efb207a59ee6c78c32137aa2 Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Thu, 14 Apr 2022 21:09:53 -0400 Subject: [PATCH 05/19] Merge in https://github.com/firebase/firebase-cpp-sdk/pull/896 (leveldb_snappy_test.cc: run the test on iOS as well, and other improvements) --- .../integration_test_internal/CMakeLists.txt | 7 + .../src/leveldb_snappy_test.cc | 144 ++++++++++++------ 2 files changed, 106 insertions(+), 45 deletions(-) diff --git a/firestore/integration_test_internal/CMakeLists.txt b/firestore/integration_test_internal/CMakeLists.txt index d9d7d20849..96c76bd824 100644 --- a/firestore/integration_test_internal/CMakeLists.txt +++ b/firestore/integration_test_internal/CMakeLists.txt @@ -333,6 +333,13 @@ else() ${FIREBASE_INTEGRATION_TEST_SRCS} ) + # Set a preprocessor define so that tests can distinguish between having been + # built by Xcode vs. cmake. + target_compile_definitions(${integration_test_target_name} + PRIVATE + FIREBASE_TESTS_BUILT_BY_CMAKE + ) + if(APPLE) set(ADDITIONAL_LIBS gssapi_krb5 diff --git a/firestore/integration_test_internal/src/leveldb_snappy_test.cc b/firestore/integration_test_internal/src/leveldb_snappy_test.cc index 33f2247cd8..4d82878691 100644 --- a/firestore/integration_test_internal/src/leveldb_snappy_test.cc +++ b/firestore/integration_test_internal/src/leveldb_snappy_test.cc @@ -22,6 +22,7 @@ #include #include +#include #include #include #include @@ -30,7 +31,6 @@ #include "Firestore/core/src/util/filesystem.h" #include "Firestore/core/src/util/path.h" -#include "firebase_test_framework.h" #include "gtest/gtest.h" #include "leveldb/db.h" @@ -49,36 +49,109 @@ using ::firebase::firestore::util::Path; // with reason "corruption". Path CreateLevelDbDatabaseThatUsesSnappyCompression(); -// This test ensures that we don't accidentally regress having added in Snappy -// compression support (https://github.com/firebase/firebase-ios-sdk/pull/9596). -TEST(LevelDbSnappy, LevelDbHasSnappySupportCompiledIn) { - // Do not run this test on iOS because LevelDb in iOS does not support Snappy. - SKIP_TEST_ON_IOS; +// Creates and opens a LevelDb database that contains at least one block that +// is compressed with Snappy compression, then iterates over it, invoking the +// given callback with the status at each point in the iteration. Once the +// callback is invoked with a `status` where `status.ok()` is not true, then +// iteration will stop and the callback will not be invoked again. +void IterateOverLevelDbDatabaseThatUsesSnappyCompression( + std::function); - Path leveldb_path = CreateLevelDbDatabaseThatUsesSnappyCompression(); - if (HasFatalFailure()) return; +#if FIREBASE_TESTS_BUILT_BY_CMAKE - leveldb::Options options; - options.create_if_missing = false; +// Ensure that LevelDb is compiled with Snappy compression support. +// See https://github.com/firebase/firebase-ios-sdk/pull/9596 for details. +TEST(LevelDbSnappy, LevelDbSupportsSnappy) { + IterateOverLevelDbDatabaseThatUsesSnappyCompression( + [](const leveldb::Status& status) { + ASSERT_TRUE(status.ok()) << ConvertStatus(status); + }); +} + +#else // FIREBASE_TESTS_BUILT_BY_CMAKE + +// Ensure that LevelDb is NOT compiled with Snappy compression support. +TEST(LevelDbSnappy, LevelDbDoesNotSupportSnappy) { + bool got_failed_status = false; + IterateOverLevelDbDatabaseThatUsesSnappyCompression( + [&](const leveldb::Status& status) { + if (!status.ok()) { + got_failed_status = true; + ASSERT_TRUE(status.IsCorruption()) << ConvertStatus(status); + } + }); + if (!HasFailure()) { + ASSERT_TRUE(got_failed_status) + << "Reading a Snappy-compressed LevelDb database was successful; " + "however, it should NOT have been successful " + "since Snappy support is expected to NOT be available."; + } +} + +#endif // FIREBASE_TESTS_BUILT_BY_CMAKE + +void IterateOverLevelDbDatabaseThatUsesSnappyCompression( + std::function callback) { std::unique_ptr db; { + Path leveldb_path = CreateLevelDbDatabaseThatUsesSnappyCompression(); + if (leveldb_path.empty()) { + return; + } + + leveldb::Options options; + options.create_if_missing = false; + leveldb::DB* db_ptr; leveldb::Status status = leveldb::DB::Open(options, leveldb_path.ToUtf8String(), &db_ptr); - ASSERT_TRUE(status.ok()); + + ASSERT_TRUE(status.ok()) + << "Opening LevelDb database " << leveldb_path.ToUtf8String() + << " failed: " << ConvertStatus(status); + db.reset(db_ptr); } - // One of the assertions below will fail when LevelDb attempts to read a block - // that is compressed with Snappy and Snappy compression support is not - // compiled in. std::unique_ptr it( db->NewIterator(leveldb::ReadOptions())); for (it->SeekToFirst(); it->Valid(); it->Next()) { - ASSERT_TRUE(it->status().ok()) << ConvertStatus(it->status()); + callback(it->status()); + if (!it->status().ok()) { + return; + } + } + + // Invoke the callback on the final status. + callback(it->status()); +} + +template +void WriteFile(const Path& dir, + const std::string& file_name, + const T& data_array) { + Filesystem* fs = Filesystem::Default(); + { + auto status = fs->RecursivelyCreateDir(dir); + if (!status.ok()) { + FAIL() << "Creating directory failed: " << dir.ToUtf8String() << " (" + << status.error_message() << ")"; + } + } + + Path file = dir.AppendUtf8(file_name); + std::ofstream out_file(file.native_value(), std::ios::binary); + if (!out_file) { + FAIL() << "Unable to open file for writing: " << file.ToUtf8String(); + } + + out_file.write(reinterpret_cast(data_array.data()), + data_array.size()); + out_file.close(); + if (!out_file) { + FAIL() << "Writing to file failed: " << file.ToUtf8String(); } - ASSERT_TRUE(it->status().ok()) << ConvertStatus(it->status()); } const std::array LevelDbSnappyFile_000005_ldb{ @@ -196,47 +269,27 @@ const std::array LevelDbSnappyFile_MANIFEST_000084{ 0x04, 0x0D, }; -template -void WriteFile(const Path& dir, - const std::string& file_name, - const T& data_array) { - Filesystem* fs = Filesystem::Default(); - { - auto status = fs->RecursivelyCreateDir(dir); - if (!status.ok()) { - FAIL() << "Creating directory failed: " << dir.ToUtf8String() << " (" - << status.error_message() << ")"; - } - } - - Path file = dir.AppendUtf8(file_name); - std::ofstream out_file(file.native_value(), std::ios::binary); - if (!out_file) { - FAIL() << "Unable to open file for writing: " << file.ToUtf8String(); - } - - out_file.write(reinterpret_cast(data_array.data()), - data_array.size()); - out_file.close(); - if (!out_file) { - FAIL() << "Writing to file failed: " << file.ToUtf8String(); - } -} - Path LevelDbDir() { Filesystem* fs = Filesystem::Default(); - Path dir = fs->TempDir().AppendUtf8("PersistenceTesting"); + Path dir = fs->TempDir().AppendUtf8("LevelDbSnappyTest"); // Delete the directory first to ensure isolation between runs. auto status = fs->RecursivelyRemove(dir); - EXPECT_TRUE(status.ok()) << "Failed to clean up leveldb in dir " + EXPECT_TRUE(status.ok()) << "Failed to clean up leveldb in directory " << dir.ToUtf8String() << ": " << status.ToString(); + if (!status.ok()) { + return {}; + } return dir; } Path CreateLevelDbDatabaseThatUsesSnappyCompression() { Path leveldb_dir = LevelDbDir(); + if (leveldb_dir.empty()) { + return {}; + } + WriteFile(leveldb_dir, "000005.ldb", LevelDbSnappyFile_000005_ldb); WriteFile(leveldb_dir, "000017.ldb", LevelDbSnappyFile_000017_ldb); WriteFile(leveldb_dir, "000085.ldb", LevelDbSnappyFile_000085_ldb); @@ -244,6 +297,7 @@ Path CreateLevelDbDatabaseThatUsesSnappyCompression() { WriteFile(leveldb_dir, "LOG.old", LevelDbSnappyFile_LOG_old); WriteFile(leveldb_dir, "LOG", LevelDbSnappyFile_LOG); WriteFile(leveldb_dir, "MANIFEST-000084", LevelDbSnappyFile_MANIFEST_000084); + return leveldb_dir; } From 2564279509f7bbaddbf992a37aadcddb904780e7 Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Thu, 14 Apr 2022 21:19:01 -0400 Subject: [PATCH 06/19] leveldb_snappy_test.cc: re-use the test from the iOS SDK instead of re-writing it. --- .../src/leveldb_snappy_test.cc | 287 +----------------- 1 file changed, 5 insertions(+), 282 deletions(-) diff --git a/firestore/integration_test_internal/src/leveldb_snappy_test.cc b/firestore/integration_test_internal/src/leveldb_snappy_test.cc index 4d82878691..b5d88b74f3 100644 --- a/firestore/integration_test_internal/src/leveldb_snappy_test.cc +++ b/firestore/integration_test_internal/src/leveldb_snappy_test.cc @@ -20,285 +20,8 @@ #error "This test should not be included in Android." #endif -#include -#include -#include -#include -#include -#include - -#include "Firestore/core/src/local/leveldb_util.h" -#include "Firestore/core/src/util/filesystem.h" -#include "Firestore/core/src/util/path.h" - -#include "gtest/gtest.h" -#include "leveldb/db.h" - -// TODO(dconeybe) Reference this test in the iOS SDK instead of making a -// copy of it here in the C++ SDK. - -namespace { - -using ::firebase::firestore::local::ConvertStatus; -using ::firebase::firestore::util::Filesystem; -using ::firebase::firestore::util::Path; - -// Creates a LevelDb database that uses Snappy compression for at least one of -// its blocks. Attempting to iterate over this database using a LevelDb library -// that does not have Snappy compression compiled in will return a failed status -// with reason "corruption". -Path CreateLevelDbDatabaseThatUsesSnappyCompression(); - -// Creates and opens a LevelDb database that contains at least one block that -// is compressed with Snappy compression, then iterates over it, invoking the -// given callback with the status at each point in the iteration. Once the -// callback is invoked with a `status` where `status.ok()` is not true, then -// iteration will stop and the callback will not be invoked again. -void IterateOverLevelDbDatabaseThatUsesSnappyCompression( - std::function); - -#if FIREBASE_TESTS_BUILT_BY_CMAKE - -// Ensure that LevelDb is compiled with Snappy compression support. -// See https://github.com/firebase/firebase-ios-sdk/pull/9596 for details. -TEST(LevelDbSnappy, LevelDbSupportsSnappy) { - IterateOverLevelDbDatabaseThatUsesSnappyCompression( - [](const leveldb::Status& status) { - ASSERT_TRUE(status.ok()) << ConvertStatus(status); - }); -} - -#else // FIREBASE_TESTS_BUILT_BY_CMAKE - -// Ensure that LevelDb is NOT compiled with Snappy compression support. -TEST(LevelDbSnappy, LevelDbDoesNotSupportSnappy) { - bool got_failed_status = false; - IterateOverLevelDbDatabaseThatUsesSnappyCompression( - [&](const leveldb::Status& status) { - if (!status.ok()) { - got_failed_status = true; - ASSERT_TRUE(status.IsCorruption()) << ConvertStatus(status); - } - }); - - if (!HasFailure()) { - ASSERT_TRUE(got_failed_status) - << "Reading a Snappy-compressed LevelDb database was successful; " - "however, it should NOT have been successful " - "since Snappy support is expected to NOT be available."; - } -} - -#endif // FIREBASE_TESTS_BUILT_BY_CMAKE - -void IterateOverLevelDbDatabaseThatUsesSnappyCompression( - std::function callback) { - std::unique_ptr db; - { - Path leveldb_path = CreateLevelDbDatabaseThatUsesSnappyCompression(); - if (leveldb_path.empty()) { - return; - } - - leveldb::Options options; - options.create_if_missing = false; - - leveldb::DB* db_ptr; - leveldb::Status status = - leveldb::DB::Open(options, leveldb_path.ToUtf8String(), &db_ptr); - - ASSERT_TRUE(status.ok()) - << "Opening LevelDb database " << leveldb_path.ToUtf8String() - << " failed: " << ConvertStatus(status); - - db.reset(db_ptr); - } - - std::unique_ptr it( - db->NewIterator(leveldb::ReadOptions())); - for (it->SeekToFirst(); it->Valid(); it->Next()) { - callback(it->status()); - if (!it->status().ok()) { - return; - } - } - - // Invoke the callback on the final status. - callback(it->status()); -} - -template -void WriteFile(const Path& dir, - const std::string& file_name, - const T& data_array) { - Filesystem* fs = Filesystem::Default(); - { - auto status = fs->RecursivelyCreateDir(dir); - if (!status.ok()) { - FAIL() << "Creating directory failed: " << dir.ToUtf8String() << " (" - << status.error_message() << ")"; - } - } - - Path file = dir.AppendUtf8(file_name); - std::ofstream out_file(file.native_value(), std::ios::binary); - if (!out_file) { - FAIL() << "Unable to open file for writing: " << file.ToUtf8String(); - } - - out_file.write(reinterpret_cast(data_array.data()), - data_array.size()); - out_file.close(); - if (!out_file) { - FAIL() << "Writing to file failed: " << file.ToUtf8String(); - } -} - -const std::array LevelDbSnappyFile_000005_ldb{ - 0x84, 0x03, 0x80, 0x00, 0x42, 0x00, 0x85, 0x71, 0x75, 0x65, 0x72, 0x79, - 0x5F, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x00, 0x01, 0x8B, 0x43, 0x6F, - 0x6C, 0x41, 0x2F, 0x44, 0x6F, 0x63, 0x41, 0x2F, 0x43, 0x6F, 0x6C, 0x42, - 0x01, 0x0A, 0x68, 0x42, 0x7C, 0x66, 0x3A, 0x7C, 0x6F, 0x62, 0x3A, 0x5F, - 0x5F, 0x6E, 0x61, 0x6D, 0x65, 0x5F, 0x5F, 0x61, 0x73, 0x63, 0x00, 0x01, - 0x8C, 0x82, 0x80, 0x01, 0x07, 0x00, 0x05, 0x01, 0x08, 0x01, 0x13, 0x50, - 0x11, 0x3E, 0x01, 0x16, 0x00, 0x0A, 0x05, 0x15, 0xF0, 0x3C, 0x00, 0x08, - 0x02, 0x20, 0x05, 0x32, 0x4A, 0x12, 0x48, 0x70, 0x72, 0x6F, 0x6A, 0x65, - 0x63, 0x74, 0x73, 0x2F, 0x54, 0x65, 0x73, 0x74, 0x54, 0x65, 0x72, 0x6D, - 0x69, 0x6E, 0x61, 0x74, 0x65, 0x2F, 0x64, 0x61, 0x74, 0x61, 0x62, 0x61, - 0x73, 0x65, 0x73, 0x2F, 0x28, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6C, 0x74, - 0x29, 0x2F, 0x64, 0x6F, 0x63, 0x75, 0x6D, 0x65, 0x6E, 0x74, 0x73, 0x01, - 0x7B, 0x3E, 0x85, 0x00, 0x0C, 0x0D, 0x07, 0x50, 0x08, 0x15, 0x5A, 0x00, - 0x03, 0xFE, 0x5A, 0x00, 0x2E, 0x5A, 0x00, 0x38, 0x07, 0x12, 0x06, 0x5F, - 0x67, 0x6C, 0x6F, 0x62, 0x61, 0x6C, 0x00, 0x01, 0x80, 0x01, 0x0B, 0x11, - 0x65, 0x1C, 0x10, 0x05, 0x20, 0x01, 0x12, 0x07, 0x06, 0x09, 0x15, 0x10, - 0x00, 0x03, 0x01, 0x10, 0x04, 0x00, 0x01, 0x09, 0x10, 0x24, 0x01, 0x12, - 0x01, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6F, 0x6E, 0x01, 0x35, 0x00, 0x06, - 0x09, 0x15, 0x10, 0x37, 0x0C, 0x07, 0x01, 0x05, 0x09, 0x0B, 0x10, 0x36, - 0x0C, 0x07, 0x01, 0x04, 0x09, 0x0B, 0x10, 0x35, 0x0C, 0x07, 0x01, 0x03, - 0x09, 0x0B, 0x4C, 0x34, 0x0C, 0x07, 0x01, 0x02, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x33, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, - 0x2C, 0x6E, 0xE0, 0xF4, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, - 0x00, 0xC0, 0xF2, 0xA1, 0xB0, 0x00, 0x09, 0x03, 0x86, 0x01, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x87, 0x02, 0x00, 0x00, 0x00, 0x00, - 0x01, 0x00, 0x00, 0x00, 0x00, 0x58, 0xC2, 0x94, 0x06, 0x8C, 0x02, 0x08, - 0x99, 0x02, 0x17, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x57, 0xFB, 0x80, 0x8B, 0x24, 0x75, 0x47, 0xDB, -}; -const std::array LevelDbSnappyFile_000017_ldb{ - 0x00, 0x14, 0x50, 0x85, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x00, 0x01, - 0x8C, 0x82, 0x80, 0x01, 0x0C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, - 0x02, 0x20, 0x0A, 0x32, 0x4A, 0x12, 0x48, 0x70, 0x72, 0x6F, 0x6A, 0x65, - 0x63, 0x74, 0x73, 0x2F, 0x54, 0x65, 0x73, 0x74, 0x54, 0x65, 0x72, 0x6D, - 0x69, 0x6E, 0x61, 0x74, 0x65, 0x2F, 0x64, 0x61, 0x74, 0x61, 0x62, 0x61, - 0x73, 0x65, 0x73, 0x2F, 0x28, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6C, 0x74, - 0x29, 0x2F, 0x64, 0x6F, 0x63, 0x75, 0x6D, 0x65, 0x6E, 0x74, 0x73, 0x2F, - 0x43, 0x6F, 0x6C, 0x41, 0x2F, 0x44, 0x6F, 0x63, 0x41, 0x2F, 0x43, 0x6F, - 0x6C, 0x42, 0x2F, 0x44, 0x6F, 0x63, 0x42, 0x07, 0x12, 0x06, 0x5F, 0x67, - 0x6C, 0x6F, 0x62, 0x61, 0x6C, 0x00, 0x01, 0x80, 0x01, 0x0D, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x08, 0x02, 0x10, 0x0A, 0x20, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0xFE, 0xCD, 0xE0, 0x39, 0x00, - 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0xC0, 0xF2, 0xA1, 0xB0, - 0x00, 0x09, 0x03, 0x86, 0x01, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0x00, 0x8A, 0x01, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, - 0xE4, 0xA7, 0x7E, 0x74, 0x8F, 0x01, 0x08, 0x9C, 0x01, 0x17, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x57, 0xFB, 0x80, 0x8B, - 0x24, 0x75, 0x47, 0xDB, -}; -const std::array LevelDbSnappyFile_000085_ldb{}; -const std::array LevelDbSnappyFile_CURRENT{ - 0x4D, 0x41, 0x4E, 0x49, 0x46, 0x45, 0x53, 0x54, - 0x2D, 0x30, 0x30, 0x30, 0x30, 0x38, 0x34, 0x0A, -}; -const std::array LevelDbSnappyFile_LOG_old{ - 0x32, 0x30, 0x32, 0x32, 0x2F, 0x30, 0x34, 0x2F, 0x30, 0x34, 0x2D, 0x31, - 0x31, 0x3A, 0x33, 0x39, 0x3A, 0x34, 0x36, 0x2E, 0x32, 0x35, 0x37, 0x32, - 0x35, 0x31, 0x20, 0x30, 0x78, 0x37, 0x30, 0x30, 0x30, 0x30, 0x35, 0x33, - 0x31, 0x34, 0x30, 0x30, 0x30, 0x20, 0x52, 0x65, 0x63, 0x6F, 0x76, 0x65, - 0x72, 0x69, 0x6E, 0x67, 0x20, 0x6C, 0x6F, 0x67, 0x20, 0x23, 0x38, 0x31, - 0x0A, 0x32, 0x30, 0x32, 0x32, 0x2F, 0x30, 0x34, 0x2F, 0x30, 0x34, 0x2D, - 0x31, 0x31, 0x3A, 0x33, 0x39, 0x3A, 0x34, 0x36, 0x2E, 0x33, 0x30, 0x34, - 0x35, 0x35, 0x32, 0x20, 0x30, 0x78, 0x37, 0x30, 0x30, 0x30, 0x30, 0x35, - 0x33, 0x31, 0x34, 0x30, 0x30, 0x30, 0x20, 0x44, 0x65, 0x6C, 0x65, 0x74, - 0x65, 0x20, 0x74, 0x79, 0x70, 0x65, 0x3D, 0x33, 0x20, 0x23, 0x38, 0x30, - 0x0A, 0x32, 0x30, 0x32, 0x32, 0x2F, 0x30, 0x34, 0x2F, 0x30, 0x34, 0x2D, - 0x31, 0x31, 0x3A, 0x33, 0x39, 0x3A, 0x34, 0x36, 0x2E, 0x33, 0x30, 0x35, - 0x30, 0x36, 0x34, 0x20, 0x30, 0x78, 0x37, 0x30, 0x30, 0x30, 0x30, 0x35, - 0x33, 0x31, 0x34, 0x30, 0x30, 0x30, 0x20, 0x44, 0x65, 0x6C, 0x65, 0x74, - 0x65, 0x20, 0x74, 0x79, 0x70, 0x65, 0x3D, 0x30, 0x20, 0x23, 0x38, 0x31, - 0x0A, -}; -const std::array LevelDbSnappyFile_LOG{ - 0x32, 0x30, 0x32, 0x32, 0x2F, 0x30, 0x34, 0x2F, 0x30, 0x34, 0x2D, 0x31, - 0x31, 0x3A, 0x35, 0x36, 0x3A, 0x35, 0x36, 0x2E, 0x34, 0x39, 0x33, 0x31, - 0x34, 0x32, 0x20, 0x30, 0x78, 0x37, 0x30, 0x30, 0x30, 0x30, 0x61, 0x32, - 0x35, 0x34, 0x30, 0x30, 0x30, 0x20, 0x52, 0x65, 0x63, 0x6F, 0x76, 0x65, - 0x72, 0x69, 0x6E, 0x67, 0x20, 0x6C, 0x6F, 0x67, 0x20, 0x23, 0x38, 0x33, - 0x0A, 0x32, 0x30, 0x32, 0x32, 0x2F, 0x30, 0x34, 0x2F, 0x30, 0x34, 0x2D, - 0x31, 0x31, 0x3A, 0x35, 0x36, 0x3A, 0x35, 0x36, 0x2E, 0x35, 0x33, 0x34, - 0x37, 0x34, 0x35, 0x20, 0x30, 0x78, 0x37, 0x30, 0x30, 0x30, 0x30, 0x61, - 0x32, 0x35, 0x34, 0x30, 0x30, 0x30, 0x20, 0x44, 0x65, 0x6C, 0x65, 0x74, - 0x65, 0x20, 0x74, 0x79, 0x70, 0x65, 0x3D, 0x33, 0x20, 0x23, 0x38, 0x32, - 0x0A, 0x32, 0x30, 0x32, 0x32, 0x2F, 0x30, 0x34, 0x2F, 0x30, 0x34, 0x2D, - 0x31, 0x31, 0x3A, 0x35, 0x36, 0x3A, 0x35, 0x36, 0x2E, 0x35, 0x33, 0x35, - 0x32, 0x34, 0x32, 0x20, 0x30, 0x78, 0x37, 0x30, 0x30, 0x30, 0x30, 0x61, - 0x32, 0x35, 0x34, 0x30, 0x30, 0x30, 0x20, 0x44, 0x65, 0x6C, 0x65, 0x74, - 0x65, 0x20, 0x74, 0x79, 0x70, 0x65, 0x3D, 0x30, 0x20, 0x23, 0x38, 0x33, - 0x0A, -}; -const std::array LevelDbSnappyFile_MANIFEST_000084{ - 0x45, 0x63, 0x9F, 0xDD, 0xAC, 0x00, 0x01, 0x01, 0x1A, 0x6C, 0x65, 0x76, - 0x65, 0x6C, 0x64, 0x62, 0x2E, 0x42, 0x79, 0x74, 0x65, 0x77, 0x69, 0x73, - 0x65, 0x43, 0x6F, 0x6D, 0x70, 0x61, 0x72, 0x61, 0x74, 0x6F, 0x72, 0x07, - 0x00, 0x05, 0xE5, 0x02, 0x42, 0x85, 0x71, 0x75, 0x65, 0x72, 0x79, 0x5F, - 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x00, 0x01, 0x8B, 0x43, 0x6F, 0x6C, - 0x41, 0x2F, 0x44, 0x6F, 0x63, 0x41, 0x2F, 0x43, 0x6F, 0x6C, 0x42, 0x2F, - 0x44, 0x6F, 0x63, 0x42, 0x7C, 0x66, 0x3A, 0x7C, 0x6F, 0x62, 0x3A, 0x5F, - 0x5F, 0x6E, 0x61, 0x6D, 0x65, 0x5F, 0x5F, 0x61, 0x73, 0x63, 0x00, 0x01, - 0x8C, 0x82, 0x80, 0x01, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x13, - 0x85, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6F, 0x6E, 0x00, 0x01, 0x80, 0x01, - 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0x00, 0x11, 0xE8, 0x01, - 0x14, 0x85, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x00, 0x01, 0x8C, 0x82, - 0x80, 0x01, 0x0C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x19, 0x85, 0x74, - 0x61, 0x72, 0x67, 0x65, 0x74, 0x5F, 0x67, 0x6C, 0x6F, 0x62, 0x61, 0x6C, - 0x00, 0x01, 0x80, 0x01, 0x0D, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xB1, - 0x03, 0xAC, 0xBA, 0x08, 0x00, 0x01, 0x02, 0x55, 0x09, 0x00, 0x03, 0x56, - 0x04, 0x0D, -}; - -Path LevelDbDir() { - Filesystem* fs = Filesystem::Default(); - Path dir = fs->TempDir().AppendUtf8("LevelDbSnappyTest"); - - // Delete the directory first to ensure isolation between runs. - auto status = fs->RecursivelyRemove(dir); - EXPECT_TRUE(status.ok()) << "Failed to clean up leveldb in directory " - << dir.ToUtf8String() << ": " << status.ToString(); - if (!status.ok()) { - return {}; - } - - return dir; -} - -Path CreateLevelDbDatabaseThatUsesSnappyCompression() { - Path leveldb_dir = LevelDbDir(); - if (leveldb_dir.empty()) { - return {}; - } - - WriteFile(leveldb_dir, "000005.ldb", LevelDbSnappyFile_000005_ldb); - WriteFile(leveldb_dir, "000017.ldb", LevelDbSnappyFile_000017_ldb); - WriteFile(leveldb_dir, "000085.ldb", LevelDbSnappyFile_000085_ldb); - WriteFile(leveldb_dir, "CURRENT", LevelDbSnappyFile_CURRENT); - WriteFile(leveldb_dir, "LOG.old", LevelDbSnappyFile_LOG_old); - WriteFile(leveldb_dir, "LOG", LevelDbSnappyFile_LOG); - WriteFile(leveldb_dir, "MANIFEST-000084", LevelDbSnappyFile_MANIFEST_000084); - - return leveldb_dir; -} - -} // namespace +// Just re-use the unit test from the iOS SDK. +// TODO(dconeybe) Import ALL the unit tests from the iOS SDK by adding them +// to the CMakeLists.txt in the parent directory of this file. That way we can +// run all of the tests from the iOS SDK on each platform targeted by this repo. +#include "Firestore/core/test/unit/local/leveldb_snappy_test.cc" \ No newline at end of file From 19e0b72ff9ecede2cca27c2365b9d49c07b375d7 Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Thu, 14 Apr 2022 21:27:49 -0400 Subject: [PATCH 07/19] leveldb_snappy_test.cc: add a newline at the end of the file. --- firestore/integration_test_internal/src/leveldb_snappy_test.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/firestore/integration_test_internal/src/leveldb_snappy_test.cc b/firestore/integration_test_internal/src/leveldb_snappy_test.cc index b5d88b74f3..781481b763 100644 --- a/firestore/integration_test_internal/src/leveldb_snappy_test.cc +++ b/firestore/integration_test_internal/src/leveldb_snappy_test.cc @@ -24,4 +24,4 @@ // TODO(dconeybe) Import ALL the unit tests from the iOS SDK by adding them // to the CMakeLists.txt in the parent directory of this file. That way we can // run all of the tests from the iOS SDK on each platform targeted by this repo. -#include "Firestore/core/test/unit/local/leveldb_snappy_test.cc" \ No newline at end of file +#include "Firestore/core/test/unit/local/leveldb_snappy_test.cc" From 16605a9911b97bc51f85b074d4d601ecc2339bce Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Thu, 14 Apr 2022 22:12:07 -0400 Subject: [PATCH 08/19] query_main.h: add missing #include "Firestore/core/src/core/field_filter.h" --- firestore/src/main/query_main.h | 1 + 1 file changed, 1 insertion(+) diff --git a/firestore/src/main/query_main.h b/firestore/src/main/query_main.h index 9993777c8a..17d13d8dfd 100644 --- a/firestore/src/main/query_main.h +++ b/firestore/src/main/query_main.h @@ -23,6 +23,7 @@ #include "Firestore/Protos/nanopb/google/firestore/v1/document.nanopb.h" #include "Firestore/core/src/api/query_core.h" #include "Firestore/core/src/core/bound.h" +#include "Firestore/core/src/core/field_filter.h" #include "Firestore/core/src/core/order_by.h" #include "Firestore/core/src/core/query.h" #include "Firestore/core/src/nanopb/message.h" From 4e823e1ac09368d01530e4511fa29080679bda3f Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Fri, 15 Apr 2022 00:25:36 -0400 Subject: [PATCH 09/19] Set the FIRESTORE_INCLUDE_OBJC cmake cache var to NO to avoid building Objective-C parts of the iOS SDK This fixes the following build error: ``` FIRFirestore.h:147:20: error: unknown attribute 'swift_async' ignored [-Werror,-Wunknown-attributes] __attribute__((swift_async(none))); // Disable async import due to #9426. ^ 1 error generated. ``` which was introduced by https://github.com/firebase/firebase-ios-sdk/pull/9502 --- CMakeLists.txt | 4 ++++ cmake/external/firestore.cmake | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index aa12462c8b..6ca6fa75b8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -215,6 +215,10 @@ if(FIREBASE_INCLUDE_FIRESTORE AND DESKTOP) set(FIRESTORE_USE_EXTERNAL_CMAKE_BUILD ON) endif() +# Disable compiling the Objective-C (and Swift) stuff from the +# firebase-ios-sdk since it's not needed and can sometimes fail to build. +set(FIRESTORE_INCLUDE_OBJC NO) + if(FIREBASE_CPP_USE_PRIOR_GRADLE_BUILD) # Quote meta characters in ${CMAKE_CURRENT_LIST_DIR} so we can # match it in a regex. diff --git a/cmake/external/firestore.cmake b/cmake/external/firestore.cmake index af34fe9a0e..55d8268270 100644 --- a/cmake/external/firestore.cmake +++ b/cmake/external/firestore.cmake @@ -24,7 +24,7 @@ ExternalProject_Add( DOWNLOAD_DIR ${FIREBASE_DOWNLOAD_DIR} GIT_REPOSITORY https://github.com/firebase/firebase-ios-sdk - GIT_TAG 582b384c99a5dd24331161d436fdb6fd088fa833 + GIT_TAG d57360f1bf2dd7ecb85a768e8912d0045b00b88d GIT_SHALLOW ON PREFIX ${PROJECT_BINARY_DIR} From 56553d996678430e756fa2eed6b469aaa8baf299 Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Fri, 15 Apr 2022 01:01:04 -0400 Subject: [PATCH 10/19] external/pip_requirements.txt: add six This is an attempt to fix the following build error: ``` Traceback (most recent call last): File "Firestore/Protos/tmphmjWeu.py", line 25, in import nanopb_generator as nanopb File "nanopb/generator/nanopb_generator.py", line 25, in import google.protobuf.text_format as text_format File "protobuf/python/google/protobuf/text_format.py", line 51, in import six ImportError: No module named six ``` e.g. https://github.com/firebase/firebase-cpp-sdk/runs/6034310890 --- external/pip_requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/external/pip_requirements.txt b/external/pip_requirements.txt index 5aaa75d727..946e204ddd 100644 --- a/external/pip_requirements.txt +++ b/external/pip_requirements.txt @@ -1,2 +1,3 @@ absl-py protobuf +six From c6346fd6c2153d3f04fb35b6da5192a2d70938b2 Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Fri, 15 Apr 2022 10:11:16 -0400 Subject: [PATCH 11/19] Improve FIREBASE_PYTHON_EXECUTABLE cmake cache var and make its usage ubiquitous --- CMakeLists.txt | 17 +++++++++++++++-- cmake/external/firestore.cmake | 3 +-- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6ca6fa75b8..8fd36b37ee 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -86,8 +86,21 @@ if (NOT FIREBASE_ANDROID_STL STREQUAL "") set(ANDROID_STL ${FIREBASE_ANDROID_STL}) endif() -set(FIREBASE_PYTHON_EXECUTABLE "python" CACHE FILEPATH - "The Python interpreter to use, such as one from a venv") +# Find a Python interpreter using the best available mechanism. +if(${CMAKE_VERSION} VERSION_LESS "3.12") + include(FindPythonInterp) + set(DEFAULT_FIREBASE_PYTHON_EXECUTABLE "${PYTHON_EXECUTABLE}") +else() + find_package(Python3 COMPONENTS Interpreter) + set(DEFAULT_FIREBASE_PYTHON_EXECUTABLE "${Python3_EXECUTABLE}") +endif() + +set( + FIREBASE_PYTHON_EXECUTABLE + "${DEFAULT_FIREBASE_PYTHON_EXECUTABLE}" + CACHE FILEPATH + "The Python interpreter to use" +) set(FIREBASE_XCODE_TARGET_FORMAT "frameworks" CACHE STRING "Format to output, 'frameworks' or 'libraries'") diff --git a/cmake/external/firestore.cmake b/cmake/external/firestore.cmake index 55d8268270..5c1e2b82a5 100644 --- a/cmake/external/firestore.cmake +++ b/cmake/external/firestore.cmake @@ -13,7 +13,6 @@ # limitations under the License. include(ExternalProject) -include(FindPythonInterp) if(TARGET firestore) return() @@ -33,6 +32,6 @@ ExternalProject_Add( BUILD_COMMAND "" INSTALL_COMMAND "" TEST_COMMAND "" - PATCH_COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_LIST_DIR}/firestore_patch.py --leveldb-version-from ${CMAKE_CURRENT_LIST_DIR}/leveldb.cmake + PATCH_COMMAND ${FIREBASE_PYTHON_EXECUTABLE} ${CMAKE_CURRENT_LIST_DIR}/firestore_patch.py --leveldb-version-from ${CMAKE_CURRENT_LIST_DIR}/leveldb.cmake HTTP_HEADER "${EXTERNAL_PROJECT_HTTP_HEADER}" ) From 5a8038d173167ce8f286d516c798e5d77b03e2aa Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Fri, 15 Apr 2022 10:15:16 -0400 Subject: [PATCH 12/19] firestore.cmake: bump pinned commit --- cmake/external/firestore.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/external/firestore.cmake b/cmake/external/firestore.cmake index 5c1e2b82a5..e9e700f3f0 100644 --- a/cmake/external/firestore.cmake +++ b/cmake/external/firestore.cmake @@ -23,7 +23,7 @@ ExternalProject_Add( DOWNLOAD_DIR ${FIREBASE_DOWNLOAD_DIR} GIT_REPOSITORY https://github.com/firebase/firebase-ios-sdk - GIT_TAG d57360f1bf2dd7ecb85a768e8912d0045b00b88d + GIT_TAG 3c718c3982a9c5e482a37213c8d8faf6a92ce8aa GIT_SHALLOW ON PREFIX ${PROJECT_BINARY_DIR} From b0ff4e87d1b95cc386050a72fa12dc41cfb596be Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Fri, 15 Apr 2022 10:15:30 -0400 Subject: [PATCH 13/19] Revert "external/pip_requirements.txt: add six" since it didn't fix the problem. This reverts commit 56553d996678430e756fa2eed6b469aaa8baf299. --- external/pip_requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/external/pip_requirements.txt b/external/pip_requirements.txt index 946e204ddd..5aaa75d727 100644 --- a/external/pip_requirements.txt +++ b/external/pip_requirements.txt @@ -1,3 +1,2 @@ absl-py protobuf -six From 05efff1a821b195667dcc14c6ece844fe3914de3 Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Fri, 15 Apr 2022 10:19:18 -0400 Subject: [PATCH 14/19] firestore.cmake: fix pinned commit --- cmake/external/firestore.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/external/firestore.cmake b/cmake/external/firestore.cmake index e9e700f3f0..e934eda75c 100644 --- a/cmake/external/firestore.cmake +++ b/cmake/external/firestore.cmake @@ -23,7 +23,7 @@ ExternalProject_Add( DOWNLOAD_DIR ${FIREBASE_DOWNLOAD_DIR} GIT_REPOSITORY https://github.com/firebase/firebase-ios-sdk - GIT_TAG 3c718c3982a9c5e482a37213c8d8faf6a92ce8aa + GIT_TAG 0ed860a0360735c28a26aad9cbe899695009a236 GIT_SHALLOW ON PREFIX ${PROJECT_BINARY_DIR} From 48a76809a2697aa6f79058e5d6394e31e5576613 Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Fri, 15 Apr 2022 17:29:14 -0400 Subject: [PATCH 15/19] Revert "Improve FIREBASE_PYTHON_EXECUTABLE cmake cache var and make its usage ubiquitous" It seemed to cause problems with the build. There is a better way in the iOS SDK anyways. This reverts commit c6346fd6c2153d3f04fb35b6da5192a2d70938b2. --- CMakeLists.txt | 17 ++--------------- cmake/external/firestore.cmake | 3 ++- 2 files changed, 4 insertions(+), 16 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8fd36b37ee..6ca6fa75b8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -86,21 +86,8 @@ if (NOT FIREBASE_ANDROID_STL STREQUAL "") set(ANDROID_STL ${FIREBASE_ANDROID_STL}) endif() -# Find a Python interpreter using the best available mechanism. -if(${CMAKE_VERSION} VERSION_LESS "3.12") - include(FindPythonInterp) - set(DEFAULT_FIREBASE_PYTHON_EXECUTABLE "${PYTHON_EXECUTABLE}") -else() - find_package(Python3 COMPONENTS Interpreter) - set(DEFAULT_FIREBASE_PYTHON_EXECUTABLE "${Python3_EXECUTABLE}") -endif() - -set( - FIREBASE_PYTHON_EXECUTABLE - "${DEFAULT_FIREBASE_PYTHON_EXECUTABLE}" - CACHE FILEPATH - "The Python interpreter to use" -) +set(FIREBASE_PYTHON_EXECUTABLE "python" CACHE FILEPATH + "The Python interpreter to use, such as one from a venv") set(FIREBASE_XCODE_TARGET_FORMAT "frameworks" CACHE STRING "Format to output, 'frameworks' or 'libraries'") diff --git a/cmake/external/firestore.cmake b/cmake/external/firestore.cmake index e934eda75c..530f78b4e1 100644 --- a/cmake/external/firestore.cmake +++ b/cmake/external/firestore.cmake @@ -13,6 +13,7 @@ # limitations under the License. include(ExternalProject) +include(FindPythonInterp) if(TARGET firestore) return() @@ -32,6 +33,6 @@ ExternalProject_Add( BUILD_COMMAND "" INSTALL_COMMAND "" TEST_COMMAND "" - PATCH_COMMAND ${FIREBASE_PYTHON_EXECUTABLE} ${CMAKE_CURRENT_LIST_DIR}/firestore_patch.py --leveldb-version-from ${CMAKE_CURRENT_LIST_DIR}/leveldb.cmake + PATCH_COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_LIST_DIR}/firestore_patch.py --leveldb-version-from ${CMAKE_CURRENT_LIST_DIR}/leveldb.cmake HTTP_HEADER "${EXTERNAL_PROJECT_HTTP_HEADER}" ) From bd56ad5445151a8c375ae14d04b50cbb12240a2e Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Fri, 15 Apr 2022 17:31:31 -0400 Subject: [PATCH 16/19] firestore.cmake: update pinned commit --- cmake/external/firestore.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/external/firestore.cmake b/cmake/external/firestore.cmake index 530f78b4e1..b21da9275d 100644 --- a/cmake/external/firestore.cmake +++ b/cmake/external/firestore.cmake @@ -24,7 +24,7 @@ ExternalProject_Add( DOWNLOAD_DIR ${FIREBASE_DOWNLOAD_DIR} GIT_REPOSITORY https://github.com/firebase/firebase-ios-sdk - GIT_TAG 0ed860a0360735c28a26aad9cbe899695009a236 + GIT_TAG 865162a0355a788728edf6798bcca5218dd50734 GIT_SHALLOW ON PREFIX ${PROJECT_BINARY_DIR} From 7e229e835a69943ad94c24aac82e4e00469f619b Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Fri, 15 Apr 2022 19:54:46 -0400 Subject: [PATCH 17/19] CMakeLists.txt: Set FIRESTORE_INCLUDE_OBJC as a cache variable so it actually works --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6ca6fa75b8..5d3410f251 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -217,7 +217,7 @@ endif() # Disable compiling the Objective-C (and Swift) stuff from the # firebase-ios-sdk since it's not needed and can sometimes fail to build. -set(FIRESTORE_INCLUDE_OBJC NO) +set(FIRESTORE_INCLUDE_OBJC OFF CACHE BOOL "Disabled for the CPP SDK") if(FIREBASE_CPP_USE_PRIOR_GRADLE_BUILD) # Quote meta characters in ${CMAKE_CURRENT_LIST_DIR} so we can From bf8bffd8bac7ae32005853470a864c21742f857a Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Sat, 16 Apr 2022 00:34:51 -0400 Subject: [PATCH 18/19] query_main.cc: IsBefore(BoundPosition) -> IsInclusive(BoundPosition) This fixes several test failures due to incorrect handling of startAfter and endBefore resulting from a change in core::Bound from "before" to "inclusive" in https://github.com/firebase/firebase-ios-sdk/pull/9519 Namely, this fixes the following failure: ``` cursor_test.cc:250: Failure Expected equality of these values: std::vector({"c", "f", "b", "e"}) Which is: { "c", "f", "b", "e" } QuerySnapshotToIds(snapshot) Which is: { "c", "f" } [ FAILED ] FirestoreIntegrationTest.TimestampsCanBePassedToQueriesAsLimits ``` e.g. https://github.com/firebase/firebase-cpp-sdk/runs/6044737005 --- firestore/src/main/query_main.cc | 10 +++++----- firestore/src/main/query_main.h | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/firestore/src/main/query_main.cc b/firestore/src/main/query_main.cc index 9ce4060fcc..d11c14002f 100644 --- a/firestore/src/main/query_main.cc +++ b/firestore/src/main/query_main.cc @@ -227,7 +227,7 @@ core::Bound QueryInternal::ToBound( components->values[i] = *DeepClone(*value).release(); } - return core::Bound::FromValue(std::move(components), IsBefore(bound_pos)); + return core::Bound::FromValue(std::move(components), IsInclusive(bound_pos)); } core::Bound QueryInternal::ToBound( @@ -261,7 +261,7 @@ core::Bound QueryInternal::ToBound( } } - return core::Bound::FromValue(std::move(components), IsBefore(bound_pos)); + return core::Bound::FromValue(std::move(components), IsInclusive(bound_pos)); } Message QueryInternal::ConvertDocumentId( @@ -327,14 +327,14 @@ api::Query QueryInternal::CreateQueryWithBound(BoundPosition bound_pos, FIRESTORE_UNREACHABLE(); } -bool QueryInternal::IsBefore(BoundPosition bound_pos) { +bool QueryInternal::IsInclusive(BoundPosition bound_pos) { switch (bound_pos) { case BoundPosition::kStartAt: - case BoundPosition::kEndBefore: + case BoundPosition::kEndAt: return true; case BoundPosition::kStartAfter: - case BoundPosition::kEndAt: + case BoundPosition::kEndBefore: return false; } diff --git a/firestore/src/main/query_main.h b/firestore/src/main/query_main.h index 17d13d8dfd..4f3e89016d 100644 --- a/firestore/src/main/query_main.h +++ b/firestore/src/main/query_main.h @@ -201,7 +201,7 @@ class QueryInternal { const nanopb::Message& from, const core::Query& internal_query) const; - static bool IsBefore(BoundPosition bound_pos); + static bool IsInclusive(BoundPosition bound_pos); core::Bound ToBound(BoundPosition bound_pos, const DocumentSnapshot& public_snapshot) const; From 3de38380765a64801ac4963c50ca837a43d6f75c Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Sat, 16 Apr 2022 00:41:08 -0400 Subject: [PATCH 19/19] firestore.cmake: update pinned commit --- cmake/external/firestore.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/external/firestore.cmake b/cmake/external/firestore.cmake index b21da9275d..28774952c1 100644 --- a/cmake/external/firestore.cmake +++ b/cmake/external/firestore.cmake @@ -24,7 +24,7 @@ ExternalProject_Add( DOWNLOAD_DIR ${FIREBASE_DOWNLOAD_DIR} GIT_REPOSITORY https://github.com/firebase/firebase-ios-sdk - GIT_TAG 865162a0355a788728edf6798bcca5218dd50734 + GIT_TAG 70aa8b82a2ed36dd14448174bea0fd7e575d4d49 GIT_SHALLOW ON PREFIX ${PROJECT_BINARY_DIR}