Skip to content

Commit

Permalink
Merge branch 'hn/reftable'
Browse files Browse the repository at this point in the history
The "reftable" backend for the refs API, without integrating into
the refs subsystem, has been added.

* hn/reftable:
  Add "test-tool dump-reftable" command.
  reftable: add dump utility
  reftable: implement stack, a mutable database of reftable files.
  reftable: implement refname validation
  reftable: add merged table view
  reftable: add a heap-based priority queue for reftable records
  reftable: reftable file level tests
  reftable: read reftable files
  reftable: generic interface to tables
  reftable: write reftable files
  reftable: a generic binary tree implementation
  reftable: reading/writing blocks
  Provide zlib's uncompress2 from compat/zlib-compat.c
  reftable: (de)serialization for the polymorphic record type.
  reftable: add blocksource, an abstraction for random access reads
  reftable: utility functions
  reftable: add error related functionality
  reftable: add LICENSE
  hash.h: provide constants for the hash IDs
  • Loading branch information
gitster committed Dec 15, 2021
2 parents e773545 + d860c86 commit a4bbd13
Show file tree
Hide file tree
Showing 69 changed files with 11,125 additions and 12 deletions.
53 changes: 50 additions & 3 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -256,6 +256,8 @@ all::
#
# Define NO_DEFLATE_BOUND if your zlib does not have deflateBound.
#
# Define NO_UNCOMPRESS2 if your zlib does not have uncompress2.
#
# Define NO_NORETURN if using buggy versions of gcc 4.6+ and profile feedback,
# as the compiler can crash (http://gcc.gnu.org/bugzilla/show_bug.cgi?id=49299)
#
Expand Down Expand Up @@ -732,6 +734,7 @@ TEST_BUILTINS_OBJS += test-read-cache.o
TEST_BUILTINS_OBJS += test-read-graph.o
TEST_BUILTINS_OBJS += test-read-midx.o
TEST_BUILTINS_OBJS += test-ref-store.o
TEST_BUILTINS_OBJS += test-reftable.o
TEST_BUILTINS_OBJS += test-regex.o
TEST_BUILTINS_OBJS += test-repository.o
TEST_BUILTINS_OBJS += test-revision-walking.o
Expand Down Expand Up @@ -810,6 +813,8 @@ TEST_SHELL_PATH = $(SHELL_PATH)

LIB_FILE = libgit.a
XDIFF_LIB = xdiff/lib.a
REFTABLE_LIB = reftable/libreftable.a
REFTABLE_TEST_LIB = reftable/libreftable_test.a

GENERATED_H += command-list.h
GENERATED_H += config-list.h
Expand Down Expand Up @@ -1189,7 +1194,7 @@ THIRD_PARTY_SOURCES += compat/regex/%
THIRD_PARTY_SOURCES += sha1collisiondetection/%
THIRD_PARTY_SOURCES += sha1dc/%

GITLIBS = common-main.o $(LIB_FILE) $(XDIFF_LIB)
GITLIBS = common-main.o $(LIB_FILE) $(XDIFF_LIB) $(REFTABLE_LIB)
EXTLIBS =

GIT_USER_AGENT = git/$(GIT_VERSION)
Expand Down Expand Up @@ -1720,6 +1725,11 @@ ifdef NO_DEFLATE_BOUND
BASIC_CFLAGS += -DNO_DEFLATE_BOUND
endif

ifdef NO_UNCOMPRESS2
BASIC_CFLAGS += -DNO_UNCOMPRESS2
REFTABLE_OBJS += compat/zlib-uncompress2.o
endif

ifdef NO_POSIX_GOODIES
BASIC_CFLAGS += -DNO_POSIX_GOODIES
endif
Expand Down Expand Up @@ -2431,7 +2441,36 @@ XDIFF_OBJS += xdiff/xutils.o
.PHONY: xdiff-objs
xdiff-objs: $(XDIFF_OBJS)

REFTABLE_OBJS += reftable/basics.o
REFTABLE_OBJS += reftable/error.o
REFTABLE_OBJS += reftable/block.o
REFTABLE_OBJS += reftable/blocksource.o
REFTABLE_OBJS += reftable/iter.o
REFTABLE_OBJS += reftable/publicbasics.o
REFTABLE_OBJS += reftable/merged.o
REFTABLE_OBJS += reftable/pq.o
REFTABLE_OBJS += reftable/reader.o
REFTABLE_OBJS += reftable/record.o
REFTABLE_OBJS += reftable/refname.o
REFTABLE_OBJS += reftable/generic.o
REFTABLE_OBJS += reftable/stack.o
REFTABLE_OBJS += reftable/tree.o
REFTABLE_OBJS += reftable/writer.o

REFTABLE_TEST_OBJS += reftable/basics_test.o
REFTABLE_TEST_OBJS += reftable/block_test.o
REFTABLE_TEST_OBJS += reftable/dump.o
REFTABLE_TEST_OBJS += reftable/merged_test.o
REFTABLE_TEST_OBJS += reftable/pq_test.o
REFTABLE_TEST_OBJS += reftable/record_test.o
REFTABLE_TEST_OBJS += reftable/readwrite_test.o
REFTABLE_TEST_OBJS += reftable/refname_test.o
REFTABLE_TEST_OBJS += reftable/stack_test.o
REFTABLE_TEST_OBJS += reftable/test_framework.o
REFTABLE_TEST_OBJS += reftable/tree_test.o

TEST_OBJS := $(patsubst %$X,%.o,$(TEST_PROGRAMS)) $(patsubst %,t/helper/%,$(TEST_BUILTINS_OBJS))

.PHONY: test-objs
test-objs: $(TEST_OBJS)

Expand All @@ -2447,6 +2486,8 @@ OBJECTS += $(PROGRAM_OBJS)
OBJECTS += $(TEST_OBJS)
OBJECTS += $(XDIFF_OBJS)
OBJECTS += $(FUZZ_OBJS)
OBJECTS += $(REFTABLE_OBJS) $(REFTABLE_TEST_OBJS)

ifndef NO_CURL
OBJECTS += http.o http-walker.o remote-curl.o
endif
Expand Down Expand Up @@ -2589,6 +2630,12 @@ $(LIB_FILE): $(LIB_OBJS)
$(XDIFF_LIB): $(XDIFF_OBJS)
$(QUIET_AR)$(RM) $@ && $(AR) $(ARFLAGS) $@ $^

$(REFTABLE_LIB): $(REFTABLE_OBJS)
$(QUIET_AR)$(RM) $@ && $(AR) $(ARFLAGS) $@ $^

$(REFTABLE_TEST_LIB): $(REFTABLE_TEST_OBJS)
$(QUIET_AR)$(RM) $@ && $(AR) $(ARFLAGS) $@ $^

export DEFAULT_EDITOR DEFAULT_PAGER

Documentation/GIT-EXCLUDED-PROGRAMS: FORCE
Expand Down Expand Up @@ -2887,7 +2934,7 @@ perf: all

t/helper/test-tool$X: $(patsubst %,t/helper/%,$(TEST_BUILTINS_OBJS))

t/helper/test-%$X: t/helper/test-%.o GIT-LDFLAGS $(GITLIBS)
t/helper/test-%$X: t/helper/test-%.o GIT-LDFLAGS $(GITLIBS) $(REFTABLE_TEST_LIB)
$(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) $(filter %.o,$^) $(filter %.a,$^) $(LIBS)

check-sha1:: t/helper/test-tool$X
Expand Down Expand Up @@ -3225,7 +3272,7 @@ cocciclean:
clean: profile-clean coverage-clean cocciclean
$(RM) *.res
$(RM) $(OBJECTS)
$(RM) $(LIB_FILE) $(XDIFF_LIB)
$(RM) $(LIB_FILE) $(XDIFF_LIB) $(REFTABLE_LIB) $(REFTABLE_TEST_LIB)
$(RM) $(ALL_PROGRAMS) $(SCRIPT_LIB) $(BUILT_INS) git$X
$(RM) $(TEST_PROGRAMS)
$(RM) $(FUZZ_PROGRAMS)
Expand Down
1 change: 1 addition & 0 deletions ci/lib.sh
Original file line number Diff line number Diff line change
Expand Up @@ -224,6 +224,7 @@ linux-gcc-default)
;;
Linux32)
CC=gcc
MAKEFLAGS="$MAKEFLAGS NO_UNCOMPRESS2=1"
;;
linux-musl)
CC=gcc
Expand Down
1 change: 1 addition & 0 deletions compat/.gitattributes
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
/zlib-uncompress2.c whitespace=-indent-with-non-tab,-trailing-space
95 changes: 95 additions & 0 deletions compat/zlib-uncompress2.c
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
/* taken from zlib's uncompr.c
commit cacf7f1d4e3d44d871b605da3b647f07d718623f
Author: Mark Adler <madler@alumni.caltech.edu>
Date: Sun Jan 15 09:18:46 2017 -0800
zlib 1.2.11
*/

#include "../reftable/system.h"
#define z_const

/*
* Copyright (C) 1995-2003, 2010, 2014, 2016 Jean-loup Gailly, Mark Adler
* For conditions of distribution and use, see copyright notice in zlib.h
*/

#include <zlib.h>

/* clang-format off */

/* ===========================================================================
Decompresses the source buffer into the destination buffer. *sourceLen is
the byte length of the source buffer. Upon entry, *destLen is the total size
of the destination buffer, which must be large enough to hold the entire
uncompressed data. (The size of the uncompressed data must have been saved
previously by the compressor and transmitted to the decompressor by some
mechanism outside the scope of this compression library.) Upon exit,
*destLen is the size of the decompressed data and *sourceLen is the number
of source bytes consumed. Upon return, source + *sourceLen points to the
first unused input byte.
uncompress returns Z_OK if success, Z_MEM_ERROR if there was not enough
memory, Z_BUF_ERROR if there was not enough room in the output buffer, or
Z_DATA_ERROR if the input data was corrupted, including if the input data is
an incomplete zlib stream.
*/
int ZEXPORT uncompress2 (
Bytef *dest,
uLongf *destLen,
const Bytef *source,
uLong *sourceLen) {
z_stream stream;
int err;
const uInt max = (uInt)-1;
uLong len, left;
Byte buf[1]; /* for detection of incomplete stream when *destLen == 0 */

len = *sourceLen;
if (*destLen) {
left = *destLen;
*destLen = 0;
}
else {
left = 1;
dest = buf;
}

stream.next_in = (z_const Bytef *)source;
stream.avail_in = 0;
stream.zalloc = (alloc_func)0;
stream.zfree = (free_func)0;
stream.opaque = (voidpf)0;

err = inflateInit(&stream);
if (err != Z_OK) return err;

stream.next_out = dest;
stream.avail_out = 0;

do {
if (stream.avail_out == 0) {
stream.avail_out = left > (uLong)max ? max : (uInt)left;
left -= stream.avail_out;
}
if (stream.avail_in == 0) {
stream.avail_in = len > (uLong)max ? max : (uInt)len;
len -= stream.avail_in;
}
err = inflate(&stream, Z_NO_FLUSH);
} while (err == Z_OK);

*sourceLen -= len + stream.avail_in;
if (dest != buf)
*destLen = stream.total_out;
else if (stream.total_out && err == Z_BUF_ERROR)
left = 1;

inflateEnd(&stream);
return err == Z_STREAM_END ? Z_OK :
err == Z_NEED_DICT ? Z_DATA_ERROR :
err == Z_BUF_ERROR && left + stream.avail_out ? Z_DATA_ERROR :
err;
}
5 changes: 5 additions & 0 deletions config.mak.uname
Original file line number Diff line number Diff line change
Expand Up @@ -261,6 +261,10 @@ ifeq ($(uname_S),FreeBSD)
FILENO_IS_A_MACRO = UnfortunatelyYes
endif
ifeq ($(uname_S),OpenBSD)
# Versions < 7.0 need compatibility layer
ifeq ($(shell expr "$(uname_R)" : "[1-6]\."),2)
NO_UNCOMPRESS2 = UnfortunatelyYes
endif
NO_STRCASESTR = YesPlease
NO_MEMMEM = YesPlease
USE_ST_TIMESPEC = YesPlease
Expand Down Expand Up @@ -516,6 +520,7 @@ ifeq ($(uname_S),Interix)
endif
endif
ifeq ($(uname_S),Minix)
NO_UNCOMPRESS2 = YesPlease
NO_IPV6 = YesPlease
NO_ST_BLOCKS_IN_STRUCT_STAT = YesPlease
NO_NSEC = YesPlease
Expand Down
13 changes: 13 additions & 0 deletions configure.ac
Original file line number Diff line number Diff line change
Expand Up @@ -664,9 +664,22 @@ AC_LINK_IFELSE([ZLIBTEST_SRC],
NO_DEFLATE_BOUND=yes])
LIBS="$old_LIBS"

AC_DEFUN([ZLIBTEST_UNCOMPRESS2_SRC], [
AC_LANG_PROGRAM([#include <zlib.h>],
[uncompress2(NULL,NULL,NULL,NULL);])])
AC_MSG_CHECKING([for uncompress2 in -lz])
old_LIBS="$LIBS"
LIBS="$LIBS -lz"
AC_LINK_IFELSE([ZLIBTEST_UNCOMPRESS2_SRC],
[AC_MSG_RESULT([yes])],
[AC_MSG_RESULT([no])
NO_UNCOMPRESS2=yes])
LIBS="$old_LIBS"

GIT_UNSTASH_FLAGS($ZLIB_PATH)

GIT_CONF_SUBST([NO_DEFLATE_BOUND])
GIT_CONF_SUBST([NO_UNCOMPRESS2])

#
# Define NEEDS_SOCKET if linking with libc is not enough (SunOS,
Expand Down
14 changes: 12 additions & 2 deletions contrib/buildsystems/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -647,6 +647,12 @@ parse_makefile_for_sources(libxdiff_SOURCES "XDIFF_OBJS")
list(TRANSFORM libxdiff_SOURCES PREPEND "${CMAKE_SOURCE_DIR}/")
add_library(xdiff STATIC ${libxdiff_SOURCES})

#reftable
parse_makefile_for_sources(reftable_SOURCES "REFTABLE_OBJS")

list(TRANSFORM reftable_SOURCES PREPEND "${CMAKE_SOURCE_DIR}/")
add_library(reftable STATIC ${reftable_SOURCES})

if(WIN32)
if(NOT MSVC)#use windres when compiling with gcc and clang
add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/git.res
Expand All @@ -669,7 +675,7 @@ endif()
#link all required libraries to common-main
add_library(common-main OBJECT ${CMAKE_SOURCE_DIR}/common-main.c)

target_link_libraries(common-main libgit xdiff ${ZLIB_LIBRARIES})
target_link_libraries(common-main libgit xdiff reftable ${ZLIB_LIBRARIES})
if(Intl_FOUND)
target_link_libraries(common-main ${Intl_LIBRARIES})
endif()
Expand Down Expand Up @@ -908,11 +914,15 @@ if(BUILD_TESTING)
add_executable(test-fake-ssh ${CMAKE_SOURCE_DIR}/t/helper/test-fake-ssh.c)
target_link_libraries(test-fake-ssh common-main)

#reftable-tests
parse_makefile_for_sources(test-reftable_SOURCES "REFTABLE_TEST_OBJS")
list(TRANSFORM test-reftable_SOURCES PREPEND "${CMAKE_SOURCE_DIR}/")

#test-tool
parse_makefile_for_sources(test-tool_SOURCES "TEST_BUILTINS_OBJS")

list(TRANSFORM test-tool_SOURCES PREPEND "${CMAKE_SOURCE_DIR}/t/helper/")
add_executable(test-tool ${CMAKE_SOURCE_DIR}/t/helper/test-tool.c ${test-tool_SOURCES})
add_executable(test-tool ${CMAKE_SOURCE_DIR}/t/helper/test-tool.c ${test-tool_SOURCES} ${test-reftable_SOURCES})
target_link_libraries(test-tool common-main)

set_target_properties(test-fake-ssh test-tool
Expand Down
11 changes: 10 additions & 1 deletion contrib/buildsystems/Generators/Vcxproj.pm
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ sub createProject {
my $libs_release = "\n ";
my $libs_debug = "\n ";
if (!$static_library) {
$libs_release = join(";", sort(grep /^(?!libgit\.lib|xdiff\/lib\.lib|vcs-svn\/lib\.lib)/, @{$$build_structure{"$prefix${name}_LIBS"}}));
$libs_release = join(";", sort(grep /^(?!libgit\.lib|xdiff\/lib\.lib|vcs-svn\/lib\.lib|reftable\/libreftable\.lib)/, @{$$build_structure{"$prefix${name}_LIBS"}}));
$libs_debug = $libs_release;
$libs_debug =~ s/zlib\.lib/zlibd\.lib/g;
$libs_debug =~ s/libexpat\.lib/libexpatd\.lib/g;
Expand Down Expand Up @@ -232,6 +232,7 @@ EOM
EOM
if (!$static_library || $target =~ 'vcs-svn' || $target =~ 'xdiff') {
my $uuid_libgit = $$build_structure{"LIBS_libgit_GUID"};
my $uuid_libreftable = $$build_structure{"LIBS_reftable/libreftable_GUID"};
my $uuid_xdiff_lib = $$build_structure{"LIBS_xdiff/lib_GUID"};

print F << "EOM";
Expand All @@ -241,6 +242,14 @@ EOM
<ReferenceOutputAssembly>false</ReferenceOutputAssembly>
</ProjectReference>
EOM
if (!($name =~ /xdiff|libreftable/)) {
print F << "EOM";
<ProjectReference Include="$cdup\\reftable\\libreftable\\libreftable.vcxproj">
<Project>$uuid_libreftable</Project>
<ReferenceOutputAssembly>false</ReferenceOutputAssembly>
</ProjectReference>
EOM
}
if (!($name =~ 'xdiff')) {
print F << "EOM";
<ProjectReference Include="$cdup\\xdiff\\lib\\xdiff_lib.vcxproj">
Expand Down
6 changes: 6 additions & 0 deletions hash.h
Original file line number Diff line number Diff line change
Expand Up @@ -95,12 +95,18 @@ static inline void git_SHA256_Clone(git_SHA256_CTX *dst, const git_SHA256_CTX *s
/* Number of algorithms supported (including unknown). */
#define GIT_HASH_NALGOS (GIT_HASH_SHA256 + 1)

/* "sha1", big-endian */
#define GIT_SHA1_FORMAT_ID 0x73686131

/* The length in bytes and in hex digits of an object name (SHA-1 value). */
#define GIT_SHA1_RAWSZ 20
#define GIT_SHA1_HEXSZ (2 * GIT_SHA1_RAWSZ)
/* The block size of SHA-1. */
#define GIT_SHA1_BLKSZ 64

/* "s256", big-endian */
#define GIT_SHA256_FORMAT_ID 0x73323536

/* The length in bytes and in hex digits of an object name (SHA-256 value). */
#define GIT_SHA256_RAWSZ 32
#define GIT_SHA256_HEXSZ (2 * GIT_SHA256_RAWSZ)
Expand Down
7 changes: 2 additions & 5 deletions object-file.c
Original file line number Diff line number Diff line change
Expand Up @@ -165,7 +165,6 @@ static void git_hash_unknown_final_oid(struct object_id *oid, git_hash_ctx *ctx)
BUG("trying to finalize unknown hash");
}


const struct git_hash_algo hash_algos[GIT_HASH_NALGOS] = {
{
NULL,
Expand All @@ -184,8 +183,7 @@ const struct git_hash_algo hash_algos[GIT_HASH_NALGOS] = {
},
{
"sha1",
/* "sha1", big-endian */
0x73686131,
GIT_SHA1_FORMAT_ID,
GIT_SHA1_RAWSZ,
GIT_SHA1_HEXSZ,
GIT_SHA1_BLKSZ,
Expand All @@ -200,8 +198,7 @@ const struct git_hash_algo hash_algos[GIT_HASH_NALGOS] = {
},
{
"sha256",
/* "s256", big-endian */
0x73323536,
GIT_SHA256_FORMAT_ID,
GIT_SHA256_RAWSZ,
GIT_SHA256_HEXSZ,
GIT_SHA256_BLKSZ,
Expand Down

0 comments on commit a4bbd13

Please sign in to comment.