Skip to content

Commit

Permalink
Add basic tracing functionality
Browse files Browse the repository at this point in the history
  • Loading branch information
terrelln committed Feb 6, 2021
1 parent e926e9c commit 54a4998
Show file tree
Hide file tree
Showing 20 changed files with 521 additions and 19 deletions.
4 changes: 4 additions & 0 deletions build/VS2008/zstd/zstd.vcproj
Original file line number Diff line number Diff line change
Expand Up @@ -480,6 +480,10 @@
RelativePath="..\..\..\programs\zstdcli.c"
>
</File>
<File
RelativePath="..\..\..\programs\zstdcli_trace.c"
>
</File>
<File
RelativePath="..\..\..\lib\compress\zstdmt_compress.c"
>
Expand Down
1 change: 1 addition & 0 deletions build/VS2010/zstd/zstd.vcxproj
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@
<ClCompile Include="..\..\..\programs\dibio.c" />
<ClCompile Include="..\..\..\programs\fileio.c" />
<ClCompile Include="..\..\..\programs\zstdcli.c" />
<ClCompile Include="..\..\..\programs\zstdcli_trace.c" />
</ItemGroup>
<ItemGroup>
<ClInclude Include="..\..\..\lib\common\pool.h" />
Expand Down
4 changes: 2 additions & 2 deletions build/cmake/programs/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ if (MSVC)
set(PlatformDependResources ${MSVC_RESOURCE_DIR}/zstd.rc)
endif ()

add_executable(zstd ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${PROGRAMS_DIR}/fileio.c ${PROGRAMS_DIR}/benchfn.c ${PROGRAMS_DIR}/benchzstd.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/dibio.c ${PlatformDependResources})
add_executable(zstd ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${PROGRAMS_DIR}/fileio.c ${PROGRAMS_DIR}/benchfn.c ${PROGRAMS_DIR}/benchzstd.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/dibio.c ${PROGRAMS_DIR}/zstdcli_trace.c ${PlatformDependResources})
target_link_libraries(zstd ${PROGRAMS_ZSTD_LINK_TARGET})
if (CMAKE_SYSTEM_NAME MATCHES "(Solaris|SunOS)")
target_link_libraries(zstd rt)
Expand Down Expand Up @@ -75,7 +75,7 @@ if (UNIX)

add_executable(zstd-frugal ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${PROGRAMS_DIR}/fileio.c)
target_link_libraries(zstd-frugal ${PROGRAMS_ZSTD_LINK_TARGET})
set_property(TARGET zstd-frugal APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_NOBENCH;ZSTD_NODICT")
set_property(TARGET zstd-frugal APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_NOBENCH;ZSTD_NODICT;ZSTD_NOTRACE")
endif ()

# Add multi-threading support definitions
Expand Down
5 changes: 3 additions & 2 deletions build/meson/programs/meson.build
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,8 @@ zstd_programs_sources = [join_paths(zstd_rootdir, 'programs/zstdcli.c'),
join_paths(zstd_rootdir, 'programs/benchfn.c'),
join_paths(zstd_rootdir, 'programs/benchzstd.c'),
join_paths(zstd_rootdir, 'programs/datagen.c'),
join_paths(zstd_rootdir, 'programs/dibio.c')]
join_paths(zstd_rootdir, 'programs/dibio.c'),
join_paths(zstd_rootdir, 'programs/zstdcli_trace.c')]

zstd_c_args = libzstd_debug_cflags
if use_multi_thread
Expand Down Expand Up @@ -73,7 +74,7 @@ zstd_frugal_sources = [join_paths(zstd_rootdir, 'programs/zstdcli.c'),
executable('zstd-frugal',
zstd_frugal_sources,
dependencies: libzstd_dep,
c_args: [ '-DZSTD_NOBENCH', '-DZSTD_NODICT' ],
c_args: [ '-DZSTD_NOBENCH', '-DZSTD_NODICT', '-DZSTD_NOTRACE' ],
install: true)

install_data(join_paths(zstd_rootdir, 'programs/zstdgrep'),
Expand Down
4 changes: 3 additions & 1 deletion contrib/linux-kernel/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,9 @@ libzstd:
-U_MSC_VER \
-U_WIN32 \
-RZSTDLIB_VISIBILITY= \
-RZSTDERRORLIB_VISIBILITY=
-RZSTDERRORLIB_VISIBILITY= \
-DZSTD_HAVE_WEAK_SYMBOLS=0 \
-DZSTD_TRACE=0
cp linux_zstd.h linux/include/linux/zstd.h
cp zstd_compress_module.c linux/lib/zstd
cp zstd_decompress_module.c linux/lib/zstd
Expand Down
1 change: 1 addition & 0 deletions contrib/single_file_libs/zstd-in.c
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@
#ifndef __EMSCRIPTEN__
#define ZSTD_MULTITHREAD
#endif
#define ZSTD_TRACE 0

/* Include zstd_deps.h first with all the options we need enabled. */
#define ZSTD_DEPS_NEED_MALLOC
Expand Down
1 change: 1 addition & 0 deletions contrib/single_file_libs/zstddeclib-in.c
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
#define XXH_INLINE_ALL
#define ZSTD_LEGACY_SUPPORT 0
#define ZSTD_STRIP_ERROR_STRINGS
#define ZSTD_TRACE 0

/* Include zstd_deps.h first with all the options we need enabled. */
#define ZSTD_DEPS_NEED_MALLOC
Expand Down
1 change: 1 addition & 0 deletions lib/common/compiler.h
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,7 @@
# endif
#endif


/* target attribute */
#ifndef __has_attribute
#define __has_attribute(x) 0 /* Compatibility with non-clang compilers. */
Expand Down
42 changes: 42 additions & 0 deletions lib/common/zstd_trace.c
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
/*
* Copyright (c) 2016-2021, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/

#include "zstd_trace.h"
#include "../zstd.h"

#include "compiler.h"

#if ZSTD_TRACE && ZSTD_HAVE_WEAK_SYMBOLS

ZSTD_WEAK_ATTR int ZSTD_trace_compress_begin(ZSTD_CCtx const* cctx)
{
(void)cctx;
return 0;
}

ZSTD_WEAK_ATTR void ZSTD_trace_compress_end(ZSTD_CCtx const* cctx, ZSTD_trace const* trace)
{
(void)cctx;
(void)trace;
}

ZSTD_WEAK_ATTR int ZSTD_trace_decompress_begin(ZSTD_DCtx const* dctx)
{
(void)dctx;
return 0;
}

ZSTD_WEAK_ATTR void ZSTD_trace_decompress_end(ZSTD_DCtx const* dctx, ZSTD_trace const* trace)
{
(void)dctx;
(void)trace;
}

#endif
118 changes: 118 additions & 0 deletions lib/common/zstd_trace.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,118 @@
/*
* Copyright (c) 2016-2021, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/

#ifndef ZSTD_TRACE_H
#define ZSTD_TRACE_H

#include <stddef.h>

/* weak symbol support */
#if !defined(ZSTD_HAVE_WEAK_SYMBOLS) && defined(__GNUC__) && \
!defined(__APPLE__) && !defined(_WIN32) && !defined(__MINGW32__) && \
!defined(__CYGWIN__)
# define ZSTD_HAVE_WEAK_SYMBOLS 1
#else
# define ZSTD_HAVE_WEAK_SYMBOLS 0
#endif
#if ZSTD_HAVE_WEAK_SYMBOLS
# define ZSTD_WEAK_ATTR __attribute__((__weak__))
#else
# define ZSTD_WEAK_ATTR
#endif

/* Only enable tracing when weak symbols are available. */
#ifndef ZSTD_TRACE
# define ZSTD_TRACE ZSTD_HAVE_WEAK_SYMBOLS
#endif

#if ZSTD_TRACE

struct ZSTD_CCtx_s;
struct ZSTD_DCtx_s;
struct ZSTD_CCtx_params_s;

typedef struct {
/**
* ZSTD_VERSION_NUMBER
*
* This is guaranteed to be the first member of ZSTD_trace.
* Otherwise, this struct is not stable between versions. If
* the version number does not match your expectation, you
* should not interpret the rest of the struct.
*/
unsigned version;
/**
* Non-zero if streaming (de)compression is used.
*/
unsigned streaming;
/**
* The dictionary ID.
*/
unsigned dictionaryID;
/**
* Is the dictionary cold?
* Only set on decompression.
*/
unsigned dictionaryIsCold;
/**
* The dictionary size or zero if no dictionary.
*/
size_t dictionarySize;
/**
* The uncompressed size of the data.
*/
size_t uncompressedSize;
/**
* The compressed size of the data.
*/
size_t compressedSize;
/**
* The fully resolved CCtx parameters (NULL on decompression).
*/
struct ZSTD_CCtx_params_s const* params;
} ZSTD_trace;

/**
* Trace the beginning of a compression call.
* @param cctx The dctx pointer for the compression.
* It can be used as a key to map begin() to end().
* @returns Non-zero if tracing is enabled.
*/
int ZSTD_trace_compress_begin(struct ZSTD_CCtx_s const* cctx);

/**
* Trace the end of a compression call.
* @param cctx The dctx pointer for the decompression.
* @param trace The zstd tracing info.
*/
void ZSTD_trace_compress_end(
struct ZSTD_CCtx_s const* cctx,
ZSTD_trace const* trace);

/**
* Trace the beginning of a decompression call.
* @param dctx The dctx pointer for the decompression.
* It can be used as a key to map begin() to end().
* @returns Non-zero if tracing is enabled.
*/
int ZSTD_trace_decompress_begin(struct ZSTD_DCtx_s const* dctx);

/**
* Trace the end of a decompression call.
* @param dctx The dctx pointer for the decompression.
* @param trace The zstd tracing info.
*/
void ZSTD_trace_decompress_end(
struct ZSTD_DCtx_s const* dctx,
ZSTD_trace const* trace);

#endif /* ZSTD_TRACE */

#endif /* ZSTD_TRACE_H */
45 changes: 44 additions & 1 deletion lib/compress/zstd_compress.c
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
#include "../common/zstd_deps.h" /* INT_MAX, ZSTD_memset, ZSTD_memcpy */
#include "../common/cpu.h"
#include "../common/mem.h"
#include "../common/zstd_trace.h"
#include "hist.h" /* HIST_countFast_wksp */
#define FSE_STATIC_LINKING_ONLY /* FSE_encodeSymbol */
#include "../common/fse.h"
Expand Down Expand Up @@ -802,7 +803,7 @@ size_t ZSTD_CCtx_getParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int* value
}

size_t ZSTD_CCtxParams_getParameter(
ZSTD_CCtx_params* CCtxParams, ZSTD_cParameter param, int* value)
ZSTD_CCtx_params const* CCtxParams, ZSTD_cParameter param, int* value)
{
switch(param)
{
Expand Down Expand Up @@ -1704,6 +1705,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
XXH64_reset(&zc->xxhState, 0);
zc->stage = ZSTDcs_init;
zc->dictID = 0;
zc->dictContentSize = 0;

ZSTD_reset_compressedBlockState(zc->blockState.prevCBlock);

Expand Down Expand Up @@ -1864,6 +1866,7 @@ ZSTD_resetCCtx_byAttachingCDict(ZSTD_CCtx* cctx,
} }

cctx->dictID = cdict->dictID;
cctx->dictContentSize = cdict->dictContentSize;

/* copy block state */
ZSTD_memcpy(cctx->blockState.prevCBlock, &cdict->cBlockState, sizeof(cdict->cBlockState));
Expand Down Expand Up @@ -1927,6 +1930,7 @@ static size_t ZSTD_resetCCtx_byCopyingCDict(ZSTD_CCtx* cctx,
}

cctx->dictID = cdict->dictID;
cctx->dictContentSize = cdict->dictContentSize;

/* copy block state */
ZSTD_memcpy(cctx->blockState.prevCBlock, &cdict->cBlockState, sizeof(cdict->cBlockState));
Expand Down Expand Up @@ -2017,6 +2021,7 @@ static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx,
dstMatchState->loadedDictEnd= srcMatchState->loadedDictEnd;
}
dstCCtx->dictID = srcCCtx->dictID;
dstCCtx->dictContentSize = srcCCtx->dictContentSize;

/* copy block state */
ZSTD_memcpy(dstCCtx->blockState.prevCBlock, srcCCtx->blockState.prevCBlock, sizeof(*srcCCtx->blockState.prevCBlock));
Expand Down Expand Up @@ -3380,6 +3385,9 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx,
const ZSTD_CCtx_params* params, U64 pledgedSrcSize,
ZSTD_buffered_policy_e zbuff)
{
#if ZSTD_TRACE
cctx->tracingEnabled = ZSTD_trace_compress_begin(cctx);
#endif
DEBUGLOG(4, "ZSTD_compressBegin_internal: wlog=%u", params->cParams.windowLog);
/* params are supposed to be fully validated at this point */
assert(!ZSTD_isError(ZSTD_checkCParams(params->cParams)));
Expand Down Expand Up @@ -3409,6 +3417,7 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx,
FORWARD_IF_ERROR(dictID, "ZSTD_compress_insertDictionary failed");
assert(dictID <= UINT_MAX);
cctx->dictID = (U32)dictID;
cctx->dictContentSize = cdict ? cdict->dictContentSize : dictSize;
}
return 0;
}
Expand Down Expand Up @@ -3503,6 +3512,28 @@ static size_t ZSTD_writeEpilogue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity)
return op-ostart;
}

static void ZSTD_CCtx_trace(ZSTD_CCtx const* cctx, size_t extraCSize)
{
#if ZSTD_TRACE
if (cctx->tracingEnabled) {
int const streaming = cctx->inBuffSize > 0 || cctx->outBuffSize > 0 || cctx->appliedParams.nbWorkers > 0;
ZSTD_trace trace;
ZSTD_memset(&trace, 0, sizeof(trace));
trace.version = ZSTD_VERSION_NUMBER;
trace.streaming = streaming;
trace.dictionaryID = cctx->dictID;
trace.dictionarySize = cctx->dictContentSize;
trace.uncompressedSize = cctx->consumedSrcSize;
trace.compressedSize = cctx->producedCSize + extraCSize;
trace.params = &cctx->appliedParams;
ZSTD_trace_compress_end(cctx, &trace);
}
#else
(void)cctx;
(void)extraCSize;
#endif
}

size_t ZSTD_compressEnd (ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize)
Expand All @@ -3525,6 +3556,7 @@ size_t ZSTD_compressEnd (ZSTD_CCtx* cctx,
(unsigned)cctx->pledgedSrcSizePlusOne-1,
(unsigned)cctx->consumedSrcSize);
}
ZSTD_CCtx_trace(cctx, endResult);
return cSize + endResult;
}

Expand Down Expand Up @@ -4408,6 +4440,9 @@ static size_t ZSTD_CCtx_init_compressStream2(ZSTD_CCtx* cctx,
params.nbWorkers = 0; /* do not invoke multi-threading when src size is too small */
}
if (params.nbWorkers > 0) {
#if ZSTD_TRACE
cctx->tracingEnabled = ZSTD_trace_compress_begin(cctx);
#endif
/* mt context creation */
if (cctx->mtctx == NULL) {
DEBUGLOG(4, "ZSTD_compressStream2: creating new mtctx for nbWorkers=%u",
Expand All @@ -4421,6 +4456,10 @@ static size_t ZSTD_CCtx_init_compressStream2(ZSTD_CCtx* cctx,
cctx->mtctx,
prefixDict.dict, prefixDict.dictSize, prefixDict.dictContentType,
cctx->cdict, params, cctx->pledgedSrcSizePlusOne-1) , "");
cctx->dictID = cctx->cdict ? cctx->cdict->dictID : 0;
cctx->dictContentSize = cctx->cdict ? cctx->cdict->dictContentSize : prefixDict.dictSize;
cctx->consumedSrcSize = 0;
cctx->producedCSize = 0;
cctx->streamStage = zcss_load;
cctx->appliedParams = params;
} else
Expand Down Expand Up @@ -4482,8 +4521,12 @@ size_t ZSTD_compressStream2( ZSTD_CCtx* cctx,
size_t const ipos = input->pos;
size_t const opos = output->pos;
flushMin = ZSTDMT_compressStream_generic(cctx->mtctx, output, input, endOp);
cctx->consumedSrcSize += (U64)(input->pos - ipos);
cctx->producedCSize += (U64)(output->pos - opos);
if ( ZSTD_isError(flushMin)
|| (endOp == ZSTD_e_end && flushMin == 0) ) { /* compression completed */
if (flushMin == 0)
ZSTD_CCtx_trace(cctx, 0);
ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only);
}
FORWARD_IF_ERROR(flushMin, "ZSTDMT_compressStream_generic failed");
Expand Down
Loading

0 comments on commit 54a4998

Please sign in to comment.