From 8b4e84249b1f6eba4ace7de3335cc2fdd34cd651 Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Thu, 6 Oct 2016 18:17:08 -0700
Subject: [PATCH] [pzstd] Fix Makefile

---
 .gitignore                        |   1 +
 .travis.yml                       |   4 +-
 appveyor.yml                      |   5 +-
 contrib/pzstd/Makefile            | 298 +++++++++++++++++++++---------
 contrib/pzstd/test/Makefile       |  48 -----
 contrib/pzstd/utils/test/Makefile |  42 -----
 6 files changed, 218 insertions(+), 180 deletions(-)
 delete mode 100644 contrib/pzstd/test/Makefile
 delete mode 100644 contrib/pzstd/utils/test/Makefile

diff --git a/.gitignore b/.gitignore
index c939f1228ef..220a1e0eb1a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -32,3 +32,4 @@ _zstdbench/
 *.swp
 .DS_Store
 googletest/
+*.d
diff --git a/.travis.yml b/.travis.yml
index 972c1b4ba7e..f328f480d64 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -22,7 +22,7 @@ matrix:
           packages:
             - gcc-4.8
             - g++-4.8
-      env: PLATFORM="Ubuntu 12.04 container" CMD="make zlibwrapper && make clean && make -C tests test-zstd_nolegacy && make clean && make clean && make cmaketest && make clean && make -C contrib/pzstd pzstd && make -C contrib/pzstd googletest && make -C contrib/pzstd test && make -C contrib/pzstd clean"
+      env: PLATFORM="Ubuntu 12.04 container" CMD="make zlibwrapper && make clean && make -C tests test-zstd_nolegacy && make clean && make clean && make cmaketest && make clean && make -C contrib/pzstd googletest && make -C contrib/pzstd all && make -C contrib/pzstd check && make -C contrib/pzstd clean"
     - os: linux
       sudo: false
       env: PLATFORM="Ubuntu 12.04 container" CMD="make usan"
@@ -69,7 +69,7 @@ matrix:
       sudo: required
       install:
         - export CXX="g++-4.8" CC="gcc-4.8"
-      env: PLATFORM="Ubuntu 14.04" CMD="make gpptest && make clean && make gnu90test && make clean && make c99test && make clean && make gnu99test && make clean && make clangtest && make clean && make -C contrib/pzstd pzstd32 && make -C contrib/pzstd googletest32 && make -C contrib/pzstd test32 && make -C contrib/pzstd clean"
+      env: PLATFORM="Ubuntu 14.04" CMD="make gpptest && make clean && make gnu90test && make clean && make c99test && make clean && make gnu99test && make clean && make clangtest && make clean && make -C contrib/pzstd googletest32 && make -C contrib/pzstd all32 && make -C contrib/pzstd check && make -C contrib/pzstd clean"
       addons:
         apt:
           packages:
diff --git a/appveyor.yml b/appveyor.yml
index 6345c7b3964..fbdc30c40b5 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -50,10 +50,9 @@ build_script:
       ECHO *** &&
       ECHO *** Building pzstd for %PLATFORM% &&
       ECHO *** &&
-      ECHO make -C contrib\pzstd pzstd &&
-      make -C contrib\pzstd pzstd &&
       make -C contrib\pzstd googletest-mingw64 &&
-      make -C contrib\pzstd test &&
+      make -C contrib\pzstd all &&
+      make -C contrib\pzstd check &&
       make -C contrib\pzstd clean
     )
   - if [%COMPILER%]==[gcc] (
diff --git a/contrib/pzstd/Makefile b/contrib/pzstd/Makefile
index e30be0beddc..b998c1fff68 100644
--- a/contrib/pzstd/Makefile
+++ b/contrib/pzstd/Makefile
@@ -7,20 +7,69 @@
 # of patent rights can be found in the PATENTS file in the same directory.
 # ##########################################################################
 
+# Standard variables for installation
+DESTDIR ?=
+PREFIX  ?= /usr/local
+BINDIR  := $(DESTDIR)$(PREFIX)/bin
+
 ZSTDDIR = ../../lib
 PROGDIR = ../../programs
 
-CPPFLAGS = -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(PROGDIR) -I.
-CXXFLAGS  ?= -O3
-CXXFLAGS  += -std=c++11
-CXXFLAGS  += $(MOREFLAGS)
-FLAGS    = $(CPPFLAGS) $(CXXFLAGS) $(LDFLAGS)
-
-
-ZSTDCOMMON_FILES := $(ZSTDDIR)/common/*.c
-ZSTDCOMP_FILES := $(ZSTDDIR)/compress/zstd_compress.c $(ZSTDDIR)/compress/fse_compress.c $(ZSTDDIR)/compress/huf_compress.c
-ZSTDDECOMP_FILES := $(ZSTDDIR)/decompress/huf_decompress.c
-ZSTD_FILES := $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES)
+# External program to use to run tests, e.g. qemu or valgrind
+TESTPROG  ?=
+# Flags to pass to the tests
+TESTFLAGS ?=
+
+# We use gcc/clang to generate the header dependencies of files
+DEPFLAGS = -MMD -MP -MF $*.Td
+POSTCOMPILE = mv -f $*.Td $*.d
+
+# CFLAGS, CXXFLAGS, and LDFLAGS are for the users to override
+CFLAGS   ?= -O3 -Wall -Wextra
+CXXFLAGS ?= -O3 -Wall -Wextra -pedantic -std=c++11
+LDFLAGS  ?=
+
+# Googletest default flags
+PZSTD_INC  = -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(PROGDIR) -I.
+GTEST_INC  = -isystem googletest/googletest/include
+
+PZSTD_CCXXFLAGS = $(PZSTD_INC) $(GTEST_INC)
+PZSTD_CFLAGS    = $(PZSTD_CCXXFLAGS)
+PZSTD_CXXFLAGS  = $(PZSTD_CCXXFLAGS)
+PZSTD_LDFLAGS   =
+EXTRA_FLAGS     =
+ALL_CFLAGS      = $(EXTRA_FLAGS) $(CPPFLAGS) $(PZSTD_CPPFLAGS) $(CFLAGS)   $(PZSTD_CFLAGS)
+ALL_CXXFLAGS    = $(EXTRA_FLAGS) $(CPPFLAGS) $(PZSTD_CPPFLAGS) $(CXXFLAGS) $(PZSTD_CXXFLAGS)
+ALL_LDFLAGS     = $(EXTRA_FLAGS) $(LDFLAGS) $(PZSTD_LDFLAGS)
+
+
+# gtest libraries need to go before "-lpthread" because they depend on it.
+GTEST_LIB  = -L googletest/build/googlemock/gtest
+LIBS       = $(GTEST_LIB) -lpthread
+
+# Compilation commands
+LD_COMMAND  = $(CXX) $^          $(ALL_LDFLAGS) $(LIBS) -o $@
+CC_COMMAND  = $(CC)  $(DEPFLAGS) $(ALL_CFLAGS)   -c $<  -o $@
+CXX_COMMAND = $(CXX) $(DEPFLAGS) $(ALL_CXXFLAGS) -c $<  -o $@
+
+# Get a list of all zstd files so we rebuild the static library when we need to
+ZSTDCOMMON_FILES := $(wildcard $(ZSTDDIR)/common/*.c) \
+                    $(wildcard $(ZSTDDIR)/common/*.h)
+ZSTDCOMP_FILES   := $(wildcard $(ZSTDDIR)/compress/*.c) \
+                    $(wildcard $(ZSTDDIR)/compress/*.h)
+ZSTDDECOMP_FILES := $(wildcard $(ZSTDDIR)/decompress/*.c) \
+                    $(wildcard $(ZSTDDIR)/decompress/*.h)
+ZSTDPROG_FILES   := $(wildcard $(PROGDIR)/*.c) \
+                    $(wildcard $(PROGDIR)/*.h)
+ZSTD_FILES       := $(wildcard $(ZSTDDIR)/*.h) \
+                    $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) \
+                    $(ZSTDPROG_FILES)
+
+# List all the pzstd source files so we can determine their dependencies
+PZSTD_SRCS  := $(wildcard *.cpp)
+PZSTD_TESTS := $(wildcard test/*.cpp)
+UTILS_TESTS := $(wildcard utils/test/*.cpp)
+ALL_SRCS    := $(PZSTD_SRCS) $(PZSTD_TESTS) $(UTILS_TESTS)
 
 
 # Define *.exe as extension for Windows systems
@@ -30,89 +79,168 @@ else
 EXT =
 endif
 
-.PHONY: default all test clean test32 googletest googletest32
-
-default: pzstd
-
-all: pzstd
-
-
-libzstd.a: $(ZSTD_FILES)
-	$(MAKE) -C $(ZSTDDIR) libzstd
-	@cp $(ZSTDDIR)/libzstd.a .
-
-Pzstd.o: Pzstd.h Pzstd.cpp ErrorHolder.h utils/*.h
-	$(CXX) $(FLAGS) -c Pzstd.cpp -o $@
-
-SkippableFrame.o: SkippableFrame.h SkippableFrame.cpp utils/*.h
-	$(CXX) $(FLAGS) -c SkippableFrame.cpp -o $@
-
-Options.o: Options.h Options.cpp
-	$(CXX) $(FLAGS) -c Options.cpp -o $@
+# Standard targets
+.PHONY: default
+default: all
+
+.PHONY: check
+check:
+	$(TESTPROG) ./utils/test/BufferTest$(EXT) $(TESTFLAGS)
+	$(TESTPROG) ./utils/test/RangeTest$(EXT) $(TESTFLAGS)
+	$(TESTPROG) ./utils/test/ScopeGuardTest$(EXT) $(TESTFLAGS)
+	$(TESTPROG) ./utils/test/ThreadPoolTest$(EXT) $(TESTFLAGS)
+	$(TESTPROG) ./utils/test/WorkQueueTest$(EXT) $(TESTFLAGS)
+	$(TESTPROG) ./test/OptionsTest$(EXT) $(TESTFLAGS)
+	$(TESTPROG) ./test/PzstdTest$(EXT) $(TESTFLAGS)
+
+.PHONY: install
+install: PZSTD_CPPFLAGS += -DNDEBUG
+install: pzstd$(EXT)
+	install -d -m 755 $(BINDIR)/
+	install -m 755 pzstd$(EXT) $(BINDIR)/pzstd$(EXT)
+
+.PHONY: uninstall
+uninstall:
+	$(RM) $(BINDIR)/pzstd$(EXT)
+
+# Targets for many different builds
+.PHONY: all
+all: PZSTD_CPPFLAGS += -DNDEBUG
+all: pzstd$(EXT) tests roundtrip
+
+.PHONY: debug
+debug: EXTRA_FLAGS += -g
+debug: pzstd$(EXT) tests roundtrip
+
+.PHONY: tsan
+tsan: PZSTD_CCXXFLAGS += -fsanitize=thread -fPIC
+tsan: PZSTD_LDFLAGS   += -fsanitize=thread -pie
+tsan: debug
+
+.PHONY: asan
+asan: EXTRA_FLAGS += -fsanitize=address
+asan: debug
+
+.PHONY: ubsan
+ubsan: EXTRA_FLAGS += -fsanitize=undefined
+ubsan: debug
+
+.PHONY: all32
+all32: EXTRA_FLAGS += -m32
+all32: all
+
+.PHONY: debug32
+debug32: EXTRA_FLAGS += -m32
+debug32: debug
+
+.PHONY: asan32
+asan32: EXTRA_FLAGS += -m32
+asan32: asan
+
+.PHONY: tsan32
+tsan32: EXTRA_FLAGS += -m32
+tsan32: tsan
+
+.PHONY: ubsan32
+ubsan32: EXTRA_FLAGS += -m32
+ubsan32: ubsan
+
+# Run long round trip tests
+.PHONY: roundtripcheck
+roundtripcheck: roundtrip check
+	$(TESTPROG) ./test/RoundTripTest$(EXT) $(TESTFLAGS)
+
+# Build the main binary
+pzstd$(EXT): main.o Options.o Pzstd.o SkippableFrame.o $(ZSTDDIR)/libzstd.a
+	$(LD_COMMAND)
+
+# Target that depends on all the tests
+.PHONY: tests
+tests: EXTRA_FLAGS += -Wno-deprecated-declarations
+tests: $(patsubst %,%$(EXT),$(basename $(PZSTD_TESTS) $(UTILS_TESTS)))
+
+# Build the round trip tests
+.PHONY: roundtrip
+roundtrip: EXTRA_FLAGS += -Wno-deprecated-declarations
+roundtrip: test/RoundTripTest$(EXT)
+
+# Use the static library that zstd builds for simplicity and
+# so we get the compiler options correct
+$(ZSTDDIR)/libzstd.a: $(ZSTD_FILES)
+	$(MAKE) -C $(ZSTDDIR) libzstd CFLAGS="$(ALL_CFLAGS)" LDFLAGS="$(ALL_LDFLAGS)"
+
+
+# Rules to build the tests
+test/RoundTripTest$(EXT): test/RoundTripTest.o $(PROGDIR)/datagen.o Options.o \
+                          Pzstd.o SkippableFrame.o $(ZSTDDIR)/libzstd.a
+	$(LD_COMMAND)
+
+test/%Test$(EXT): GTEST_LIB += -lgtest -lgtest_main
+test/%Test$(EXT): test/%Test.o $(PROGDIR)/datagen.o Options.o Pzstd.o  \
+                  SkippableFrame.o $(ZSTDDIR)/libzstd.a
+	$(LD_COMMAND)
+
+utils/test/%Test$(EXT): GTEST_LIB += -lgtest -lgtest_main
+utils/test/%Test$(EXT): utils/test/%Test.o
+	$(LD_COMMAND)
+
+
+GTEST_CMAKEFLAGS =
+
+# Install googletest
+.PHONY: googletest
+googletest: PZSTD_CCXXFLAGS += -fPIC
+googletest:
+	@$(RM) -rf googletest
+	@git clone https://github.com/google/googletest
+	@mkdir -p googletest/build
+	@cd googletest/build && cmake $(GTEST_CMAKEFLAGS) -DCMAKE_CXX_FLAGS="$(ALL_CXXFLAGS)" .. && $(MAKE)
 
-main.o: main.cpp *.h utils/*.h
-	$(CXX) $(FLAGS) -c main.cpp -o $@
+.PHONY: googletest32
+googletest32: PZSTD_CCXXFLAGS  += -m32
+googletest32: googletest
 
-pzstd: Pzstd.o SkippableFrame.o Options.o main.o libzstd.a
-	$(CXX) $(FLAGS) $^ -o $@$(EXT) -lpthread
+.PHONY: googletest-mingw64
+googletest-mingw64: GTEST_CMAKEFLAGS += -G "MSYS Makefiles"
+googletest-mingw64: googletest
 
-libzstd32.a: $(ZSTD_FILES)
-	$(MAKE) -C $(ZSTDDIR) libzstd MOREFLAGS="-m32"
-	@cp $(ZSTDDIR)/libzstd.a libzstd32.a
+.PHONY: clean
+clean:
+	$(RM) -f *.o pzstd$(EXT) *.Td *.d
+	$(RM) -f test/*.o test/*Test$(EXT) test/*.Td test/*.d
+	$(RM) -f utils/test/*.o utils/test/*Test$(EXT) utils/test/*.Td utils/test/*.d
+	$(RM) -f $(PROGDIR)/*.o $(PROGDIR)/*.Td $(PROGDIR)/*.d
+	$(MAKE) -C $(ZSTDDIR) clean
+	@echo Cleaning completed
 
-Pzstd32.o: Pzstd.h Pzstd.cpp ErrorHolder.h utils/*.h
-	$(CXX) -m32 $(FLAGS) -c Pzstd.cpp -o $@
 
-SkippableFrame32.o: SkippableFrame.h SkippableFrame.cpp utils/*.h
-	$(CXX) -m32 $(FLAGS) -c SkippableFrame.cpp -o $@
+# Cancel implicit rules
+%.o: %.c
+%.o: %.cpp
 
-Options32.o: Options.h Options.cpp
-	$(CXX) -m32 $(FLAGS) -c Options.cpp -o $@
+# Object file rules
+%.o: %.c
+	$(CC_COMMAND)
+	$(POSTCOMPILE)
 
-main32.o: main.cpp *.h utils/*.h
-	$(CXX) -m32 $(FLAGS) -c main.cpp -o $@
+$(PROGDIR)/%.o: $(PROGDIR)/%.c
+	$(CC_COMMAND)
+	$(POSTCOMPILE)
 
-pzstd32: Pzstd32.o SkippableFrame32.o Options32.o main32.o libzstd32.a
-	$(CXX) -m32 $(FLAGS) $^ -o $@$(EXT) -lpthread
+%.o: %.cpp
+	$(CXX_COMMAND)
+	$(POSTCOMPILE)
 
-googletest:
-	@$(RM) -rf googletest
-	@git clone https://github.com/google/googletest
-	@mkdir -p googletest/build
-	@cd googletest/build && cmake .. && make
+test/%.o: test/%.cpp
+	$(CXX_COMMAND)
+	$(POSTCOMPILE)
 
-googletest32:
-	@$(RM) -rf googletest
-	@git clone https://github.com/google/googletest
-	@mkdir -p googletest/build
-	@cd googletest/build && cmake .. -DCMAKE_CXX_FLAGS=-m32 && make
-
-googletest-mingw64:
-	$(RM) -rf googletest
-	git clone https://github.com/google/googletest
-	mkdir -p googletest/build
-	cd googletest/build && cmake -G "MSYS Makefiles" .. && $(MAKE)
-
-test:
-	$(MAKE) libzstd.a
-	$(MAKE) pzstd MOREFLAGS="-Wall -Wextra -pedantic -Werror"
-	$(MAKE) -C utils/test clean
-	$(MAKE) -C utils/test test MOREFLAGS="-Wall -Wextra -pedantic -Werror"
-	$(MAKE) -C test clean
-	$(MAKE) -C test test MOREFLAGS="-Wall -Wextra -pedantic -Werror"
-
-test32:
-	$(MAKE) libzstd.a MOREFLAGS="-m32"
-	$(MAKE) pzstd MOREFLAGS="-m32 -Wall -Wextra -pedantic -Werror"
-	$(MAKE) -C utils/test clean
-	$(MAKE) -C utils/test test MOREFLAGS="-m32 -Wall -Wextra -pedantic -Werror"
-	$(MAKE) -C test clean
-	$(MAKE) -C test test MOREFLAGS="-m32 -Wall -Wextra -pedantic -Werror"
+utils/test/%.o: utils/test/%.cpp
+	$(CXX_COMMAND)
+	$(POSTCOMPILE)
 
+# Dependency file stuff
+.PRECIOUS: %.d test/%.d utils/test/%.d
 
-clean:
-	$(MAKE) -C $(ZSTDDIR) clean
-	$(MAKE) -C utils/test clean
-	$(MAKE) -C test clean
-	@$(RM) -rf libzstd.a *.o pzstd$(EXT) pzstd32$(EXT)
-	@echo Cleaning completed
+# Include rules that specify header file dependencies
+-include $(patsubst %,%.d,$(basename $(ALL_SRCS)))
diff --git a/contrib/pzstd/test/Makefile b/contrib/pzstd/test/Makefile
deleted file mode 100644
index 4f6ba999701..00000000000
--- a/contrib/pzstd/test/Makefile
+++ /dev/null
@@ -1,48 +0,0 @@
-# ##########################################################################
-# Copyright (c) 2016-present, Facebook, Inc.
-# All rights reserved.
-#
-# This source code is licensed under the BSD-style license found in the
-# LICENSE file in the root directory of this source tree. An additional grant
-# of patent rights can be found in the PATENTS file in the same directory.
-# ##########################################################################
-
-# Define *.exe as extension for Windows systems
-ifneq (,$(filter Windows%,$(OS)))
-EXT =.exe
-else
-EXT =
-endif
-
-PZSTDDIR = ..
-PROGDIR = ../../../programs
-ZSTDDIR = ../../../lib
-
-# Set GTEST_INC and GTEST_LIB to work with your install of gtest
-GTEST_INC ?= -isystem $(PZSTDDIR)/googletest/googletest/include
-GTEST_LIB ?= -L $(PZSTDDIR)/googletest/build/googlemock/gtest
-GTEST_FLAGS = $(GTEST_INC) $(GTEST_LIB)
-CPPFLAGS = -I$(PZSTDDIR) -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(PROGDIR) -I.
-
-CXXFLAGS  ?= -O3
-CXXFLAGS  += -std=c++11 -Wno-deprecated-declarations
-CXXFLAGS  += $(MOREFLAGS)
-FLAGS    = $(CPPFLAGS) $(CXXFLAGS) $(LDFLAGS)
-
-datagen.o: $(PROGDIR)/datagen.*
-	$(CC) $(CPPFLAGS) -O3 $(MOREFLAGS) $(LDFLAGS) -Wno-long-long -Wno-variadic-macros $(PROGDIR)/datagen.c -c -o $@
-
-%: %.cpp *.h datagen.o
-	$(CXX) $(FLAGS) $@.cpp datagen.o $(PZSTDDIR)/Pzstd.o $(PZSTDDIR)/SkippableFrame.o $(PZSTDDIR)/Options.o $(PZSTDDIR)/libzstd.a -o $@$(EXT) $(GTEST_FLAGS) -lgtest -lgtest_main -lpthread
-
-.PHONY: test clean
-
-test: OptionsTest PzstdTest
-	@./OptionsTest$(EXT)
-	@./PzstdTest$(EXT)
-
-roundtrip: RoundTripTest
-	@./RoundTripTest$(EXT)
-
-clean:
-	@rm -f datagen.o OptionsTest PzstdTest RoundTripTest
diff --git a/contrib/pzstd/utils/test/Makefile b/contrib/pzstd/utils/test/Makefile
deleted file mode 100644
index b9ea73e32ae..00000000000
--- a/contrib/pzstd/utils/test/Makefile
+++ /dev/null
@@ -1,42 +0,0 @@
-# ##########################################################################
-# Copyright (c) 2016-present, Facebook, Inc.
-# All rights reserved.
-#
-# This source code is licensed under the BSD-style license found in the
-# LICENSE file in the root directory of this source tree. An additional grant
-# of patent rights can be found in the PATENTS file in the same directory.
-# ##########################################################################
-
-# Define *.exe as extension for Windows systems
-ifneq (,$(filter Windows%,$(OS)))
-EXT =.exe
-else
-EXT =
-endif
-
-PZSTDDIR = ../..
-
-# Set GTEST_INC and GTEST_LIB to work with your install of gtest
-GTEST_INC ?= -isystem $(PZSTDDIR)/googletest/googletest/include
-GTEST_LIB ?= -L $(PZSTDDIR)/googletest/build/googlemock/gtest
-
-CPPFLAGS = -I$(PZSTDDIR) $(GTEST_INC) $(GTEST_LIB)
-CXXFLAGS  ?= -O3
-CXXFLAGS  += -std=c++11
-CXXFLAGS  += $(MOREFLAGS)
-FLAGS    = $(CPPFLAGS) $(CXXFLAGS) $(LDFLAGS)
-
-%: %.cpp
-	$(CXX) $(FLAGS) $^ -o $@$(EXT) -lgtest -lgtest_main -lpthread
-
-.PHONY: test clean
-
-test: BufferTest RangeTest ScopeGuardTest ThreadPoolTest WorkQueueTest
-	@./BufferTest$(EXT)
-	@./RangeTest$(EXT)
-	@./ScopeGuardTest$(EXT)
-	@./ThreadPoolTest$(EXT)
-	@./WorkQueueTest$(EXT)
-
-clean:
-	@rm -f BufferTest RangeTest ScopeGuardTest ThreadPoolTest WorkQueueTest