Merge pull request bitcoin#599 from BitcoinUnlimitedJanitor/rel.fmt2

release source formatting

.travis.yml

@@ -16,6 +16,7 @@ env:
   global:
     - MAKEJOBS=-j3
     - RUN_TESTS=false
+    - RUN_FORMATTING_CHECK=false
     - CHECK_DOC=0
     - BOOST_TEST_RANDOM=1$TRAVIS_BUILD_ID
     - CCACHE_SIZE=100M
@@ -26,6 +27,8 @@ env:
     - PYTHON_DEBUG=1
     - WINEDEBUG=fixme-all
   matrix:
+# bitcoind
+    - HOST=x86_64-unknown-linux-gnu PACKAGES="bc python3-zmq openjdk-7-jre-headless clang-format-3.8" DEP_OPTS="NO_QT=1 NO_UPNP=1 DEBUG=1" RUN_TESTS=true RUN_FORMATTING_CHECK=true GOAL="install" BITCOIN_CONFIG="--enable-zmq --enable-glibc-back-compat --enable-reduce-exports CPPFLAGS=-DDEBUG_LOCKORDER"  
 # ARM64
     - HOST=aarch64-linux-gnu PACKAGES="g++-aarch64-linux-gnu" DEP_OPTS="NO_QT=1" GOAL="install" BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports"
 # ARMHF
@@ -36,8 +39,6 @@ env:
     - HOST=i686-pc-linux-gnu PACKAGES="g++-multilib bc python3-zmq openjdk-7-jre-headless" DEP_OPTS="NO_QT=1" RUN_TESTS=true GOAL="install" BITCOIN_CONFIG="--enable-zmq --enable-glibc-back-compat --enable-reduce-exports LDFLAGS=-static-libstdc++" USE_SHELL="/bin/dash"
 # Win64
     - HOST=x86_64-w64-mingw32 DPKG_ADD_ARCH="i386" DEP_OPTS="NO_QT=1" PACKAGES="python3 nsis g++-mingw-w64-x86-64 wine1.6 bc openjdk-7-jre-headless" RUN_TESTS=true GOAL="install" BITCOIN_CONFIG="--enable-reduce-exports"
-# bitcoind
-    - HOST=x86_64-unknown-linux-gnu PACKAGES="bc python3-zmq openjdk-7-jre-headless" DEP_OPTS="NO_QT=1 NO_UPNP=1 DEBUG=1" RUN_TESTS=true GOAL="install" BITCOIN_CONFIG="--enable-zmq --enable-glibc-back-compat --enable-reduce-exports CPPFLAGS=-DDEBUG_LOCKORDER"
 # No wallet
     - HOST=x86_64-unknown-linux-gnu PACKAGES="openjdk-7-jre-headless python3" DEP_OPTS="NO_WALLET=1" RUN_TESTS=true GOAL="install" BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports"
 # Cross-Mac
@@ -65,6 +66,7 @@ script:
     - test -n "$USE_SHELL" && eval '"$USE_SHELL" -c "./autogen.sh"' || ./autogen.sh
     - mkdir build && cd build
     - ../configure $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG || ( cat config.log && false)
+    - if [ "$RUN_FORMATTING_CHECK" = "true" ]; then make $MAKEJOBS check-formatting VERBOSE=1; fi
     - make $MAKEJOBS $GOAL || ( echo "Build failure. Verbose build follows." && make $GOAL V=1 ; false )
     - export LD_LIBRARY_PATH=$TRAVIS_BUILD_DIR/depends/$HOST/lib
     - if [ "$RUN_TESTS" = "true" ] && { [ "$HOST" = "i686-w64-mingw32" ] || [ "$HOST" = "x86_64-w64-mingw32" ]; }; then travis_wait make $MAKEJOBS check VERBOSE=1; fi

Makefile.am

@@ -204,6 +204,9 @@ check-local:
 	@qa/pull-tester/run-bitcoind-for-test.sh $(JAVA) -jar $(JAVA_COMPARISON_TOOL) qa/tmp/compTool $(COMPARISON_TOOL_REORG_TESTS) 2>&1
 endif
 
+check-formatting:
+	$(MAKE) -C src $@
+
 dist_noinst_SCRIPTS = autogen.sh
 
 EXTRA_DIST = $(top_srcdir)/share/genbuild.sh qa/pull-tester/rpc-tests.py qa/pull-tester/test_classes.py qa/rpc-tests $(DIST_DOCS) $(WINDOWS_PACKAGING) $(OSX_PACKAGING) $(BIN_CHECKS)

contrib/devtools/clang-format.py

@@ -12,9 +12,15 @@
 import os
 import sys
 import subprocess
+import difflib
+import StringIO
+import pdb
+import tempfile
 
-tested_versions = ['3.6.0', '3.6.1', '3.6.2'] # A set of versions known to produce the same output
+tested_versions = ['3.6.0', '3.6.1', '3.6.2', '3.8.0'] # A set of versions known to produce the same output
 accepted_file_extensions = ('.h', '.cpp') # Files to format
+trailing_comment_exe = "trailing-comment.py"
+max_col_len = 120
 
 def check_clang_format_version(clang_format_exe):
     try:
@@ -29,8 +35,8 @@ def check_clang_format_version(clang_format_exe):
         raise e
 
 def check_command_line_args(argv):
-    required_args = ['{clang-format-exe}', '{files}']
-    example_args = ['clang-format-3.x', 'src/main.cpp', 'src/wallet/*']
+    required_args = ['{operation}','{clang-format-exe}', '{files}']
+    example_args = ['clang-format', 'format', 'src/main.cpp', 'src/wallet/*']
 
     if(len(argv) < len(required_args) + 1):
         for word in (['Usage:', argv[0]] + required_args):
@@ -41,23 +47,74 @@ def check_command_line_args(argv):
         print ''
         sys.exit(1)
 
+def run_clang_check(clang_format_exe, files):
+    changed = set()
+    nonexistent = []
+    for target in files:
+        if os.path.isdir(target):
+            for path, dirs, files in os.walk(target):
+                run_clang_check(clang_format_exe, (os.path.join(path, f) for f in files))
+        elif not os.path.exists(target):
+            print("File %s does not exist" % target)
+            nonexistent.append(target)
+        elif target.endswith(accepted_file_extensions):
+            formattedFile = tempfile.TemporaryFile()
+            trailingCommentFile = tempfile.TemporaryFile()
+            subprocess.check_call([trailing_comment_exe, str(max_col_len)], stdin=open(target,"rb"), stdout=trailingCommentFile, stderr=subprocess.STDOUT)
+            trailingCommentFile.seek(0)
+            subprocess.check_call([clang_format_exe,'-style=file','-assume-filename=%s' % target], stdin=trailingCommentFile, stdout=formattedFile, stderr=subprocess.STDOUT)
+            with open(target,"rb") as f: inputContents = f.readlines()
+            formattedFile.seek(0)
+            formattedContents = formattedFile.readlines()
+            for l in difflib.unified_diff(inputContents,formattedContents, target, target+"_formatted"):
+                sys.stdout.write(l)
+                changed.add(target)            
+        else:
+            print "Skip " + target
+    if nonexistent:
+        print("\nNonexistent files: " + ",".join(nonexistent))
+    if changed:
+        print("\nImproper formatting found in: " + ",".join(list(changed)))
+        print("To properly format these files run: " + sys.argv[0] + " format clang-format " + " ".join(list(changed)))
+        return 1
+    else:
+        print("All existing files are properly formatted")
+
+    if nonexistent:
+        return 2        
+    return 0
+
 def run_clang_format(clang_format_exe, files):
     for target in files:
         if os.path.isdir(target):
             for path, dirs, files in os.walk(target):
                 run_clang_format(clang_format_exe, (os.path.join(path, f) for f in files))
         elif target.endswith(accepted_file_extensions):
             print "Format " + target
+            subprocess.check_call([trailing_comment_exe, str(max_col_len), target])
             subprocess.check_call([clang_format_exe, '-i', '-style=file', target], stdout=open(os.devnull, 'wb'), stderr=subprocess.STDOUT)
         else:
             print "Skip " + target
 
 def main(argv):
+    global trailing_comment_exe
     check_command_line_args(argv)
-    clang_format_exe = argv[1]
-    files = argv[2:]
+    mypath = os.path.realpath(__file__)
+    trailing_comment_exe = os.path.join(os.path.dirname(mypath), trailing_comment_exe)
+    print trailing_comment_exe
+    operation = argv[1]
+    clang_format_exe = argv[2]
+    files = argv[3:]
     check_clang_format_version(clang_format_exe)
-    run_clang_format(clang_format_exe, files)
+    if operation == "check":
+        return run_clang_check(clang_format_exe, files)
+    if operation == "format":
+        run_clang_format(clang_format_exe, files)
 
 if __name__ == "__main__":
-    main(sys.argv)
+    if len(sys.argv) == 1:
+        print("clang-format.py: helper wrapper for clang-format")
+        check_command_line_args(sys.argv)   
+
+    result = main(sys.argv)
+    sys.exit(result)    

contrib/devtools/trailing-comment.py

@@ -0,0 +1,86 @@
+#!/usr/bin/env python
+"""
+Searches for all instances of trailing single line comments where the total line
+length exceeds the provided line length, and converts to a separate comment line
+then code line.
+
+For example:
+void my_func(int param); // This is a very long comment explaining what this function does.
+
+becomes:
+// This is a very long comment explaining what this function does.
+void my_func(int param);
+
+Copyright (c) 2017 The Bitcoin Unlimited developers
+Distributed under the MIT software license, see the accompanying
+file COPYING or http://www.opensource.org/licenses/mit-license.php.
+"""
+import pdb
+import sys
+import types
+
+TAB_SIZE = 8
+
+def main(files, cutoff,verbose=False):
+    if not files:
+        files.append(sys.stdin)
+    for f in files:
+        changed = False
+        if type(f) in types.StringTypes:
+            fo = open(f,"r")
+            if verbose:
+                print("Processing %s" % f)
+        else:
+            fo = f
+            verbose = False # if using stdin/stdout, don't clutter it with logging
+
+        lines = fo.readlines()      
+        output = []
+        for line in lines:
+            #print len(line), "//" in line, line
+            line = line.replace("\t",TAB_SIZE*" ")
+            if len(line) > cutoff and "//" in line:
+                try:
+                    (code, comment) = line.rsplit("//",1)
+                except ValueError:
+                    print line
+                    pdb.set_trace()
+                # If there is a quote both the code and the comment, the // is likely in a comment    
+                if comment.count('"')%2==1 and code.count('"')%2==1:
+                    print("Warning, this line is weird, not touching...")
+                    print(line)
+                    output.append(line)
+                    continue
+                codeLen = len(code.lstrip())
+                indentation = len(code) - codeLen
+                if codeLen: # trailing comments has at least some code on the line
+                    # pdb.set_trace()
+                    newComment = " "*indentation + "//" + comment
+                    if verbose:
+                        print("Was:\n%s" % line.rstrip())
+                        print("Now:")
+                        print(newComment.rstrip())
+                        print(code.rstrip())
+                        #sys.stdout.write(newComment)
+                        #sys.stdout.write(code)
+                    output.append(newComment)
+                    output.append(code.rstrip() + "\n")
+                    changed = True
+                else:
+                    output.append(line)
+            else:
+                output.append(line)
+        if type(f) in types.StringTypes:
+            if changed:
+                print("%s changed" % f)        
+                fo = open(f,"w")
+                fo.writelines(output)
+        else:
+            sys.stdout.writelines(output)
+
+
+if __name__ == '__main__':
+    if len(sys.argv) < 2:
+        print("usage: trailing-comment.py <max columns> [files]")
+        sys.exit(-1)
+    main(sys.argv[2:],int(sys.argv[1]))

src/.clang-format

@@ -1,8 +1,8 @@
 Language:        Cpp
 AccessModifierOffset: -4
-AlignAfterOpenBracket: false
+AlignAfterOpenBracket: DontAlign
 AlignEscapedNewlinesLeft: true
-AlignTrailingComments: true
+AlignTrailingComments: false
 AllowAllParametersOfDeclarationOnNextLine: false
 AllowShortBlocksOnASingleLine: false
 AllowShortFunctionsOnASingleLine: All
@@ -15,7 +15,7 @@ BreakBeforeBinaryOperators: false
 BreakBeforeBraces: Allman
 BreakBeforeTernaryOperators: false
 BreakConstructorInitializersBeforeComma: false
-ColumnLimit:     0
+ColumnLimit:     120
 CommentPragmas:  '^ IWYU pragma:'
 ConstructorInitializerAllOnOneLineOrOnePerLine: false
 ConstructorInitializerIndentWidth: 4
@@ -38,7 +38,7 @@ PenaltyBreakFirstLessLess: 120
 PenaltyBreakString: 1000
 PenaltyExcessCharacter: 1000000
 PenaltyReturnTypeOnItsOwnLine: 200
-PointerAlignment: Left
+PointerAlignment: Right
 SpaceBeforeAssignmentOperators: true
 SpaceBeforeParens: ControlStatements
 SpaceInEmptyParentheses: false
@@ -50,3 +50,4 @@ SpacesInParentheses: false
 Standard:        Cpp03
 TabWidth:        8
 UseTab:          Never
+AlignTrailingComments: false

src/.formatted-files

@@ -0,0 +1,21 @@
+stat.h
+tweak.cpp
+tweak.h
+thinblock.cpp
+thinblock.h
+leakybucket.h
+unlimited.cpp
+unlimited.h
+globals.cpp
+expedited.cpp
+expedited.h
+main.cpp
+main.h
+net.cpp
+net.h
+miner.cpp
+miner.h
+qt/unlimiteddialog.cpp
+qt/unlimiteddialog.h
+qt/unlimitedmodel.cpp
+qt/unlimitedmodel.h

src/Makefile.am

@@ -68,7 +68,7 @@ if BUILD_BITCOIN_UTILS
   bin_PROGRAMS += bitcoin-cli bitcoin-tx
 endif
 
-.PHONY: FORCE check-symbols check-security
+.PHONY: FORCE check-symbols check-security check-formatting
 # bitcoin core #
 BITCOIN_CORE_H = \
   addrman.h \
@@ -479,6 +479,10 @@ if HARDEN
 	$(AM_V_at) READELF=$(READELF) OBJDUMP=$(OBJDUMP) $(top_srcdir)/contrib/devtools/security-check.py < $(bin_PROGRAMS)
 endif
 
+check-formatting:
+	@echo "Checking source formatting style..."
+	(cd $(top_srcdir)/src; ../contrib/devtools/clang-format.py check clang-format-3.8 `cat ./.formatted-files`)
+
 %.pb.cc %.pb.h: %.proto
 	@test -f $(PROTOC)
 	$(AM_V_GEN) $(PROTOC) --cpp_out=$(@D) --proto_path=$(abspath $(<D) $<)

src/expedited.cpp

@@ -3,9 +3,9 @@
 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
 #include "expedited.h"
-#include "tweak.h"
 #include "main.h"
 #include "thinblock.h"
+#include "tweak.h"
 #include "unlimited.h"
 
 #include <sstream>
@@ -72,7 +72,7 @@ bool CheckAndRequestExpeditedBlocks(CNode *pfrom)
                 {
                     LogPrintf("Requesting expedited blocks from peer %s (%d).\n", strListeningPeerIP, pfrom->id);
                     pfrom->PushMessage(NetMsgType::XPEDITEDREQUEST, ((uint64_t)EXPEDITED_BLOCKS));
-                    
+
                     LOCK(cs_xpedited);
                     xpeditedBlkUp.push_back(pfrom);
 
@@ -171,8 +171,8 @@ bool HandleExpeditedRequest(CDataStream &vRecv, CNode *pfrom)
                 }
                 else
                 {
-                    LogPrint("blk", "Expedited transactions requested from peer %s but I am full\n",
-                        pfrom->GetLogName());
+                    LogPrint(
+                        "blk", "Expedited transactions requested from peer %s but I am full\n", pfrom->GetLogName());
                 }
             }
         }

src/expedited.h

@@ -8,8 +8,8 @@
 #include "net.h"
 #include "thinblock.h"
 
-#include <vector>
 #include <univalue.h>
+#include <vector>
 
 
 enum
@@ -47,4 +47,3 @@ extern bool HandleExpeditedBlock(CDataStream &vRecv, CNode *pfrom);
 extern bool IsExpeditedNode(const CNode *pfrom);
 
 #endif
-