Merge pull request mongodb#317 from igorsol/m3.6.0-rc3

merge 3.6.0-rc2 & rc3 into PSMDB

.eslintignore

@@ -1 +1,2 @@
 src/mongo/gotools/*
+*.tpl.js

SConstruct

@@ -394,6 +394,11 @@ add_option('disable-warnings-as-errors',
     nargs=0,
 )
 
+add_option('detect-odr-violations',
+    help="Have the linker try to detect ODR violations, if supported",
+    nargs=0,
+)
+
 add_option('variables-help',
     help='Print the help text for SCons variables',
     nargs=0,
@@ -1451,7 +1456,7 @@ else:
     env.AppendUnique( CPPDEFINES=[ 'NDEBUG' ] )
 
 if env.TargetOSIs('linux'):
-    env.Append( LIBS=['m'] )
+    env.Append( LIBS=['m',"resolv"] )
 
 elif env.TargetOSIs('solaris'):
      env.Append( LIBS=["socket","resolv","lgrp"] )
@@ -1460,6 +1465,9 @@ elif env.TargetOSIs('freebsd'):
     env.Append( LIBS=[ "kvm" ] )
     env.Append( CCFLAGS=[ "-fno-omit-frame-pointer" ] )
 
+elif env.TargetOSIs('darwin'):
+     env.Append( LIBS=["resolv"] )
+
 elif env.TargetOSIs('openbsd'):
     env.Append( LIBS=[ "kvm" ] )
 
@@ -1620,6 +1628,7 @@ elif env.TargetOSIs('windows'):
             'advapi32.lib',
             'bcrypt.lib',
             'crypt32.lib',
+            'dnsapi.lib',
             'kernel32.lib',
             'shell32.lib',
             'pdh.lib',
@@ -2501,7 +2510,8 @@ def doConfigure(myenv):
         # probably built with GCC. That combination appears to cause
         # false positives for the ODR detector. See SERVER-28133 for
         # additional details.
-        if not (myenv.ToolchainIs('clang') and usingLibStdCxx):
+        if (get_option('detect-odr-violations') and
+                not (myenv.ToolchainIs('clang') and usingLibStdCxx)):
             AddToLINKFLAGSIfSupported(myenv, '-Wl,--detect-odr-violations')
 
         # Disallow an executable stack. Also, issue a warning if any files are found that would

buildscripts/aws_ec2.py

@@ -20,9 +20,17 @@
 class AwsEc2(object):
     """Class to support controlling AWS EC2 istances."""
 
-    InstanceStatus = collections.namedtuple(
-        "InstanceStatus",
-        "instance_id image_id instance_type state private_ip_address public_ip_address tags")
+    InstanceStatus = collections.namedtuple("InstanceStatus", [
+        "instance_id",
+        "image_id",
+        "instance_type",
+        "state",
+        "private_ip_address",
+        "public_ip_address",
+        "private_dns_name",
+        "public_dns_name",
+        "tags"
+        ])
 
     def __init__(self):
         try:
@@ -33,7 +41,29 @@ def __init__(self):
                   " for the variable names, file names and precedence order.")
             raise
 
-    def control_instance(self, mode, image_id):
+    @staticmethod
+    def wait_for_state(instance, state, wait_time_secs=0, show_progress=False):
+        """Wait up to 'wait_time_secs' for instance to be in 'state'.
+           Return True if 'state' reached."""
+        end_time = time.time() + wait_time_secs
+        if show_progress:
+            print("Waiting for instance {} to reach '{}' state".format(instance, state),
+                  end="",
+                  file=sys.stdout)
+        while time.time() < end_time:
+            if show_progress:
+                print(".", end="", file=sys.stdout)
+                sys.stdout.flush()
+            time.sleep(5)
+            instance.load()
+            if instance.state["Name"] == state:
+                if show_progress:
+                    print(" Instance {}!".format(state), file=sys.stdout)
+                    sys.stdout.flush()
+                return True
+        return False
+
+    def control_instance(self, mode, image_id, wait_time_secs=0, show_progress=False):
         """Controls an AMI instance. Returns 0 & status information, if successful."""
         if mode not in _MODES:
             raise ValueError(
@@ -42,18 +72,32 @@ def control_instance(self, mode, image_id):
         instance = self.connection.Instance(image_id)
         try:
             if mode == "start":
+                state = "running"
                 instance.start()
             elif mode == "stop":
+                state = "stopped"
                 instance.stop()
             elif mode == "force-stop":
+                state = "stopped"
                 instance.stop(Force=True)
             elif mode == "terminate":
+                state = "terminated"
                 instance.terminate()
             elif mode == "reboot":
+                state = "running"
                 instance.reboot()
-        except botocore.exceptions.ClientError as e:
-            return 1, e.message
-
+            else:
+                state = None
+                wait_time_secs = 0
+        except botocore.exceptions.ClientError as err:
+            return 1, err.message
+
+        if wait_time_secs > 0:
+            self.wait_for_state(
+                instance=instance,
+                state=state,
+                wait_time_secs=wait_time_secs,
+                show_progress=show_progress)
         try:
             # Always provide status after executing command.
             status = self.InstanceStatus(
@@ -63,9 +107,11 @@ def control_instance(self, mode, image_id):
                 getattr(instance, "state", None),
                 getattr(instance, "private_ip_address", None),
                 getattr(instance, "public_ip_address", None),
+                getattr(instance, "private_dns_name", None),
+                getattr(instance, "public_dns_name", None),
                 getattr(instance, "tags", None))
-        except botocore.exceptions.ClientError as e:
-            return 1, e.message
+        except botocore.exceptions.ClientError as err:
+            return 1, err.message
 
         return 0, status
 
@@ -80,8 +126,8 @@ def tag_instance(self, image_id, tags):
                 try:
                     instance = self.connection.Instance(image_id)
                     break
-                except botocore.exceptions.ClientError as e:
-                    if e.response["Error"]["Code"] != "InvalidInstanceID.NotFound":
+                except botocore.exceptions.ClientError as err:
+                    if err.response["Error"]["Code"] != "InvalidInstanceID.NotFound":
                         raise
                 time.sleep(i + 1)
             instance.create_tags(Tags=tags)
@@ -122,34 +168,24 @@ def launch_instance(self,
                 MaxCount=1,
                 MinCount=1,
                 **kwargs)
-        except (botocore.exceptions.ClientError, botocore.exceptions.ParamValidationError) as e:
-            return 1, e.message
+        except (botocore.exceptions.ClientError, botocore.exceptions.ParamValidationError) as err:
+            return 1, err.message
 
         instance = instances[0]
-
-        if wait_time_secs:
-            # Wait up to 'wait_time_secs' for instance to be 'running'.
-            end_time = time.time() + wait_time_secs
-            if show_progress:
-                print("Waiting for instance {} ".format(instance), end="", file=sys.stdout)
-            while time.time() < end_time:
-                if show_progress:
-                    print(".", end="", file=sys.stdout)
-                    sys.stdout.flush()
-                time.sleep(5)
-                instance.load()
-                if instance.state["Name"] == "running":
-                    if show_progress:
-                        print(" Instance running!", file=sys.stdout)
-                        sys.stdout.flush()
-                    break
+        if wait_time_secs > 0:
+            self.wait_for_state(
+                instance=instance,
+                state="running",
+                wait_time_secs=wait_time_secs,
+                show_progress=show_progress)
 
         self.tag_instance(instance.instance_id, tags)
 
         return self.control_instance("status", instance.instance_id)
 
 
 def main():
+    """Main program."""
 
     required_create_options = ["ami", "key_name"]
 
@@ -169,6 +205,13 @@ def main():
                                default=None,
                                help="EC2 image_id to perform operation on [REQUIRED for control].")
 
+    control_options.add_option("--waitTimeSecs",
+                               dest="wait_time_secs",
+                               type=int,
+                               default=60,
+                               help="Time to wait for EC2 instance to reach it's new state,"
+                                    " defaults to '%default'.")
+
     create_options.add_option("--ami",
                               dest="ami",
                               default=None,
@@ -229,7 +272,7 @@ def main():
     parser.add_option_group(control_options)
     parser.add_option_group(create_options)
 
-    (options, args) = parser.parse_args()
+    (options, _) = parser.parse_args()
 
     aws_ec2 = AwsEc2()
 
@@ -265,15 +308,19 @@ def main():
             key_name=options.key_name,
             security_groups=options.security_groups,
             tags=tags,
-            wait_time_secs=60,
+            wait_time_secs=options.wait_time_secs,
             show_progress=True,
             **my_kwargs)
     else:
         if not getattr(options, "image_id", None):
             parser.print_help()
             parser.error("Missing required control option")
 
-        (ret_code, instance_status) = aws_ec2.control_instance(options.mode, options.image_id)
+        (ret_code, instance_status) = aws_ec2.control_instance(
+            mode=options.mode,
+            image_id=options.image_id,
+            wait_time_secs=options.wait_time_secs,
+            show_progress=True)
 
     print("Return code: {}, Instance status:".format(ret_code))
     if ret_code:

buildscripts/cpplint.py

@@ -1676,6 +1676,12 @@ def CheckForMongoVolatile(filename, clean_lines, linenum, error):
           'Illegal use of the volatile storage keyword, use AtomicWord instead '
           'from "mongo/platform/atomic_word.h"')
 
+def CheckForNonMongoAssert(filename, clean_lines, linenum, error):
+  line = clean_lines.elided[linenum]
+  if re.search(r'\bassert\s*\(', line):
+    error(filename, linenum, 'mongodb/assert', 5,
+          'Illegal use of the bare assert function, use a function from assert_utils.h instead.')
+
 def CheckForCopyright(filename, lines, error):
   """Logs an error if no Copyright message appears at the top of the file."""
 
@@ -5820,6 +5826,7 @@ def ProcessLine(filename, file_extension, clean_lines, line,
   CheckForMongoPolyfill(filename, clean_lines, line, error)
   CheckForMongoAtomic(filename, clean_lines, line, error)
   CheckForMongoVolatile(filename, clean_lines, line, error)
+  CheckForNonMongoAssert(filename, clean_lines, line, error)
   if nesting_state.InAsmBlock(): return
   CheckForFunctionLengths(filename, clean_lines, line, function_state, error)
   CheckForMultilineCommentsAndStrings(filename, clean_lines, line, error)

buildscripts/errorcodes.py

@@ -54,17 +54,16 @@ def assignErrorCodes():
 def parseSourceFiles( callback ):
     """Walks MongoDB sourcefiles and invokes callback for each AssertLocation found."""
 
-    quick = [ "assert" , "Exception"]
+    quick = ["assert", "Exception", "ErrorCodes::Error"]
 
     patterns = [
         re.compile( r"(?:u|m(?:sg)?)asser(?:t|ted)(?:NoTrace)?\s*\(\s*(\d+)", re.MULTILINE ) ,
-        re.compile( r"(?:DB|Assertion)Exception\s*\(\s*(\d+)", re.MULTILINE ),
+        re.compile( r"(?:DB|Assertion)Exception\s*[({]\s*(\d+)", re.MULTILINE ),
         re.compile( r"fassert(?:Failed)?(?:WithStatus)?(?:NoTrace)?(?:StatusOK)?\s*\(\s*(\d+)",
                     re.MULTILINE ),
+        re.compile( r"ErrorCodes::Error\s*[({]\s*(\d+)", re.MULTILINE )
     ]
 
-    bad = [ re.compile( r"^\s*assert *\(" ) ]
-
     for sourceFile in utils.getAllSourceFiles(prefix='src/mongo/'):
         if list_files:
             print 'scanning file: ' + sourceFile
@@ -75,28 +74,24 @@ def parseSourceFiles( callback ):
             if not any([zz in text for zz in quick]):
                 continue
 
-            # TODO: move check for bad assert type to the linter.
-            for b in bad:
-                if b.search(text):
-                    msg = "Bare assert prohibited. Replace with [umwdf]assert"
-                    print( "%s: %s" % (sourceFile, msg) )
-                    raise Exception(msg)
-
             matchiters = [p.finditer(text) for p in patterns]
             for matchiter in matchiters:
                 for match in matchiter:
                     code = match.group(1)
-                    span = match.span()
+                    codeOffset = match.start(1)
 
+                    # Note that this will include the text of the full match but will report the
+                    # position of the beginning of the code portion rather than the beginning of the
+                    # match. This is to position editors on the spot that needs to change.
                     thisLoc = AssertLocation(sourceFile,
-                                             span[1],
-                                             text[span[0]:span[1]],
+                                             codeOffset,
+                                             text[match.start():match.end()],
                                              code)
 
                     callback( thisLoc )
 
 # Converts an absolute position in a file into a line number.
-def getLineForPosition(loc, _file_cache={}):
+def getLineAndColumnForPosition(loc, _file_cache={}):
     if loc.sourceFile not in _file_cache:
         with open(loc.sourceFile) as f:
             text = f.read()
@@ -105,7 +100,10 @@ def getLineForPosition(loc, _file_cache={}):
                 line_offsets.append(line_offsets[-1] + len(line))
             _file_cache[loc.sourceFile] = line_offsets
 
-    return bisect.bisect(_file_cache[loc.sourceFile], loc.byteOffset)
+    # These are both 1-based, but line is handled by starting the list with 0.
+    line = bisect.bisect(_file_cache[loc.sourceFile], loc.byteOffset)
+    column = loc.byteOffset - _file_cache[loc.sourceFile][line - 1] + 1
+    return (line, column)
 
 def isTerminated( lines ):
     """Given .cpp/.h source lines as text, determine if assert is terminated."""
@@ -165,13 +163,15 @@ def checkDups( assertLoc ):
         code = "0"
         bad = seen[code]
         errors.append( bad )
+        line, col = getLineAndColumnForPosition(bad)
         print( "ZERO_CODE:" )
-        print( "  %s:%d:%s" % (bad.sourceFile, getLineForPosition(bad), bad.lines) )
+        print( "  %s:%d:%d:%s" % (bad.sourceFile, line, col, bad.lines) )
 
     for code, locations in dups.items():
         print( "DUPLICATE IDS: %s" % code )
         for loc in locations:
-            print( "  %s:%d:%s" % (loc.sourceFile, getLineForPosition(loc), loc.lines) )
+            line, col = getLineAndColumnForPosition(loc)
+            print( "  %s:%d:%d:%s" % (loc.sourceFile, line, col, loc.lines) )
 
     return (codes, errors)
 
@@ -188,13 +188,14 @@ def replaceBadCodes( errors, nextCode ):
     skip_errors = [e for e in errors if int(e.code) != 0]
 
     for loc in skip_errors:
-        print ("SKIPPING NONZERO code=%s: %s:%s"
-                % (loc.code, loc.sourceFile, getLineForPosition(loc)))
+        line, col = getLineAndColumnForPosition(loc)
+        print ("SKIPPING NONZERO code=%s: %s:%d:%d"
+                % (loc.code, loc.sourceFile, line, col))
 
     # Dedupe, sort, and reverse so we don't have to update offsets as we go.
     for assertLoc in reversed(sorted(set(zero_errors))):
         (sourceFile, byteOffset, lines, code) = assertLoc
-        lineNum = getLineForPosition(assertLoc)
+        lineNum, _ = getLineAndColumnForPosition(assertLoc)
         print "UPDATING_FILE: %s:%s" % (sourceFile, lineNum)
 
         ln = lineNum - 1
@@ -204,11 +205,11 @@ def replaceBadCodes( errors, nextCode ):
 
             f.seek(0)
             text = f.read()
-            assert text[byteOffset-1] == '0'
+            assert text[byteOffset] == '0'
             f.seek(0)
-            f.write(text[:byteOffset-1])
+            f.write(text[:byteOffset])
             f.write(str(nextCode))
-            f.write(text[byteOffset:])
+            f.write(text[byteOffset+1:])
             f.seek(0)
 
             print "LINE_%d_AFTER :%s" % (lineNum, f.readlines()[ln].rstrip())