diff --git a/.ci/appveyor.yml b/.ci/appveyor.yml
index 4a3e0feba9..fc0c449b13 100644
--- a/.ci/appveyor.yml
+++ b/.ci/appveyor.yml
@@ -51,6 +51,10 @@ install:
 build: false  # Not a C# project, build stuff at the test step instead.
 
 test_script:
+  # Force DOS format, as Checkstyle configs enable NewlineAtEndOfFile,
+  # which defaults to CRLF on Windows, and Appveyor CI ignores .gitattributes
+  # http://help.appveyor.com/discussions/problems/5687-gitattributes-changes-dont-have-any-effect
+  - unix2dos tests/java/test_files/CheckstyleGood.java
   # Clang DLLs x64 were nowadays installed, but the x64 version hangs, so we
   # exclude according tests. See https://github.com/appveyor/ci/issues/495 and
   # https://github.com/appveyor/ci/issues/688
diff --git a/.gitattributes b/.gitattributes
new file mode 100644
index 0000000000..3114a98fca
--- /dev/null
+++ b/.gitattributes
@@ -0,0 +1 @@
+*.java text eol=native
diff --git a/bears/java/CheckstyleBear.py b/bears/java/CheckstyleBear.py
index dba5d7b487..29b77feb52 100644
--- a/bears/java/CheckstyleBear.py
+++ b/bears/java/CheckstyleBear.py
@@ -33,7 +33,7 @@ def known_checkstyle_or_path(setting):
 
 @linter(executable='java',
         output_format='regex',
-        output_regex=r'\[(?P<severity>WARN|INFO)\].*?'
+        output_regex=r'\[(?P<severity>ERROR|WARN|INFO)\].*?'
                      r'(?P<line>\d+):?(?P<column>\d+)?. '
                      r'(?P<message>.*?) *\[(?P<origin>[a-zA-Z]+?)\]')
 class CheckstyleBear:
diff --git a/tests/java/CheckstyleBearTest.py b/tests/java/CheckstyleBearTest.py
index 0cae329971..b9eed1b56d 100644
--- a/tests/java/CheckstyleBearTest.py
+++ b/tests/java/CheckstyleBearTest.py
@@ -24,24 +24,24 @@ def setUp(self):
         self.empty_config = os.path.join(test_files,
                                          'checkstyle_empty_config.xml')
 
-    def test_run(self):
-        self.check_validity(self.uut, [], self.good_file)
-        self.check_validity(self.uut, [], self.bad_file, valid=False)
-
     def test_style_google(self):
         self.section['checkstyle_configs'] = 'google'
         self.check_validity(self.uut, [], self.good_file)
+        self.check_validity(self.uut, [], self.bad_file, valid=False)
 
     def test_style_sun(self):
         self.section['checkstyle_configs'] = 'sun'
-        self.check_validity(self.uut, [], self.good_file)
+        self.check_validity(self.uut, [], self.good_file, valid=False)
+        self.check_validity(self.uut, [], self.bad_file, valid=False)
 
     def test_style_android(self):
         self.section['checkstyle_configs'] = 'android-check-easy'
         self.check_validity(self.uut, [], self.good_file)
+        self.check_validity(self.uut, [], self.bad_file, valid=False)
 
         self.section['checkstyle_configs'] = 'android-check-hard'
         self.check_validity(self.uut, [], self.good_file)
+        self.check_validity(self.uut, [], self.bad_file, valid=False)
 
     def test_config_failure_use_spaces(self):
         self.section['checkstyle_configs'] = 'google'