Skip to content
This repository has been archived by the owner. It is now read-only.
Permalink
Browse files
AMBARI-24981. JournalNode may fail to start due to unreadable config …
…file (#2679)
  • Loading branch information
adoroszlai committed Feb 5, 2019
1 parent d44c1b1 commit 8613d58c347cc21961726811ac8d7bf43389ee6f
Showing 9 changed files with 158 additions and 151 deletions.
@@ -99,13 +99,14 @@ def test_download_source_get_content_nocache(self, exists_mock, request_mock, op
self.assertEqual(opener_mock.call_count, 1)
request_mock.assert_called_with('http://download/source')
self.assertEqual(web_file_mock.read.call_count, 1)

@patch("__builtin__.open")
@patch.object(urllib2, "Request")
@patch.object(urllib2, "build_opener")
@patch.object(os, "makedirs")
@patch.object(os.path, "exists")
def test_download_source_get_content_cache_new(self, exists_mock, makedirs_mock, opener_mock, request_mock, open_mock):
@patch("resource_management.core.sudo.create_file")
def test_download_source_get_content_cache_new(self, create_mock, exists_mock, makedirs_mock, opener_mock, request_mock, open_mock):
"""
Testing DownloadSource.get_content with cache on non-cached resource
"""
@@ -18,7 +18,7 @@


from unittest import TestCase
from mock.mock import patch, MagicMock
from mock.mock import patch, MagicMock, ANY
from only_for_platform import get_platform, not_for_platform, os_distro_value, PLATFORM_WINDOWS

from ambari_commons.os_check import OSCheck
@@ -103,10 +103,9 @@ def test_action_create_non_existent_file(self, isdir_mock, exists_mock, create_f
mode=0777,
content='file-content'
)


create_file_mock.assert_called_with('/directory/file', 'file-content', encoding=None)
self.assertEqual(create_file_mock.call_count, 1)

create_file_mock.assert_called_once('/directory/file', 'file-content', encoding=None, on_file_created=ANY)
ensure_mock.assert_called()


@@ -130,8 +129,8 @@ def test_action_create_replace(self, isdir_mock, exists_mock, create_file_mock,
content='new-content'
)

read_file_mock.assert_called_with('/directory/file', encoding=None)
create_file_mock.assert_called_with('/directory/file', 'new-content', encoding=None)
read_file_mock.assert_called_with('/directory/file', encoding=None)
create_file_mock.assert_called_with('/directory/file', 'new-content', encoding=None, on_file_created=ANY)


@patch("resource_management.core.sudo.unlink")
@@ -276,16 +275,12 @@ def test_attribute_replace(self, isdir_mock, exists_mock, open_mock, ensure_mock
@patch("resource_management.core.sudo.chown")
@patch("resource_management.core.sudo.chmod")
@patch("resource_management.core.sudo.stat")
@patch("resource_management.core.sudo.create_file")
@patch("resource_management.core.sudo.path_exists")
@patch("resource_management.core.sudo.path_isdir")
def test_ensure_metadata(self, isdir_mock, exists_mock, create_file_mock, stat_mock, chmod_mock, chown_mock, getgrnam_mock,
def test_ensure_metadata(self, stat_mock, chmod_mock, chown_mock, getgrnam_mock,
getpwnam_mock):
"""
Tests if _ensure_metadata changes owner, usergroup and permissions of file to proper values
"""
isdir_mock.side_effect = [False, True, False, True]
exists_mock.return_value = False
from resource_management.core.providers.system import _ensure_metadata

class stat():
def __init__(self):
@@ -300,17 +295,8 @@ def __init__(self):
getgrnam_mock.return_value.gr_gid = 0

with Environment('/') as env:
File('/directory/file',
action='create',
mode=0777,
content='file-content',
owner='root',
group='hdfs'
)

_ensure_metadata('/directory/file', user='root', group='hdfs', mode=0777)

create_file_mock.assert_called_with('/directory/file', 'file-content', encoding=None)
self.assertEqual(create_file_mock.call_count, 1)
stat_mock.assert_called_with('/directory/file')
self.assertEqual(chmod_mock.call_count, 1)
self.assertEqual(chown_mock.call_count, 1)
@@ -325,14 +311,7 @@ def __init__(self):
getgrnam_mock.return_value.gr_gid = 1

with Environment('/') as env:
File('/directory/file',
action='create',
mode=0777,
content='file-content',
owner='root',
group='hdfs'
)

_ensure_metadata('/directory/file', user='root', group='hdfs', mode=0777)

self.assertEqual(chmod_mock.call_count, 1)
chown_mock.assert_called_with('/directory/file', None, None)
@@ -23,7 +23,7 @@
import os
import time
from unittest import TestCase
from mock.mock import patch, MagicMock
from mock.mock import patch, MagicMock, ANY
from only_for_platform import get_platform, not_for_platform, os_distro_value, PLATFORM_WINDOWS

from ambari_commons.os_check import OSCheck
@@ -69,7 +69,7 @@ def test_action_create_empty_properties_without_dir(self,
properties={}
)

create_file_mock.assert_called_with('/somewhere_in_system/one_file.properties', u'# Generated by Apache Ambari. Today is Wednesday\n \n ', encoding="UTF-8")
create_file_mock.assert_called_with('/somewhere_in_system/one_file.properties', u'# Generated by Apache Ambari. Today is Wednesday\n \n ', encoding="UTF-8", on_file_created=ANY)
ensure_mock.assert_called()


@@ -102,7 +102,7 @@ def test_action_create_empty_properties_with_dir(self,
properties={},
)

create_file_mock.assert_called_with('/dir/and/dir/file.txt', u'# Generated by Apache Ambari. Some other day\n \n ', encoding="UTF-8")
create_file_mock.assert_called_with('/dir/and/dir/file.txt', u'# Generated by Apache Ambari. Some other day\n \n ', encoding="UTF-8", on_file_created=ANY)
ensure_mock.assert_called()


@@ -127,15 +127,15 @@ def test_action_create_properties_simple(self,
os_path_exists_mock.return_value = False
time_asctime_mock.return_value = 777




with Environment('/') as env:
PropertiesFile('/dir/new_file',
properties={'property1': 'value1'},
)

create_file_mock.assert_called_with('/dir/new_file', u'# Generated by Apache Ambari. 777\n \nproperty1=value1\n ', encoding="UTF-8")
create_file_mock.assert_called_with('/dir/new_file', u'# Generated by Apache Ambari. 777\n \nproperty1=value1\n ', encoding="UTF-8", on_file_created=ANY)
ensure_mock.assert_called()


@@ -173,7 +173,7 @@ def test_action_create_properties_with_metacharacters(self,
},
)

create_file_mock.assert_called_with('/dir/new_file', u"# Generated by Apache Ambari. 777\n \n=\nprop.1='.'yyyy-MM-dd-HH\nprop.2=INFO, openjpa\nprop.3=%d{ISO8601} %5p %c{1}:%L - %m%n\nprop.4=${oozie.log.dir}/oozie.log\nprop.empty=\n ", encoding="UTF-8")
create_file_mock.assert_called_with('/dir/new_file', u"# Generated by Apache Ambari. 777\n \n=\nprop.1='.'yyyy-MM-dd-HH\nprop.2=INFO, openjpa\nprop.3=%d{ISO8601} %5p %c{1}:%L - %m%n\nprop.4=${oozie.log.dir}/oozie.log\nprop.empty=\n ", encoding="UTF-8", on_file_created=ANY)
ensure_mock.assert_called()


@@ -210,5 +210,5 @@ def test_action_create_properties_rewrite_content(self,
)

read_file_mock.assert_called()
create_file_mock.assert_called_with('/dir1/new_file', u'# Generated by Apache Ambari. 777\n \nproperty_1=value1\n ', encoding="UTF-8")
create_file_mock.assert_called_with('/dir1/new_file', u'# Generated by Apache Ambari. 777\n \nproperty_1=value1\n ', encoding="UTF-8", on_file_created=ANY)
ensure_mock.assert_called()
@@ -194,8 +194,9 @@ def test_recreate_repo_suse(self, filecmp_mock, read_file_mock, checked_call_moc
@patch("resource_management.libraries.providers.repository.File")
@patch("os.path.isfile", new=MagicMock(return_value=True))
@patch("filecmp.cmp", new=MagicMock(return_value=False))
@patch.object(System, "os_release_name", new='precise')
@patch.object(System, "os_release_name", new='precise')
@patch.object(System, "os_family", new='ubuntu')
@patch("ambari_commons.os_utils.current_user", new=MagicMock(return_value='ambari-agent'))
def test_create_repo_ubuntu_repo_exists(self, file_mock, execute_mock,
tempfile_mock, call_mock, is_redhat_family, is_ubuntu_family, is_suse_family):
is_redhat_family.return_value = False
@@ -218,13 +219,13 @@ def test_create_repo_ubuntu_repo_exists(self, file_mock, execute_mock,
call_content = file_mock.call_args_list[0]
template_name = call_content[0][0]
template_content = call_content[1]['content']

self.assertEquals(template_name, '/tmp/1.txt')
self.assertEquals(template_content, 'deb http://download.base_url.org/rpm/ a b c')

copy_item0 = str(file_mock.call_args_list[1])
copy_item1 = str(file_mock.call_args_list[2])
self.assertEqual(copy_item0, "call('/tmp/1.txt', content=StaticFile('/etc/apt/sources.list.d/HDP.list'))")
self.assertEqual(copy_item0, "call('/tmp/1.txt', owner='ambari-agent', content=StaticFile('/etc/apt/sources.list.d/HDP.list'))")
self.assertEqual(copy_item1, "call('/etc/apt/sources.list.d/HDP.list', content=StaticFile('/tmp/1.txt'))")
#'apt-get update -qq -o Dir::Etc::sourcelist="sources.list.d/HDP.list" -o APT::Get::List-Cleanup="0"')
execute_command_item = execute_mock.call_args_list[0][0][0]
@@ -240,6 +241,7 @@ def test_create_repo_ubuntu_repo_exists(self, file_mock, execute_mock,
@patch("filecmp.cmp", new=MagicMock(return_value=False))
@patch.object(System, "os_release_name", new='precise')
@patch.object(System, "os_family", new='ubuntu')
@patch("ambari_commons.os_utils.current_user", new=MagicMock(return_value='ambari-agent'))
def test_create_repo_ubuntu_gpg_key_wrong_output(self, file_mock, execute_mock,
tempfile_mock, call_mock):
"""
@@ -268,7 +270,7 @@ def test_create_repo_ubuntu_gpg_key_wrong_output(self, file_mock, execute_mock,

copy_item0 = str(file_mock.call_args_list[1])
copy_item1 = str(file_mock.call_args_list[2])
self.assertEqual(copy_item0, "call('/tmp/1.txt', content=StaticFile('/etc/apt/sources.list.d/HDP.list'))")
self.assertEqual(copy_item0, "call('/tmp/1.txt', owner='ambari-agent', content=StaticFile('/etc/apt/sources.list.d/HDP.list'))")
self.assertEqual(copy_item1, "call('/etc/apt/sources.list.d/HDP.list', content=StaticFile('/tmp/1.txt'))")
execute_command_item = execute_mock.call_args_list[0][0][0]

@@ -23,7 +23,7 @@
import os
import time
from unittest import TestCase
from mock.mock import patch, MagicMock
from mock.mock import patch, MagicMock, ANY

from only_for_platform import get_platform, not_for_platform, os_distro_value, PLATFORM_WINDOWS

@@ -68,7 +68,8 @@ def test_action_create_empty_xml_config(self,
configuration_attributes={}
)

create_file_mock.assert_called_with('/dir/conf/file.xml', u' <configuration xmlns:xi="http://www.w3.org/2001/XInclude">\n \n </configuration>', encoding='UTF-8')
create_file_mock.assert_called_with('/dir/conf/file.xml', u' <configuration xmlns:xi="http://www.w3.org/2001/XInclude">\n \n </configuration>',
encoding='UTF-8', on_file_created=ANY)


@patch("resource_management.core.providers.system._ensure_metadata")
@@ -97,7 +98,8 @@ def test_action_create_simple_xml_config(self,
configuration_attributes={'attr': {'property1': 'attr_value'}}
)

create_file_mock.assert_called_with('/dir/conf/file.xml', u' <configuration xmlns:xi="http://www.w3.org/2001/XInclude">\n \n <property>\n <name>property1</name>\n <value>value1</value>\n <attr>attr_value</attr>\n </property>\n \n </configuration>', encoding='UTF-8')
create_file_mock.assert_called_with('/dir/conf/file.xml', u' <configuration xmlns:xi="http://www.w3.org/2001/XInclude">\n \n <property>\n <name>property1</name>\n <value>value1</value>\n <attr>attr_value</attr>\n </property>\n \n </configuration>',
encoding='UTF-8', on_file_created=ANY)

@patch("resource_management.core.providers.system._ensure_metadata")
@patch("resource_management.core.sudo.create_file")
@@ -126,7 +128,8 @@ def test_action_create_simple_xml_config_with_inclusion(self,
xml_include_file="/dif/conf/include_file.xml"
)

create_file_mock.assert_called_with('/dir/conf/file.xml', u' <configuration xmlns:xi="http://www.w3.org/2001/XInclude">\n \n <property>\n <name>property1</name>\n <value>value1</value>\n <attr>attr_value</attr>\n </property>\n \n <xi:include href="/dif/conf/include_file.xml"/>\n \n </configuration>', encoding='UTF-8')
create_file_mock.assert_called_with('/dir/conf/file.xml', u' <configuration xmlns:xi="http://www.w3.org/2001/XInclude">\n \n <property>\n <name>property1</name>\n <value>value1</value>\n <attr>attr_value</attr>\n </property>\n \n <xi:include href="/dif/conf/include_file.xml"/>\n \n </configuration>',
encoding='UTF-8', on_file_created=ANY)

@patch("resource_management.core.providers.system._ensure_metadata")
@patch("resource_management.core.sudo.create_file")
@@ -178,7 +181,8 @@ def test_action_create_xml_config_with_metacharacters(self,
}
})

create_file_mock.assert_called_with('/dir/conf/file.xml', u' <configuration xmlns:xi="http://www.w3.org/2001/XInclude">\n \n <property>\n <name></name>\n <value></value>\n </property>\n \n <property>\n <name>prop.1</name>\n <value>&#39;.&#39;yyyy-MM-dd-HH</value>\n <attr1>x</attr1>\n </property>\n \n <property>\n <name>prop.2</name>\n <value>INFO, openjpa</value>\n </property>\n \n <property>\n <name>prop.3</name>\n <value>%d{ISO8601} %5p %c{1}:%L - %m%n</value>\n <attr2>value3</attr2>\n </property>\n \n <property>\n <name>prop.4</name>\n <value>${oozie.log.dir}/oozie.log</value>\n <attr_value_empty></attr_value_empty>\n <attr2>value4</attr2>\n </property>\n \n <property>\n <name>prop.empty</name>\n <value></value>\n <attr_value_empty></attr_value_empty>\n </property>\n \n </configuration>', encoding='UTF-8')
create_file_mock.assert_called_with('/dir/conf/file.xml', u' <configuration xmlns:xi="http://www.w3.org/2001/XInclude">\n \n <property>\n <name></name>\n <value></value>\n </property>\n \n <property>\n <name>prop.1</name>\n <value>&#39;.&#39;yyyy-MM-dd-HH</value>\n <attr1>x</attr1>\n </property>\n \n <property>\n <name>prop.2</name>\n <value>INFO, openjpa</value>\n </property>\n \n <property>\n <name>prop.3</name>\n <value>%d{ISO8601} %5p %c{1}:%L - %m%n</value>\n <attr2>value3</attr2>\n </property>\n \n <property>\n <name>prop.4</name>\n <value>${oozie.log.dir}/oozie.log</value>\n <attr_value_empty></attr_value_empty>\n <attr2>value4</attr2>\n </property>\n \n <property>\n <name>prop.empty</name>\n <value></value>\n <attr_value_empty></attr_value_empty>\n </property>\n \n </configuration>',
encoding='UTF-8', on_file_created=ANY)

@patch("resource_management.core.providers.system._ensure_metadata")
@patch("resource_management.core.sudo.create_file")
@@ -211,7 +215,8 @@ def test_action_create_xml_config_sorted_by_key(self,
configuration_attributes={}
)

create_file_mock.assert_called_with('/dir/conf/file.xml', u' <configuration xmlns:xi="http://www.w3.org/2001/XInclude">\n \n <property>\n <name></name>\n <value></value>\n </property>\n \n <property>\n <name>first</name>\n <value>should be first</value>\n </property>\n \n <property>\n <name>second</name>\n <value>should be second</value>\n </property>\n \n <property>\n <name>third</name>\n <value>should be third</value>\n </property>\n \n <property>\n <name>z_last</name>\n <value>should be last</value>\n </property>\n \n </configuration>', encoding='UTF-8')
create_file_mock.assert_called_with('/dir/conf/file.xml', u' <configuration xmlns:xi="http://www.w3.org/2001/XInclude">\n \n <property>\n <name></name>\n <value></value>\n </property>\n \n <property>\n <name>first</name>\n <value>should be first</value>\n </property>\n \n <property>\n <name>second</name>\n <value>should be second</value>\n </property>\n \n <property>\n <name>third</name>\n <value>should be third</value>\n </property>\n \n <property>\n <name>z_last</name>\n <value>should be last</value>\n </property>\n \n </configuration>',
encoding='UTF-8', on_file_created=ANY)

@patch("resource_management.libraries.providers.xml_config.File")
@patch("resource_management.core.sudo.path_exists")
@@ -43,6 +43,12 @@
from ambari_commons.exceptions import FatalException
from ambari_commons.logging_utils import print_info_msg, print_warning_msg

def current_user():
if OSCheck.is_windows_family():
return None
else:
return pwd.getpwuid(os.geteuid())[0]

def get_used_ram():
"""
Returns resident RAM used by current process in kilobytes

0 comments on commit 8613d58

Please sign in to comment.