Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Windows build for 2 Ambari Metrics service: Host Monitoring and Timel…
…ine Service (Collector).

+ Added Windows profiles to the Maven project files
+ Added the necessary Windows assemblies
+ Created Windows service skeletons
+ Host Monitoring: added OS-independent process termination handler
+ Collector: added debugging support for the Java process
+ Fixed services shutdown, especially when joining spawned threads
+ Fixed unit tests
+ Added support for unit testing on MacOS and Windows

Windows-specific:
+ Moved the assembly descriptors to ambari-metrics-assembly
+ Fixed comments in the configuration files
+ Added soft dependencies on the embedded HBase service
+ Added support for the embedded HBase service setup
  • Loading branch information
fbarca committed Dec 30, 2014
1 parent 9884cbd commit 6c21b09
Show file tree
Hide file tree
Showing 36 changed files with 2,238 additions and 118 deletions.
2 changes: 1 addition & 1 deletion ambari-agent/conf/windows/service_wrapper.py
Expand Up @@ -92,7 +92,7 @@ def SvcDoRun(self):
# Soft dependency on the Windows Time service
ensure_time_service_is_started()

self.heartbeat_stop_handler = HeartbeatStopHandlers(self._heventSvcStop)
self.heartbeat_stop_handler = HeartbeatStopHandlers(AmbariAgentService._heventSvcStop)

self.ReportServiceStatus(win32service.SERVICE_RUNNING)

Expand Down
17 changes: 9 additions & 8 deletions ambari-common/src/main/python/ambari_commons/os_utils.py
Expand Up @@ -48,14 +48,17 @@ def quote_path(filepath):
filepath_ret = filepath
return filepath_ret

def search_file(filename, search_path, pathsep=os.pathsep):
""" Given a search path, find file with requested name """
def _search_file(filename, search_path, pathsep):
for path in string.split(search_path, pathsep):
candidate = os.path.join(path, filename)
if os.path.exists(candidate):
return os.path.abspath(candidate)
return None

def search_file(filename, search_path, pathsep=os.pathsep):
""" Given a search path, find file with requested name """
return _search_file(filename, search_path, pathsep)

def copy_file(src, dest_file):
try:
shutil.copyfile(src, dest_file)
Expand Down Expand Up @@ -103,9 +106,7 @@ def get_password(prompt):
return os_getpass(prompt)

def find_in_path(file):
dirs = os.environ["PATH"].split(os.pathsep)
for dir in dirs:
full_path = os.path.join(dir, file)
if os.path.exists(full_path):
return full_path
raise Exception("File {} not found in PATH".format(file))
full_path = _search_file(file, os.environ["PATH"], os.pathsep)
if full_path is None:
raise Exception("File {0} not found in PATH".format(file))
return full_path
21 changes: 12 additions & 9 deletions ambari-common/src/main/python/ambari_commons/os_windows.py
Expand Up @@ -402,7 +402,7 @@ class WinService(win32serviceutil.ServiceFramework):
# _svc_display_name_ = The service display name
# _svc_description_ = The service description

_heventSvcStop = win32event.CreateEvent(None, 0, 0, None)
_heventSvcStop = win32event.CreateEvent(None, 1, 0, None)
_hmtxOut = win32event.CreateMutex(None, False, None) #[fbarca] Python doesn't support critical sections

def __init__(self, *args):
Expand All @@ -418,17 +418,18 @@ def SvcDoRun(self):

def SvcStop(self):
self.ReportServiceStatus(win32service.SERVICE_STOP_PENDING)
win32event.SetEvent(self._heventSvcStop)
win32event.SetEvent(WinService._heventSvcStop)

# Service code entry point. Override it to implement the intended functionality.
def ServiceMain(self):
#Default implementation, does nothing.
win32event.WaitForSingleObject(self._heventSvcStop, win32event.INFINITE)
win32event.WaitForSingleObject(WinService._heventSvcStop, win32event.INFINITE)
pass

def DefCtrlCHandler(self):
@staticmethod
def DefCtrlCHandler():
print_info_msg("Ctrl+C handler invoked. Stopping.")
win32event.SetEvent(self._heventSvcStop)
win32event.SetEvent(WinService._heventSvcStop)
pass

#username domain\\username : The Username the service is to run under
Expand All @@ -438,7 +439,7 @@ def DefCtrlCHandler(self):
#perfmonini file: .ini file to use for registering performance monitor data
#perfmondll file: .dll file to use when querying the service for performance data, default = perfmondata.dll
@classmethod
def Install(cls, startupMode = "auto", username = None, password = None, interactive = False,
def Install(cls, classPath = None, startupMode = "auto", username = None, password = None, interactive = False,
perfMonIni = None, perfMonDll = None):
installArgs = [sys.argv[0], "--startup=" + startupMode]
if username is not None and username:
Expand All @@ -452,7 +453,8 @@ def Install(cls, startupMode = "auto", username = None, password = None, interac
if perfMonDll is not None and perfMonDll:
installArgs.append("--perfmondll=" + perfMonDll)
installArgs.append("install")
win32serviceutil.HandleCommandLine(cls, None, installArgs)

win32serviceutil.HandleCommandLine(cls, classPath, installArgs)

@classmethod
def Start(cls, waitSecs = 30):
Expand Down Expand Up @@ -483,11 +485,12 @@ def _RedirectOutputStreamsToFile(self, outFilePath):

def CheckForStop(self):
#Check for stop event to be signaled
return win32event.WAIT_OBJECT_0 == win32event.WaitForSingleObject(self._heventSvcStop, 1)
return win32event.WAIT_OBJECT_0 == win32event.WaitForSingleObject(WinService._heventSvcStop, 1)

def _StopOrWaitForChildProcessToFinish(self, childProcess):
#Wait for the child process to finish or for the stop event to be signaled
if(win32event.WAIT_OBJECT_0 == win32event.WaitForMultipleObjects([self._heventSvcStop, childProcess._handle], False, win32event.INFINITE)):
if(win32event.WAIT_OBJECT_0 == win32event.WaitForMultipleObjects([WinService._heventSvcStop, childProcess._handle],
False, win32event.INFINITE)):
# The OS only detaches the child process when the master process exits.
# We must kill it manually.
try:
Expand Down
33 changes: 33 additions & 0 deletions ambari-common/src/main/python/ambari_commons/xml_utils.py
@@ -0,0 +1,33 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
#* Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import inspect
import os

class ConvertToXml:
"Template class, allow to output fields in xml format"
def getField(self):
return [name for name, obj in inspect.getmembers(self)
if not name.startswith("__") and not inspect.isroutine(obj)]

def attributesToXml(self):
result = ""
listOfAttr = self.getField()
for attrName in listOfAttr:
result += "<" + attrName + ">"
result += str(getattr(self, attrName))
result += "</" + attrName + ">\n"
return result
77 changes: 73 additions & 4 deletions ambari-metrics/ambari-metrics-assembly/pom.xml
Expand Up @@ -26,9 +26,9 @@
<version>0.1.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

<artifactId>ambari-metrics-assembly</artifactId>
<packaging>pom</packaging>
<version>0.1.0-SNAPSHOT</version>

<properties>
<collector.dir>${project.basedir}/../ambari-metrics-timelineservice</collector.dir>
Expand All @@ -47,6 +47,33 @@

<build>
<plugins>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>build-helper-maven-plugin</artifactId>
<version>1.8</version>
<executions>
<execution>
<id>parse-version</id>
<phase>validate</phase>
<goals>
<goal>parse-version</goal>
</goals>
</execution>
<execution>
<id>regex-property</id>
<goals>
<goal>regex-property</goal>
</goals>
<configuration>
<name>ambariVersion</name>
<value>${project.version}</value>
<regex>^([0-9]+)\.([0-9]+)\.([0-9]+)(\.|-).*</regex>
<replacement>$1.$2.$3</replacement>
<failIfNoMatch>false</failIfNoMatch>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-resources-plugin</artifactId>
<version>2.7</version>
Expand Down Expand Up @@ -94,7 +121,7 @@
<finalName>ambari-metrics-collector-${project.version}</finalName>
<appendAssemblyId>false</appendAssemblyId>
<descriptors>
<descriptor>src/main/assembly/collector.xml</descriptor>
<descriptor>${assemblydescriptor.collector}</descriptor>
</descriptors>
<tarLongFileMode>gnu</tarLongFileMode>
</configuration>
Expand All @@ -110,7 +137,7 @@
<finalName>ambari-metrics-monitor-${project.version}</finalName>
<appendAssemblyId>false</appendAssemblyId>
<descriptors>
<descriptor>src/main/assembly/monitor.xml</descriptor>
<descriptor>${assemblydescriptor.monitor}</descriptor>
</descriptors>
<tarLongFileMode>gnu</tarLongFileMode>
</configuration>
Expand All @@ -126,7 +153,7 @@
<finalName>ambari-metrics-hadoop-sink-${project.version}</finalName>
<appendAssemblyId>false</appendAssemblyId>
<descriptors>
<descriptor>src/main/assembly/sink.xml</descriptor>
<descriptor>${assemblydescriptor.sink}</descriptor>
</descriptors>
<tarLongFileMode>gnu</tarLongFileMode>
</configuration>
Expand Down Expand Up @@ -710,6 +737,48 @@
</plugins>
</build>
</profile>
<profile>
<id>windows</id>
<activation>
<os>
<family>win</family>
</os>
</activation>
<properties>
<envClassifier>win</envClassifier>
<dirsep>\</dirsep>
<pathsep>;</pathsep>
<executable.python>python</executable.python>
<executable.shell>cmd</executable.shell>
<fileextension.shell>cmd</fileextension.shell>
<fileextension.dot.shell-default>.cmd</fileextension.dot.shell-default>
<assemblydescriptor.collector>src/main/assembly/collector-windows.xml</assemblydescriptor.collector>
<assemblydescriptor.monitor>src/main/assembly/monitor-windows.xml</assemblydescriptor.monitor>
<assemblydescriptor.sink>src/main/assembly/sink-windows.xml</assemblydescriptor.sink>
<packagingFormat>jar</packagingFormat>
</properties>
</profile>
<profile>
<id>linux</id>
<activation>
<os>
<family>unix</family>
</os>
</activation>
<properties>
<envClassifier>linux</envClassifier>
<dirsep>/</dirsep>
<pathsep>:</pathsep>
<executable.python>${project.basedir}/../ambari-common/src/main/unix/ambari-python-wrap</executable.python>
<executable.shell>sh</executable.shell>
<fileextension.shell>sh</fileextension.shell>
<fileextension.dot.shell-default></fileextension.dot.shell-default>
<assemblydescriptor.collector>src/main/assembly/collector.xml</assemblydescriptor.collector>
<assemblydescriptor.monitor>src/main/assembly/monitor.xml</assemblydescriptor.monitor>
<assemblydescriptor.sink>src/main/assembly/sink.xml</assemblydescriptor.sink>
<packagingFormat>jar</packagingFormat>
</properties>
</profile>
</profiles>


Expand Down
@@ -0,0 +1,101 @@
<?xml version="1.0"?>

<!--
~ Licensed to the Apache Software Foundation (ASF) under one
~ or more contributor license agreements. See the NOTICE file
~ distributed with this work for additional information
~ regarding copyright ownership. The ASF licenses this file
~ to you under the Apache License, Version 2.0 (the
~ "License"); you may not use this file except in compliance
~ with the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->

<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.1 http://maven.apache.org/xsd/assembly-1.1.1.xsd">
<id>collector</id>
<formats>
<format>dir</format>
</formats>
<includeBaseDirectory>false</includeBaseDirectory>
<fileSets>
<fileSet>
<directory>${collector.dir}/target/embedded/${hbase.folder}</directory>
<outputDirectory>hbase</outputDirectory>
</fileSet>
<fileSet>
<directory>${collector.dir}/conf/windows</directory>
<outputDirectory>/</outputDirectory>
<includes>
<include>ambari-metrics-collector.cmd</include>
</includes>
</fileSet>
<fileSet>
<directory>${collector.dir}/conf/windows</directory>
<outputDirectory>conf</outputDirectory>
<includes>
<include>ams.properties</include>
<include>ams-env.cmd</include>
<include>ams-site.xml</include>
<include>log4j.properties</include>
</includes>
</fileSet>
<fileSet>
<directory>${collector.dir}/target/lib</directory>
<outputDirectory>lib</outputDirectory>
</fileSet>
<fileSet>
<directory>${collector.dir}/src/main/python</directory>
<outputDirectory>/sbin</outputDirectory>
<includes>
<include>*.py</include>
</includes>
</fileSet>
<fileSet>
<directory>${collector.dir}/src/main/python/ambari_metrics_collector</directory>
<outputDirectory>/sbin/ambari_metrics_collector</outputDirectory>
<includes>
<include>*.py</include>
</includes>
</fileSet>
<fileSet>
<directory>${project.basedir}/../../ambari-common/src/main/python/ambari_commons</directory>
<outputDirectory>/sbin/ambari_commons</outputDirectory>
<includes>
<include>*.py</include>
</includes>
</fileSet>
<fileSet>
<directory>${project.basedir}/../../ambari-common/src/main/python/ambari_commons/resources</directory>
<outputDirectory>/sbin/ambari_commons/resources</outputDirectory>
<includes>
<include>*.json</include>
</includes>
</fileSet>
</fileSets>
<dependencySets>
<dependencySet>
<unpack>false</unpack>
<outputDirectory>hbase/lib</outputDirectory>
<useProjectArtifact>false</useProjectArtifact>
<includes>
<include>org.antlr:antlr*</include>
<include>org.apache.phoenix:phoenix-core</include>
</includes>
</dependencySet>
<dependencySet>
<unpack>false</unpack>
<outputDirectory>lib</outputDirectory>
<useProjectArtifact>false</useProjectArtifact>
<includes>
<include>org.apache.ambari:ambari-metrics-timelineservice</include>
</includes>
</dependencySet>
</dependencySets>
</assembly>

0 comments on commit 6c21b09

Please sign in to comment.