Reorganize repo for amber service architecture

metainfo.xml

@@ -0,0 +1,31 @@
+<?xml version="1.0"?>
+<metainfo>
+    <schemaVersion>2.0</schemaVersion>
+    <services>
+        <service>
+            <name>DEMOSERVICE</name>
+            <displayName>HDP Demo</displayName>
+            <comment>An Ambari service to manage the demo</comment>
+            <version>0.0.1</version>
+            <components>
+                <component>
+                    <name>DEMO MASTER</name>
+                    <displayName>Demo Master Service</displayName>
+                    <category>MASTER</category>
+                    <cardinality>1</cardinality>
+                    <commandScript>
+                        <script>scripts/master.py</script>
+                        <scriptType>PYTHON</scriptType>
+                        <timeout>600</timeout>
+                    </commandScript>
+                </component>
+            </components>
+            <osSpecifics>
+                <osSpecific>
+                    <osFamily>any</osFamily>
+                </osSpecific>
+            </osSpecifics>
+        </service>
+    </services>
+</metainfo>
+

package/scripts/master.py

@@ -0,0 +1,38 @@
+import sys, util
+from resource_management import *
+
+# INSTRUCTIONS ON IMPLEMENTING SERVICE METHODS
+# 
+# Note: all of this information can be found
+# under ./docs/ambari-service.md
+# 
+# 
+
+
+
+
+
+
+class Master(Script):
+
+
+	def install(self, env):
+		print 'Install the Sample Srv Master';
+
+	def stop(self, env):
+		print 'Stop the Sample Srv Master';
+
+	def start(self, env):
+		print 'Start the Sample Srv Master';
+
+	def status(self, env):
+		print 'Status of the Sample Srv Master';
+
+	def configure(self, env):
+		print 'Configure the Sample Srv Master';
+
+
+
+if __name__ == "__main__":
+	Master().execute()
+

package/scripts/params.py

@@ -0,0 +1,220 @@
+import os
+import sys, os, pwd, grp, signal, time, glob
+from resource_management import *
+from subprocess import call
+
+reload(sys)
+sys.setdefaultencoding('utf8')
+
+class Master(Script):
+  def install(self, env):
+
+    import params
+    import status_params
+
+
+    #location of prebuilt package from 9/3   
+    #snapshot_package='https://www.dropbox.com/s/la1c25xq9zd8w5g/nifi-0.3.0-SNAPSHOT-bin.zip'
+
+    #official 0.3
+    #snapshot_package='http://apache.arvixe.com/nifi/0.3.0/nifi-0.3.0-bin.zip'
+
+    #official HDF 1.0 package (same as apache 0.3.0)
+    #snapshot_package='http://public-repo-1.hortonworks.com/HDF/1.0.0.0/nifi-1.0.0.0-7-bin.zip'
+
+    #official HDF 1.1.1 package (nifi 0.4.1)
+    #snapshot_package='http://public-repo-1.hortonworks.com/HDF/1.1.1.0/nifi-1.1.1.0-12-bin.zip'           
+
+    #official HDF 1.1.2 package     
+    #snapshot_package='http://public-repo-1.hortonworks.com/HDF/1.1.2.0/nifi-0.5.1.1.1.2.0-32-bin.zip'
+
+    #official HDF 1.2 package (nifi 0.6.0)
+    #snapshot_package='http://public-repo-1.hortonworks.com/HDF/centos6/1.x/updates/1.2.0.0/HDF-1.2.0.0-91.zip'
+
+    # HDF 1.2.0.1
+    snapshot_package='http://public-repo-1.hortonworks.com/HDF/centos6/1.x/updates/1.2.0.1/HDF-1.2.0.1-1.zip'
+
+    #e.g. /var/lib/ambari-agent/cache/stacks/HDP/2.3/services/NIFI/package
+    service_packagedir = os.path.realpath(__file__).split('/scripts')[0] 
+
+    #Execute('find '+service_packagedir+' -iname "*.sh" | xargs chmod +x')
+
+    #Create user and group if they don't exist
+    self.create_linux_user(params.nifi_user, params.nifi_group)
+    if params.nifi_user != 'root':
+      Execute('cp /etc/sudoers /etc/sudoers.bak')        
+      Execute('echo "'+params.nifi_user+'    ALL=(ALL)       NOPASSWD: ALL" >> /etc/sudoers')
+      Execute('echo Creating ' +  params.nifi_log_dir +  ' ' +  status_params.nifi_pid_dir)    
+
+
+
+    #create the log dir if it not already present
+    Directory([status_params.nifi_pid_dir, params.nifi_log_dir],
+            owner=params.nifi_user,
+            group=params.nifi_group,
+            recursive=True
+    )   
+
+    Execute('touch ' +  params.nifi_log_file, user=params.nifi_user)    
+    Execute('rm -rf ' + params.nifi_dir, ignore_failures=True)
+    #Execute('mkdir -p '+params.nifi_dir)
+    #Execute('chown -R ' + params.nifi_user + ':' + params.nifi_group + ' ' + params.nifi_dir)
+
+    Directory([params.nifi_dir],
+            owner=params.nifi_user,
+            group=params.nifi_group,
+            recursive=True
+    )  
+
+    #User selected option to use prebuilt nifi package 
+    if params.setup_prebuilt:
+
+      Execute('echo Installing packages')
+
+      #Install maven repo if needed      
+      #self.install_mvn_repo()
+      # Install packages listed in metainfo.xml
+      #self.install_packages(env)    
+
+
+
+      #Fetch and unzip snapshot build, if no cached nifi tar package exists on Ambari server node
+      if not os.path.exists(params.temp_file):
+        Execute('wget '+snapshot_package+' -O '+params.temp_file+' -a '  + params.nifi_log_file, user=params.nifi_user)
+      Execute('unzip '+params.temp_file+' -d ' + params.nifi_install_dir + ' >> ' + params.nifi_log_file, user=params.nifi_user)
+      Execute('mv '+params.nifi_dir+'/*/* ' + params.nifi_dir, user=params.nifi_user)
+
+
+      #params.conf_dir = os.path.join(*[params.nifi_install_dir,params.nifi_dirname,'conf'])
+      #params.bin_dir = os.path.join(*[params.nifi_install_dir,params.nifi_dirname,'bin'])
+
+      #update the configs specified by user
+      self.configure(env, True)
+
+      #Execute('wget https://www.dropbox.com/s/n82hxkeg8ri0z70/flow.xml.gz -O '+params.conf_dir+'/flow.xml.gz',user=params.nifi_user)
+
+      #run setup_snapshot.sh in FIRSTLAUNCH mode
+      #Execute(service_packagedir + '/scripts/setup_snapshot.sh '+params.nifi_dir+' '+params.hive_server_host+' '+params.hive_metastore_host+' '+params.hive_metastore_port+' FIRSTLAUNCH ' + params.spark_jar + ' ' + params.nifi_host + ' ' + str(params.nifi_port) + ' '+ str(params.setup_view) + ' >> ' + params.nifi_log_file, user=params.nifi_user)
+
+      #if nifi installed on ambari server, copy view jar into ambari views dir
+      #if params.setup_view:
+      #  if params.ambari_host == params.nifi_internalhost and not os.path.exists('/var/lib/ambari-server/resources/views/nifi-view-1.0-SNAPSHOT.jar'):
+      #    Execute('echo "Copying nifi view jar to ambari views dir"')      
+      #    Execute('cp /home/'+params.nifi_user+'/nifi-view/target/*.jar /var/lib/ambari-server/resources/views')
+
+    else:
+      #User selected option to build nifi from source
+
+      #if params.setup_view:
+        #Install maven repo if needed
+      self.install_mvn_repo()      
+      # Install packages listed in metainfo.xml
+      self.install_packages(env)    
+
+      # Execute('yum -y install java-1.7.0-openjdk-devel >> ' + params.nifi_log_file)
+
+      Execute('echo Compiling nifi from source')
+      Execute('cd '+params.nifi_install_dir+'; git clone https://git-wip-us.apache.org/repos/asf/nifi.git '+params.nifi_dir+' >> ' + params.nifi_log_file)
+      Execute('chown -R ' + params.nifi_user + ':' + params.nifi_group + ' ' + params.nifi_dir)
+
+      Execute('cd '+params.nifi_dir+'; mvn -T C2.0 clean install -DskipTests >> ' + params.nifi_log_file, user=params.nifi_user)
+
+      #params.conf_dir =  glob.glob(params.nifi_install_dir + '/' + params.nifi_dirname + '/nifi-assembly/target/nifi-*/nifi-*/conf')[0]
+      #params.bin_dir =  glob.glob(params.nifi_install_dir + '/' + params.nifi_dirname + '/nifi-assembly/target/nifi-*/nifi-*/bin')[0]
+
+      #update the configs specified by user
+      self.configure(env, True)
+      #Execute('wget https://www.dropbox.com/s/n82hxkeg8ri0z70/flow.xml.gz -O '+params.conf_dir+'/flow.xml.gz',user=params.nifi_user)
+
+      #if nifi installed on ambari server, copy view jar into ambari views dir
+      #if params.setup_view:
+      #  if params.ambari_host == params.nifi_internalhost and not os.path.exists('/var/lib/ambari-server/resources/views/nifi-view-1.0-SNAPSHOT.jar'):
+      #    Execute('echo "Copying nifi view jar to ambari views dir"')      
+      #    Execute('cp /home/'+params.nifi_user+'/nifi-view/target/*.jar /var/lib/ambari-server/resources/views')
+
+
+
+  def create_linux_user(self, user, group):
+    try: pwd.getpwnam(user)
+    except KeyError: Execute('adduser ' + user)
+    try: grp.getgrnam(group)
+    except KeyError: Execute('groupadd ' + group)
+
+
+
+  def configure(self, env, isInstall=False):
+    import params
+    import status_params
+    env.set_params(params)
+    env.set_params(status_params)
+
+    self.set_conf_bin(env)
+
+    #write out nifi.properties
+    properties_content=InlineTemplate(params.nifi_properties_content)
+    File(format("{params.conf_dir}/nifi.properties"), content=properties_content, owner=params.nifi_user, group=params.nifi_group) # , mode=0777)    
+
+    #write out flow.xml.gz only during install
+    if isInstall:
+      Execute('echo "First time setup so generating flow.xml.gz" >> ' + params.nifi_log_file)    
+      flow_content=InlineTemplate(params.nifi_flow_content)
+      File(format("{params.conf_dir}/flow.xml"), content=flow_content, owner=params.nifi_user, group=params.nifi_group)
+      Execute(format("cd {params.conf_dir}; mv flow.xml.gz flow_$(date +%d-%m-%Y).xml.gz ;"),user=params.nifi_user,ignore_failures=True)
+      Execute(format("cd {params.conf_dir}; gzip flow.xml;"), user=params.nifi_user)
+
+    #write out boostrap.conf
+    bootstrap_content=InlineTemplate(params.nifi_boostrap_content)
+    File(format("{params.conf_dir}/bootstrap.conf"), content=bootstrap_content, owner=params.nifi_user, group=params.nifi_group) 
+
+    #write out logback.xml
+    logback_content=InlineTemplate(params.nifi_logback_content)
+    File(format("{params.conf_dir}/logback.xml"), content=logback_content, owner=params.nifi_user, group=params.nifi_group) 
+
+
+
+  def stop(self, env):
+    import params
+    import status_params    
+    self.set_conf_bin(env)    
+    Execute (params.bin_dir+'/nifi.sh stop >> ' + params.nifi_log_file, user=params.nifi_user)
+    Execute ('rm ' + status_params.nifi_pid_file)
+
+
+  def start(self, env):
+    import params
+    import status_params
+    self.configure(env) 
+    self.set_conf_bin(env)    
+    Execute('echo pid file ' + status_params.nifi_pid_file)
+    Execute('echo JAVA_HOME=' + params.jdk64_home)
+
+    Execute ('export JAVA_HOME='+params.jdk64_home+';'+params.bin_dir+'/nifi.sh start >> ' + params.nifi_log_file, user=params.nifi_user)
+
+    Execute('cat '+params.bin_dir+'/nifi.pid'+" | grep pid | sed 's/pid=\(\.*\)/\\1/' > " + status_params.nifi_pid_file)
+    Execute('chown '+params.nifi_user+':'+params.nifi_group+' ' + status_params.nifi_pid_file)
+
+  def status(self, env):
+    import status_params       
+    check_process_status(status_params.nifi_pid_file)
+
+  def install_mvn_repo(self):
+    #for centos/RHEL 6/7 maven repo needs to be installed
+    distribution = platform.linux_distribution()[0].lower()
+    if distribution in ['centos', 'redhat'] and not os.path.exists('/etc/yum.repos.d/epel-apache-maven.repo'):
+      Execute('curl -o /etc/yum.repos.d/epel-apache-maven.repo https://repos.fedorapeople.org/repos/dchen/apache-maven/epel-apache-maven.repo')
+
+  def set_conf_bin(self, env):
+    import params
+
+    if params.setup_prebuilt:
+      params.conf_dir = os.path.join(*[params.nifi_install_dir,params.nifi_dirname,'conf'])
+      params.bin_dir = os.path.join(*[params.nifi_install_dir,params.nifi_dirname,'bin'])
+    else:
+      params.conf_dir =  glob.glob(params.nifi_install_dir + '/' + params.nifi_dirname + '/nifi-assembly/target/nifi-*/nifi-*/conf')[0]
+      params.bin_dir =  glob.glob(params.nifi_install_dir + '/' + params.nifi_dirname + '/nifi-assembly/target/nifi-*/nifi-*/bin')[0]
+
+
+if __name__ == "__main__":
+  Master().execute()
+Status API Training Shop Blog About
+© 2016 GitHub, Inc. Terms Privacy Security Contact Help

scripts/config.py → package/util/config.py

@@ -29,15 +29,16 @@ def read_config(configFile):
 	return params
 
 def get_conf_dir():
-	dirs = [str(os.getcwd()), str(os.curdir), '../']
+	dirs = [str(os.getcwd()), str(os.curdir), '../', os.path.dirname(os.path.abspath(__file__)) + '/..']
 
 	for loc in dirs:
 		if not (str(loc).endswith('/')):
 			loc += '/'
-		loc += 'conf/'
+		loc += 'configuration/'
 		if(os.path.exists(loc)):
+			print('LOCATION: ' + loc)
 			return loc
-
+	print(str(dirs))
 	raise EnvironmentError('Could not find conf directory')
 
 

package/util/cw.py

@@ -0,0 +1,5 @@
+import os
+
+print('WORKING DIRECTORY: ' + os.getcwd())
+print('FILE DIRECTORY: ' + os.path.abspath(__file__))
+print('OS DIR: ' + os.path.dirname(os.path.abspath(__file__)))

package/util/demo.py

@@ -0,0 +1,74 @@
+import sys
+
+
+
+def install():
+	print('installing')
+
+
+
+	pass
+
+def start():
+	print('uninstall')
+	pass
+
+
+usage = """
+usage: python demo.py [-h]|{install|start|stop|restart}
+"""
+
+help_text = """
+
+Use this script to install/start/stop/restart any demo services or scripts on the local machine
+
+-h          
+	Shows this help text. Must always be first argument. You can combine it with any other command to get the help text for that command as well.
+
+install     
+	Installs necessary functions and components to the current host. This behavior is defined by the user in the install() function.
+
+start		
+	Starts the demo services. User defined due to dependency on the demo. If using Ambari service this command may not be implemented
+
+stop        
+	Stops the demo services. User defined due to dependency on the demo. If using Ambari service this command may not be implemented
+
+restart     
+	Stops the demo and then starts the demo services. This command may perform no function if start() and stop() are not implemented
+""" 
+
+accepted_args = ['-h', 'install', 'start', 'stop', 'restart']
+
+help_dict = {
+	"install": "Installs necessary functions and components to the current host. This behavior is defined by the user in the install() function.",
+	"start": "Starts the demo services. User defined due to dependency on the demo. If using Ambari service this command may not be implemented",
+	"stop": "Stops the demo services. User defined due to dependency on the demo. If using Ambari service this command may not be implemented",
+	"restart": "Stops the demo and then starts the demo services. This command may perform no function if start() and stop() are not implemented"
+}
+
+if len(sys.argv) > 1:
+	args = sys.argv[1:]
+
+	if args[0] == '-h' and len(args) > 1:
+		print(args[1] + '\n\t\t' + help_dict[args[1]])
+	elif args[0] == '-h':
+		print(help_text)
+	elif len(args) > 0:
+		a = args
+		if 'install' in a:
+			install()
+		elif 'start' in a:
+			start()
+		elif 'stop' in a:
+			stop()
+		elif 'restart' in a:
+			restart()
+		else:
+			print(usage)
+	else:
+		print(usage)
+else:
+	print(usage)
+
+