Skip to content

Commit

Permalink
Add scripts to Macos folder when building App
Browse files Browse the repository at this point in the history 
Allows to run airflow from the app
  • Loading branch information
@michael-kotliar
michael-kotliar committed Sep 13, 2019
1 parent 32d1c7f commit c5a9654
Show file tree
Hide file tree
Showing 7 changed files with 82 additions and 44 deletions.
1 change: 1 addition & 0 deletions cwl_airflow/app/app.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ def unload(self, widget):
def startup(self):
# Create Launcher
self.__launcher = Launcher(self.__airflow_home_default)
self.__launcher.configure()

# Create a main window with a name matching the app
self.main_window = toga.MainWindow(title=self.name, size=(250,100))
Expand Down
89 changes: 45 additions & 44 deletions cwl_airflow/app/launch.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,22 +14,23 @@ class Launcher:
__AIRFLOW_WEB = "~/Library/LaunchAgents/com.biowardrobe.airflow-webserver.plist"
__AIRFLOW_API = "~/Library/LaunchAgents/com.biowardrobe.airflow-apiserver.plist"

__airflow_home = None
__airflow_cfg = None
airflow_home = None
airflow_cfg = None
contents_dir = None
__sch_conf = None
__web_conf = None
__api_conf = None
__contents_dir = None



def __init__(self, airflow_home = "~/airflow"):
self.__contents_dir = os.path.dirname(os.path.abspath(os.path.join(__file__, "../../../../")))
self.__airflow_home = os.path.expanduser(airflow_home)
self.__airflow_cfg = os.path.join(self.__airflow_home, "airflow.cfg")
self.__configure()
if airflow_home == None:
airflow_home = "~/airflow"
self.contents_dir = os.path.dirname(os.path.abspath(os.path.join(__file__, "../../../../")))
self.airflow_home = os.path.expanduser(airflow_home)
self.airflow_cfg = os.path.join(self.airflow_home, "airflow.cfg")


def __configure(self):
def configure(self):
try:
self.__sch_conf = self.__read_plist(os.path.expanduser(self.__AIRFLOW_SCH))
self.__web_conf = self.__read_plist(os.path.expanduser(self.__AIRFLOW_WEB))
Expand All @@ -39,13 +40,13 @@ def __configure(self):
self.__sch_conf = self.__read_plist(os.path.join(default_conf_folder, os.path.basename(self.__AIRFLOW_SCH)))
self.__web_conf = self.__read_plist(os.path.join(default_conf_folder, os.path.basename(self.__AIRFLOW_WEB)))
self.__api_conf = self.__read_plist(os.path.join(default_conf_folder, os.path.basename(self.__AIRFLOW_API)))
self.__update_variables()
self.__update_shebang(os.path.join(self.__contents_dir, "Resources/app_packages/bin"))
self.__update_shebang(os.path.join(self.__contents_dir, "Resources/app/bin"))
self.__init_airflow_db()
self.__update_airflow_config()
self.__copy_dags()
self.__add_connections()
self.__update_plist_variables()
self.update_shebang(os.path.join(self.contents_dir, "Resources/app_packages/bin"))
self.update_shebang(os.path.join(self.contents_dir, "Resources/app/bin"))
self.init_airflow_db()
self.update_airflow_config()
self.copy_dags()
self.add_connections()
self.__write_plist(self.__sch_conf, os.path.expanduser(self.__AIRFLOW_SCH))
self.__write_plist(self.__web_conf, os.path.expanduser(self.__AIRFLOW_WEB))
self.__write_plist(self.__api_conf, os.path.expanduser(self.__AIRFLOW_API))
Expand All @@ -63,10 +64,10 @@ def __write_plist(self, data, path):

def __get_path(self):
path_list = [
os.path.join(self.__contents_dir, "Resources/python/bin"),
os.path.join(self.__contents_dir, "Resources/app_packages/bin"),
os.path.join(self.__contents_dir, "Resources/app/bin"),
os.path.join(self.__contents_dir, "MacOS"),
os.path.join(self.contents_dir, "Resources/python/bin"),
os.path.join(self.contents_dir, "Resources/app_packages/bin"),
os.path.join(self.contents_dir, "Resources/app/bin"),
os.path.join(self.contents_dir, "MacOS"),
"/usr/local/bin",
"/usr/local/sbin",
"/usr/bin",
Expand All @@ -79,20 +80,20 @@ def __get_path(self):

def __get_pythonpath(self):
pythonpath_list = [
os.path.join(self.__contents_dir, "Resources/app"),
os.path.join(self.__contents_dir, "Resources/app_packages")
os.path.join(self.contents_dir, "Resources/app"),
os.path.join(self.contents_dir, "Resources/app_packages")
]
return ":".join(dict.fromkeys(pythonpath_list).keys())


def __update_variables(self):
def __update_plist_variables(self):
path = self.__get_path()
pythonpath = self.__get_pythonpath()
airflow_exe = os.path.join(self.__contents_dir, "Resources/app_packages/bin/airflow")
cwlairflow_exe = os.path.join(self.__contents_dir, "Resources/app/bin/cwl-airflow")
log_sch = os.path.join(self.__airflow_home, os.path.splitext(os.path.basename(self.__AIRFLOW_SCH))[0])
log_web = os.path.join(self.__airflow_home, os.path.splitext(os.path.basename(self.__AIRFLOW_WEB))[0])
log_api = os.path.join(self.__airflow_home, os.path.splitext(os.path.basename(self.__AIRFLOW_API))[0])
airflow_exe = os.path.join(self.contents_dir, "Resources/app_packages/bin/airflow")
cwlairflow_exe = os.path.join(self.contents_dir, "Resources/app/bin/cwl-airflow")
log_sch = os.path.join(self.airflow_home, os.path.splitext(os.path.basename(self.__AIRFLOW_SCH))[0])
log_web = os.path.join(self.airflow_home, os.path.splitext(os.path.basename(self.__AIRFLOW_WEB))[0])
log_api = os.path.join(self.airflow_home, os.path.splitext(os.path.basename(self.__AIRFLOW_API))[0])

self.__sch_conf["EnvironmentVariables"]["PATH"] = path
self.__web_conf["EnvironmentVariables"]["PATH"] = path
Expand All @@ -102,17 +103,17 @@ def __update_variables(self):
self.__web_conf["EnvironmentVariables"]["PYTHONPATH"] = pythonpath
self.__api_conf["EnvironmentVariables"]["PYTHONPATH"] = pythonpath

self.__sch_conf["EnvironmentVariables"]["AIRFLOW_HOME"] = self.__airflow_home
self.__web_conf["EnvironmentVariables"]["AIRFLOW_HOME"] = self.__airflow_home
self.__api_conf["EnvironmentVariables"]["AIRFLOW_HOME"] = self.__airflow_home
self.__sch_conf["EnvironmentVariables"]["AIRFLOW_HOME"] = self.airflow_home
self.__web_conf["EnvironmentVariables"]["AIRFLOW_HOME"] = self.airflow_home
self.__api_conf["EnvironmentVariables"]["AIRFLOW_HOME"] = self.airflow_home

self.__sch_conf["ProgramArguments"][0] = airflow_exe
self.__web_conf["ProgramArguments"][0] = airflow_exe
self.__api_conf["ProgramArguments"][0] = cwlairflow_exe

self.__sch_conf["WorkingDirectory"] = self.__airflow_home
self.__web_conf["WorkingDirectory"] = self.__airflow_home
self.__api_conf["WorkingDirectory"] = self.__airflow_home
self.__sch_conf["WorkingDirectory"] = self.airflow_home
self.__web_conf["WorkingDirectory"] = self.airflow_home
self.__api_conf["WorkingDirectory"] = self.airflow_home

self.__sch_conf["StandardErrorPath"] = log_sch + ".stderr"
self.__sch_conf["StandardOutPath"] = log_sch + ".stdout"
Expand All @@ -124,8 +125,8 @@ def __update_variables(self):
self.__api_conf["StandardOutPath"] = log_api + ".stdout"


def __update_shebang(self, lookup_dir):
shebang = "#!{}".format(os.path.join(self.__contents_dir, "Resources/python/bin/python3"))
def update_shebang(self, lookup_dir):
shebang = "#!{}".format(os.path.join(self.contents_dir, "Resources/python/bin/python3"))
for filename in os.listdir(lookup_dir):
filename = os.path.join(lookup_dir, filename)
if not os.path.isfile(filename):
Expand All @@ -142,9 +143,9 @@ def __update_shebang(self, lookup_dir):
f.write("\n".join(script).encode("utf-8"))


def __init_airflow_db(self):
def init_airflow_db(self):
env = {
"AIRFLOW_HOME": self.__airflow_home,
"AIRFLOW_HOME": self.airflow_home,
"PATH": self.__get_path(),
"PYTHONPATH": self.__get_pythonpath()
}
Expand All @@ -153,18 +154,18 @@ def __init_airflow_db(self):

def __get_configuration(self):
conf = configparser.ConfigParser()
conf.read(self.__airflow_cfg)
conf.read(self.airflow_cfg)
return conf


def __update_airflow_config(self):
def update_airflow_config(self):
conf = self.__get_configuration()
with open(self.__airflow_cfg, 'w') as f:
with open(self.airflow_cfg, 'w') as f:
try:
conf.add_section('cwl')
except configparser.DuplicateSectionError:
pass
conf.set("cwl", "tmp_folder", os.path.join(self.__airflow_home, 'tmp'))
conf.set("cwl", "tmp_folder", os.path.join(self.airflow_home, 'tmp'))
conf.set("core", "logging_level", "INFO")
conf.set("core", "load_examples", "False")
conf.set("core", "dags_are_paused_at_creation", "False")
Expand All @@ -174,7 +175,7 @@ def __update_airflow_config(self):
conf.write(f)


def __copy_dags(self):
def copy_dags(self):
conf = self.__get_configuration()
source_dags_folder = os.path.join(os.path.dirname(os.path.abspath(os.path.join(__file__, "../"))), "dags")
target_dags_folder = conf.get("core", "dags_folder")
Expand All @@ -185,7 +186,7 @@ def __copy_dags(self):
shutil.copy(os.path.join(root, filename), target_dags_folder)


def __add_connections(self):
def add_connections(self):
merge_conn(models.Connection(conn_id = "process_report",
conn_type = "http",
host = "localhost",
Expand Down
15 changes: 15 additions & 0 deletions cwl_airflow/bin/cwl-airflow
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,19 @@
#!/usr/bin/env python
import sys
import os
import argparse
from cwl_airflow.wes.server import run_wes_server
from cwl_airflow.app.launch import Launcher


def run_appinit(args):
launcher = Launcher(os.environ.get("AIRFLOW_HOME"))
launcher.update_shebang(os.path.join(launcher.contents_dir, "Resources/app_packages/bin"))
launcher.update_shebang(os.path.join(launcher.contents_dir, "Resources/app/bin"))
launcher.init_airflow_db()
launcher.update_airflow_config()
launcher.copy_dags()
launcher.add_connections()


def arg_parser():
Expand All @@ -15,6 +27,9 @@ def arg_parser():
wes_parser.add_argument("--port", dest="port", type=int, help="Port to run API server (default: 8081)", default=8081)
wes_parser.add_argument("--host", dest="host", help="Host to run API server (default: 127.0.0.1)", default="127.0.0.1")

init_parser = subparsers.add_parser('appinit', help="Init CWL-Airflow configuration for App", parents=[parent_parser])
init_parser.set_defaults(func=run_appinit)

return general_parser


Expand Down
5 changes: 5 additions & 0 deletions dev/macos/airflow
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
#!/bin/bash
CONTENTS_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )
export PYTHONPATH="$CONTENTS_DIR/Resources/app":"$CONTENTS_DIR/Resources/app_packages"
cd "$CONTENTS_DIR/Resources"
python/bin/python3 ./app_packages/bin/airflow "$@"
5 changes: 5 additions & 0 deletions dev/macos/apiserver
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
#!/bin/bash
CONTENTS_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )
export PYTHONPATH="$CONTENTS_DIR/Resources/app":"$CONTENTS_DIR/Resources/app_packages"
cd "$CONTENTS_DIR/Resources"
python/bin/python3 ./app/bin/cwl-airflow apiserver "$@"
5 changes: 5 additions & 0 deletions dev/macos/appinit
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
#!/bin/bash
CONTENTS_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )
export PYTHONPATH="$CONTENTS_DIR/Resources/app":"$CONTENTS_DIR/Resources/app_packages"
cd "$CONTENTS_DIR/Resources"
python/bin/python3 ./app/bin/cwl-airflow appinit
6 changes: 6 additions & 0 deletions dev/macos/cwl-airflow
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
#!/bin/bash
CONTENTS_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )
export PYTHONPATH="$CONTENTS_DIR/Resources/app":"$CONTENTS_DIR/Resources/app_packages"
cd "$CONTENTS_DIR/Resources"
python/bin/python3 ./app/bin/cwl-airflow appinit
python/bin/python3 -m cwl_airflow

0 comments on commit c5a9654

Please sign in to comment.