From e1cc4b3595e374dff19ce267eb4d57a6b785d57e Mon Sep 17 00:00:00 2001 From: Roman Nyschuk Date: Sat, 4 May 2019 05:22:44 +1200 Subject: [PATCH] Create Device42 to FreshService Script --- LICENSE | 13 ++ README.md | 55 +++++++++ d42_sd_sync.py | 295 +++++++++++++++++++++++++++++++++++++++++++++ device42.py | 120 ++++++++++++++++++ freshservice.py | 241 ++++++++++++++++++++++++++++++++++++ mapping.xml.sample | 89 ++++++++++++++ requirements.txt | 5 + 7 files changed, 818 insertions(+) create mode 100644 LICENSE create mode 100644 README.md create mode 100755 d42_sd_sync.py create mode 100644 device42.py create mode 100755 freshservice.py create mode 100644 mapping.xml.sample create mode 100644 requirements.txt diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..170f5d5 --- /dev/null +++ b/LICENSE @@ -0,0 +1,13 @@ +Copyright [2019] [Device42, Inc.] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..c5e1aec --- /dev/null +++ b/README.md @@ -0,0 +1,55 @@ +[Device42](http://www.device42.com/) is a Continuous Discovery software for your IT Infrastructure. It helps you automatically maintain an up-to-date inventory of your physical, virtual, and cloud servers and containers, network components, software/services/applications, and their inter-relationships and inter-dependencies. + + +This repository contains script that helps you sync data from Device42 to FreshService. + +### Download and Installation +----------------------------- +To utilize the Device42_freshservice_mapping script, Python 3.5+ is required. The following Python Packages are required as well: + +* pycrypto==2.6.1 +* pyparsing==2.1.10 +* pyzmq==16.0.2 +* requests==2.13.0 +* xmljson==0.2.0 + +These can all be installed by running `pip install -r requirements.txt`. + +Once installed, the script itself is run by this command: `python d42_sd_sync.py`. + +### Configuration +----------------------------- +Prior to using the script, it must be configured to connect to your Device42 instance and your FreshService instance. +* Save a copy of mapping.xml.sample as mapping.xml. +* Enter your URL, User, Password, API Key in the FreshService and Device42 sections (lines 2-10). +API Key can be obtained from FreshService profile page + +Below the credential settings, you’ll see a Tasks section. +Multiple Tasks can be setup to synchronize various CIs from Device42 to FreshService. +In the section of each task, there will be a section that queries Device42 to obtain the desired CIs. +Full documentation of the Device42 API and endpoints is available at https://api.device42.com. +Individual tasks within a mapping.xml file can be enabled or disabled at will by changing the `enable="true"` to `enable="false"` in the section. + +Once the Device42 API resource and FreshService Target are entered, the section is where fields from Device42 (the `resource` value) can be mapped to fields in FreshService (the `target` value). +It is very important to adjust the list of default values in accordance between freshservice and device 42 (for example, service_level). + +After configuring the fields to map as needed, the script should be ready to run. + +### Compatibility +----------------------------- +* Script runs on Linux and Windows + +### Info +----------------------------- +* mapping.xml - file from where we get fields relations between D42 and FreshService +* devicd42.py - file with integration device42 instance +* freshservice.py - file with integration freshservice instance +* d42_sd_sync.py - initialization and processing file, where we prepare API calls + +### Support +----------------------------- +We will support any issues you run into with the script and help answer any questions you have. Please reach out to us at support@device42.com + +###Version +----------------------------- +1.0.0.190411 \ No newline at end of file diff --git a/d42_sd_sync.py b/d42_sd_sync.py new file mode 100755 index 0000000..28f840f --- /dev/null +++ b/d42_sd_sync.py @@ -0,0 +1,295 @@ +__author__ = 'Roman Nyschuk' + +import os +import sys +import logging +import json +import argparse +import datetime +from device42 import Device42 +from freshservice import FreshService +import xml.etree.ElementTree as eTree +from xmljson import badgerfish as bf +import time + +logger = logging.getLogger('log') +logger.setLevel(logging.INFO) +ch = logging.StreamHandler(sys.stdout) +ch.setFormatter(logging.Formatter('%(asctime)-15s\t%(levelname)s\t %(message)s')) +logger.addHandler(ch) +CUR_DIR = os.path.dirname(os.path.abspath(__file__)) + +parser = argparse.ArgumentParser(description="freshservice") + +parser.add_argument('-d', '--debug', action='store_true', help='Enable debug output') +parser.add_argument('-q', '--quiet', action='store_true', help='Quiet mode - outputs only errors') +parser.add_argument('-c', '--config', help='Config file', default='mapping.xml') +parser.add_argument('-l', '--logfolder', help='log folder path', default='.') + +freshservice = None + + +class JSONEncoder(json.JSONEncoder): + def default(self, o): + if isinstance(o, datetime): + return o.strftime("%Y %m %d %H:%M:%S") + return json.JSONEncoder.default(self, o) + + +def find_object_by_name(assets, name): + for asset in assets: + if asset["name"] == name: + return asset + + return None + + +def get_asset_type_field(asset_type_fields, map_info): + for section in asset_type_fields: + if section["field_header"] == map_info["@target-header"]: + for field in section["fields"]: + name = map_info["@target"] + if "@target-field" in map_info: + name = map_info["@target-field"] + if field["asset_type_id"] is not None: + name += "_" + str(field["asset_type_id"]) + if field["name"] == name: + return field + + return None + + +def get_map_value_from_device42(source, map_info, b_add=False, asset_type_id=None): + d42_value = source[map_info["@resource"]] + if d42_value is None and "@resource-secondary" in map_info: + d42_value = source[map_info["@resource-secondary"]] + if "@is-array" in map_info and map_info["@is-array"]: + d42_vals = d42_value + d42_value = None + for d42_val in d42_vals: + if map_info["@sub-key"] in d42_val: + d42_value = d42_val[map_info["@sub-key"]] + break + else: + if "value-mapping" in map_info: + d42_val = None + if isinstance(map_info["value-mapping"]["item"], list): + items = map_info["value-mapping"]["item"] + else: + items = [map_info["value-mapping"]["item"]] + for item in items: + if item["@key"] == d42_value: + d42_val = item["@value"] + if d42_val is None and "@default" in map_info["value-mapping"]: + d42_val = map_info["value-mapping"]["@default"] + + d42_value = d42_val + else: + pass + + if "@target-foregin-key" in map_info: + value = freshservice.get_id_by_name(map_info["@target-foregin"], d42_value) + if b_add and value is None and "@not-null" in map_info and map_info[ + "@not-null"] and "@required" in map_info and map_info["@required"]: + name = d42_value + id = freshservice.insert_and_get_id_by_name(map_info["@target-foregin"], name, asset_type_id) + d42_value = id + else: + d42_value = value + + return d42_value + + +def update_objects_from_server(sources, _target, mapping, doql=False): + global freshservice + + logger.info("Getting all existing devices in FS.") + existing_objects = freshservice.request(_target["@path"] + "?include=type_fields", "GET", _target["@model"]) + logger.info("finished getting all existing devices in FS.") + + asset_type = freshservice.get_ci_type_by_name(_target["@asset-type"]) + + asset_type_fields = freshservice.get_asset_type_fields(asset_type["id"]) + + for source in sources: + try: + existing_object = find_object_by_name(existing_objects, source["name"]) + data = dict() + data["type_fields"] = dict() + for map_info in mapping["field"]: + asset_type_field = get_asset_type_field(asset_type_fields, map_info) + if asset_type_field is None: + continue + + value = get_map_value_from_device42(source, map_info) + + if asset_type_field["asset_type_id"] is not None: + data["type_fields"][asset_type_field["name"]] = value + else: + data[map_info["@target"]] = value + + # validation + for map_info in mapping["field"]: + asset_type_field = get_asset_type_field(asset_type_fields, map_info) + if asset_type_field is None: + continue + + if asset_type_field["asset_type_id"] is not None: + value = data["type_fields"][asset_type_field["name"]] + else: + value = data[map_info["@target"]] + + is_valid = True + if value is not None and "@min-length" in map_info and len(value) < map_info["@min-length"]: + is_valid = False + if value is None and "@not-null" in map_info and map_info["@not-null"]: + is_valid = False + if not is_valid and "@required" in map_info and map_info["@required"]: + value = get_map_value_from_device42(source, map_info, True, data["asset_type_id"]) + if value is not None: + is_valid = True + if "@target-type" in map_info and value is not None: + target_type = map_info["@target-type"] + if target_type == "integer": + try: + value = int(value) + except: + is_valid = False + + if not is_valid: + logger.debug("argument '%s' is invalid." % map_info["@target"]) + if asset_type_field["asset_type_id"] is not None: + data["type_fields"].pop(asset_type_field["name"], None) + else: + data.pop(map_info["@target"], None) + if is_valid: + if asset_type_field["asset_type_id"] is not None: + data["type_fields"][asset_type_field["name"]] = value + else: + data[map_info["@target"]] = value + + if existing_object is None: + logger.info("adding device %s" % source["name"]) + new_asset_id = freshservice.insert_asset(data) + logger.info("added new asset %d" % new_asset_id) + else: + logger.info("updating device %s" % source["name"]) + updated_asset_id = freshservice.update_asset(data, existing_object["display_id"]) + logger.info("updated new asset %d" % updated_asset_id) + except Exception as e: + logger.exception("Error (%s) updating device %s" % (type(e), source["name"])) + + +def delete_objects_from_server(sources, _target, mapping): + global freshservice + + logger.info("Getting all existing devices in FS.") + existing_objects = freshservice.request(_target["@path"] + "?include=type_fields", "GET", _target["@model"]) + logger.info("finished getting all existing devices in FS.") + + for existing_object in existing_objects: + exist = False + for source in sources: + if source[mapping["@key"]] == existing_object[mapping["@key"]]: + exist = True + break + + if not exist: + try: + logger.info("deleting device %s" % existing_object["name"]) + freshservice.delete_asset(existing_object["display_id"]) + logger.info("deleted asset %s" % existing_object["name"]) + except Exception as e: + logger.exception("Error (%s) deleting device %s" % (type(e), existing_object["name"])) + + +def parse_config(url): + config = eTree.parse(url) + meta = config.getroot() + config_json = bf.data(meta) + + return config_json + + +def task_execute(task, device42): + if "@description" in task: + logger.info("Execute task - %s" % task["@description"]) + + _resource = task["api"]["resource"] + _target = task["api"]["target"] + + method = _resource['@method'] + if "@doql" in _resource: + doql = _resource['@doql'] + else: + doql = None + + source_url = _resource['@path'] + if "@extra-filter" in _resource: + source_url += _resource["@extra-filter"] + "&" + + _type = None + if "@type" in task: + _type = task["@type"] + + mapping = task['mapping'] + + if doql is not None and doql: + sources = device42.doql(source_url, method, query=doql) + else: + sources = device42.request(source_url, method, _resource["@model"]) + + if "@delete" in _target and _target["@delete"]: + delete_objects_from_server(sources, _target, mapping) + return + + update_objects_from_server(sources, _target, mapping, doql=False) + + +def main(): + global freshservice + + args = parser.parse_args() + if args.debug: + logger.setLevel(logging.DEBUG) + if args.quiet: + logger.setLevel(logging.ERROR) + + try: + log_file = "%s/d42_fs_sync_%d.log" % (args.logfolder, int(time.time())) + logging.basicConfig(filename=log_file) + except Exception as e: + print("Error in config log: %s" % str(e)) + return -1 + + config = parse_config(args.config) + logger.debug("configuration info: %s" % (json.dumps(config))) + + settings = config["meta"]["settings"] + device42 = Device42(settings['device42']['@url'], settings['device42']['@user'], settings['device42']['@pass']) + freshservice = FreshService(settings['freshservice']['@url'], settings['freshservice']['@api_key'], logger) + + if not "task" in config["meta"]["tasks"]: + logger.debug("No task") + return 0 + + if isinstance(config["meta"]["tasks"]["task"], list): + tasks = config["meta"]["tasks"]["task"] + else: + tasks = [config["meta"]["tasks"]["task"]] + + for task in tasks: + if not task["@enable"]: + continue + + task_execute(task, device42) + + print("Completed! View log at %s" % log_file) + return 0 + + +if __name__ == "__main__": + print('Running...') + ret_val = main() + print('Done') + sys.exit(ret_val) diff --git a/device42.py b/device42.py new file mode 100644 index 0000000..4014aa7 --- /dev/null +++ b/device42.py @@ -0,0 +1,120 @@ +# -*- coding: utf-8 -*- + + +import os +import requests + +requests.packages.urllib3.disable_warnings() + + +class Device42BaseException(Exception): + pass + + +class Device42BadArgumentError(Exception): + pass + + +class Device42HTTPError(Device42BaseException): + pass + + +class Device42WrongRequest(Device42HTTPError): + pass + + +class Device42(object): + def __init__(self, endpoint, user, password, **kwargs): + self.base = endpoint + self.user = user + self.pwd = password + self.verify_cert = False + self.debug = kwargs.get('debug', False) + self.logger = kwargs.get('logger', None) + self.base_url = "%s" % self.base + self.headers = {} + + def _send(self, method, path, data=None): + """ General method to send requests """ + url = "%s/%s" % (self.base_url, path) + params = None + if method == 'GET': + params = data + data = None + resp = requests.request(method, url, data=data, params=params, + auth=(self.user, self.pwd), + verify=self.verify_cert, headers=self.headers) + if not resp.ok: + raise Device42HTTPError("HTTP %s (%s) Error %s: %s\n request was %s" % + (method, path, resp.status_code, resp.text, data)) + retval = resp.json() + return retval + + def _get(self, path, data=None): + return self._send("GET", path, data=data) + + def _post(self, path, data): + if not path.endswith('/'): + path += '/' + return self._send("POST", path, data=data) + + def _put(self, path, data): + if not path.endswith('/'): + path += '/' + return self._send("PUT", path, data=data) + + def _delete(self, path): + return self._send("DELETE", path) + + def _log(self, message, level="DEBUG"): + if self.logger: + self.logger.log(level.upper(), message) + + def get_device_by_name(self, name): + path = "api/1.0/devices/name/%s" % name + return self._get(path) + + def get_all_devices(self): + path = "api/1.0/devices/all/" + devices = [] + init_data = self._get(path, {'limit': 1, 'offset': 0}) + total_count = init_data['total_count'] + i = 0 + limit = 1000 + while i < total_count: + devices_data = self._get(path, {'limit': limit, 'offset': i}) + devices = devices + devices_data['Devices'] + i += limit + + return devices + + def doql(self, url, method, query=None): + path = url + if query is None: + query = "SELECT * FROM view_device_v1 order by device_pk" + + data = {"output_type": "json", "query": query} + + result = self._post(path, data) + return result + + def request(self, source_url, method, model): + models = [] + if method == "GET": + result = self._get(source_url) + if model in result: + models = result[model] + limit = 0 + total_count = 0 + if "limit" in result: + limit = result["limit"] + if "total_count" in result: + total_count = result["total_count"] + offset = limit + while offset < total_count: + result = self._get(source_url, data={"offset":offset, "limit":limit}) + if model in result: + models += result[model] + offset += limit + + return models diff --git a/freshservice.py b/freshservice.py new file mode 100755 index 0000000..e0ec786 --- /dev/null +++ b/freshservice.py @@ -0,0 +1,241 @@ +# -*- coding: utf-8 -*- + + +import requests +from datetime import datetime +import time + +requests.packages.urllib3.disable_warnings() + + +class FreshServiceBaseException(Exception): + pass + + +class FreshServiceBadArgumentError(Exception): + pass + + +class FreshServiceHTTPError(FreshServiceBaseException): + pass + + +class FreshServiceWrongRequest(FreshServiceHTTPError): + pass + + +class FreshService(object): + CITypeServerName = "Server" + + def __init__(self, endpoint, api_key, logger, **kwargs): + self.base = endpoint + self.api_key = api_key + self.verify_cert = False + self.debug = kwargs.get('debug', False) + self.logger = logger + self.base_url = "https://%s" % self.base + self.headers = {} + self.last_time_call_api = None + self.period_call_api = 4 + self.api_call_count = 0 + self.asset_types = None + self.server_data = dict() + + def _send(self, method, path, data=None): + """ General method to send requests """ + now = datetime.now() + self.api_call_count += 1 + + is_getting_exist = False + if method == 'GET' and data is not None and "page" in data: + is_getting_exist = True + + if not is_getting_exist and self.last_time_call_api is not None and ( + now - self.last_time_call_api).total_seconds() < self.period_call_api: + time.sleep(self.period_call_api - (now - self.last_time_call_api).total_seconds()) + + url = "%s/%s" % (self.base_url, path) + params = None + if method == 'GET': + params = data + data = None + + while True: + if method == 'GET': + resp = requests.request(method, url, data=data, params=params, + auth=(self.api_key, "X"), + verify=self.verify_cert, headers=self.headers) + else: + resp = requests.request(method, url, json=data, params=params, + auth=(self.api_key, "X"), + verify=self.verify_cert, headers=self.headers) + + self.last_time_call_api = datetime.now() + + if not resp.ok: + if resp.status_code == 429: + self._log("HTTP %s (%s) Error %s: %s\n request was %s" % + (method, path, resp.status_code, resp.text, data)) + self._log("Throttling 1 min...") + time.sleep(60) + continue + + raise FreshServiceHTTPError("HTTP %s (%s) Error %s: %s\n request was %s" % + (method, path, resp.status_code, resp.text, data)) + + if method == "DELETE": + return True + + if resp.status_code == 204: + return {} + + retval = resp.json() + return retval + + def _get(self, path, data=None): + return self._send("GET", path, data=data) + + def _post(self, path, data): + if not path.endswith('/'): + path += '/' + return self._send("POST", path, data=data) + + def _put(self, path, data): + if not path.endswith('/'): + path += '/' + return self._send("PUT", path, data=data) + + def _delete(self, path): + return self._send("DELETE", path) + + def _log(self, message, level="DEBUG"): + if self.logger: + self.logger.log(level.upper(), message) + + def insert_asset(self, data): + path = "api/v2/assets" + result = self._post(path, data) + return result["asset"]["id"] + + def update_asset(self, data, display_id): + path = "api/v2/assets/%d" % display_id + result = self._put(path, data) + return result["asset"]["id"] + + def delete_asset(self, display_id): + path = "api/v2/assets/%d/delete_forever" % display_id + result = self._put(path, {"No": 1}) + return result + + def get_assets_by_asset_type(self, asset_type_id): + path = "api/v2/assets?include=type_fields&query=\"asset_type_id:%d\"" % asset_type_id + assets = self._get(path) + return assets["assets"] + + def get_all_ci_types(self): + if self.asset_types is not None: + return self.asset_types + path = "api/v2/asset_types" + asset_types = self._get(path) + self.asset_types = asset_types["asset_types"] + return self.asset_types + + def get_ci_type_by_name(self, name, all_ci_types=None): + if all_ci_types is None: + all_ci_types = self.get_all_ci_types() + + for ci_type in all_ci_types: + if ci_type["name"] == name: + return ci_type + + return None + + def get_all_server_ci_types(self): + all_ci_types = self.get_all_ci_types() + + server_types = [] + server_type = self.get_ci_type_by_name("Server", all_ci_types) + if server_type is None: + return [] + + server_types.append(server_type) + for ci_type in all_ci_types: + if ci_type["parent_asset_type_id"] == server_type["id"]: + server_types.append(ci_type) + + return server_types + + def get_server_ci_type(self): + return self.get_ci_type_by_name(self.CITypeServerName) + + def get_windows_server_ci_type(self): + return self.get_ci_type_by_name(self.CITypeWindowsServerName) + + def get_unix_server_ci_type(self): + return self.get_ci_type_by_name(self.CITypeUnixServerName) + + def get_asset_type_fields(self, asset_type_id): + path = "api/v2/asset_types/%d/fields" % asset_type_id + return self._get(path)["asset_type_fields"] + + def get_all_server_assets(self): + server_asset_types = self.get_all_server_ci_types() + server_assets = [] + for asset_type in server_asset_types: + assets = self.get_assets_by_asset_type(asset_type["id"]) + server_assets += assets + + return server_assets + + def get_products(self): + path = "/api/v2/products" + products = self._get(path) + return products["products"] + + def get_vendors(self): + path = "/api/v2/vendors" + vendors = self._get(path) + return vendors["vendors"] + + def get_id_by_name(self, model, name): + path = "/api/v2/%s" % model + models = self.request(path, "GET", model) + for model in models: + if "name" in model and model["name"] is not None and name is not None and model[ + "name"].lower() == name.lower(): + return model["id"] + + return None + + def insert_and_get_id_by_name(self, model, name, asset_type_id): + path = "/api/v2/%s" % model + data = {"name": name, "asset_type_id": asset_type_id} + models = self._post(path, data) + for key in models: + if model in self.server_data: + self.server_data[model] += [models[key]] + + return models[key]["id"] + + return None + + def request(self, source_url, method, model): + if method == "GET": + if model in self.server_data: + return self.server_data[model] + + models = [] + page = 1 + while True: + result = self._get(source_url, data={"page": page}) + if model in result: + models += result[model] + if len(result[model]) == 0: + break + else: + break + + page += 1 + self.server_data[model] = models + return models + return [] diff --git a/mapping.xml.sample b/mapping.xml.sample new file mode 100644 index 0000000..e549640 --- /dev/null +++ b/mapping.xml.sample @@ -0,0 +1,89 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..6085abd --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +pycrypto==2.6.1 +pyparsing==2.1.10 +pyzmq==16.0.2 +requests==2.13.0 +xmljson==0.2.0