From 8fbbf88e99b52008e478dca8e0341c3b65946d6f Mon Sep 17 00:00:00 2001 From: Omar Sanchez Date: Fri, 10 Jul 2020 18:04:25 -0400 Subject: [PATCH 01/13] huge revamp on query method. now uses doql endpoint to contruct device inventory which is much faster than the object based query used before --- plugins/inventory/d42.py | 530 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 519 insertions(+), 11 deletions(-) diff --git a/plugins/inventory/d42.py b/plugins/inventory/d42.py index 1f95e1c..18cefa4 100644 --- a/plugins/inventory/d42.py +++ b/plugins/inventory/d42.py @@ -1,6 +1,11 @@ -from __future__ import (absolute_import, division, print_function) -from ansible.plugins.inventory import BaseInventoryPlugin, Constructable, Cacheable, to_safe_group_name +# from __future__ import (absolute_import, division, print_function) + +# from ansible.plugins.inventory import BaseInventoryPlugin, Constructable, Cacheable, to_safe_group_name import requests +import io +import csv +import json +import sys __metaclass__ = type @@ -80,20 +85,12 @@ def parse(self, inventory, loader, path, cache=False): self._read_config_data(path) - base_url = self.get_option('url') - username = self.get_option('username') - password = self.get_option('password') - ssl_check = self.get_option('ssl_check') strict = self.get_option('strict') try: objects = [] - response = requests.get(base_url + "/api/1.0/devices/all", auth=(username, password), verify=ssl_check, - timeout=10) - - print('response code: ' + str(response.status_code)) - json_response = response.json() + json_response = self.get_devices() if 'Devices' in json_response: objects = json_response['Devices'] @@ -115,3 +112,514 @@ def parse(self, inventory, loader, path, cache=False): self._add_host_to_keyed_groups(self.get_option('keyed_groups'), dict(), host_name, strict) except Exception as e: print(e) + + def get_doql_csv(self, query): + base_url = self.get_option('url') + username = self.get_option('username') + password = self.get_option('password') + ssl_check = self.get_option('ssl_check') + + data = {'header': 'yes', 'query': query} + unformatted_d42_inventory = [] + + try: + response = requests.get(base_url + "/services/data/v1.0/query/", params=data, + auth=(username, password), verify=ssl_check) + + # csv response to json object + doql_csv = response.text + reader = csv.DictReader(io.StringIO(doql_csv)) + unformatted_d42_inventory = json.dumps(list(reader)) + unformatted_d42_inventory = json.loads(unformatted_d42_inventory) + except Exception as e: + print(e) + return [] + + return unformatted_d42_inventory + + def get_d42_inventory(self): + # the final inventory that will be returned + inventory = { + 'total_count': None, + 'Devices': [], + } + + # the dictionary below is used to build the json object + d42_inventory = {} + + # get all the items from device42 that will be used to build the json object + d42_devices = self.get_devices() + d42_device_ips = self.get_ip_addresses() + d42_device_mac_addresses = self.get_mac_addresses() + d42_device_hdd_details = self.get_hdd_details() + d42_device_external_links = self.get_external_links() + d42_device_entitlements = self.get_device_entitlements() + d42_device_custom_fields = self.get_custom_fields() + + for device in d42_devices: + + if device['device_pk'] in d42_inventory: + print('device already encountered, ensure that the doql for devices is working correctly') + + device_record = {} + + device_record['datastores'] = device.get('datastores') + device_record['last_updated'] = device.get('last_edited') + device_record['ip_addresses'] = [] + device_record['serial_no'] = device.get('serial_no') + device_record['hw_depth'] = device.get('hw_depth') + device_record['nonauthoritativealiases'] = str(device.get('nonauthoritativealiases')).split(',') + + device_record['service_level'] = device.get('service_level') + device_record['is_it_blade_host'] = 'no' if device.get('blade_chassis') == 'false' else 'yes' + device_record['is_it_switch'] = 'no' if device.get('is_it_switch') == 'false' else 'yes' + device_record['virtual_subtype_id'] = device.get('virtual_subtype_id') + device_record['hw_size'] = device.get('hw_size') + device_record['id'] = device.get('device_pk') + + # custom field + device_record['custom_fields'] = [] + + device_record['aliases'] = str(device.get('aliases')).split(',') + device_record['category'] = device.get('category') + + # hdd details + device_record['hdd_details'] = [] + + device_record['uuid'] = device.get('uuid') + device_record['virtual_subtype'] = device.get('virtual_subtype') + device_record['cpuspeed'] = device.get('cpuspeed') + device_record['hw_model'] = device.get('hw_model') + device_record['osverno'] = device.get('os_version_no') + device_record['type'] = device.get('type') + device_record['hddcount'] = device.get('hard_disk_count') + + # external links + device_record['device_external_links'] = [] + + device_record['tags'] = device.get('tags') + device_record['hw_model_id'] = device.get('hw_model_id') + device_record['in_service'] = device.get('in_service') + device_record['hddsize'] = device.get('disk_size') + device_record['hddsize_type'] = 'GB' + + # mac addresses + device_record['mac_addresses'] = [] + + device_record['hddraid'] = device.get('hw_sw_raid') + device_record['corethread'] = device.get('corethread') + device_record['cpucount'] = device.get('cpucount') + device_record["virtual_host_name"] = device.get('virtual_host_name') + device_record["is_it_virtual_host"] = 'no' if device.get('virtual_host') == 'false' else 'yes' + device_record["manufacturer"] = device.get('manufacturer') + device_record["customer"] = device.get('customer') + device_record["customer_id"] = device.get('customer_id') + device_record["ucs_manager"] = device.get('ucsmanager') + device_record["hddraid_type"] = device.get('raid_type') + device_record["name"] = device.get('name') + device_record["notes"] = device.get('notes') + device_record["ram"] = device.get('ram') + device_record["asset_no"] = device.get('asset_no') + device_record["osarch"] = device.get('os_arch') + device_record["osver"] = device.get('os_version_no') + + # line items + device_record["device_purchase_line_items"] = [] + + device_record["cpucore"] = device.get('cpucore') + device_record["os"] = device.get('os_name') + device_record["device_id"] = device.get('device_pk') + + # store the device in a dict for updates + d42_inventory[device['device_pk']] = device_record + + for custom_field_record in d42_device_custom_fields: + device_id = custom_field_record.get('device_pk') + + if device_id in d42_inventory: + d42_inventory[device_id]['custom_fields'].append( + { + 'notes': custom_field_record.get('custom_field_notes'), + 'key': custom_field_record.get('custom_field_key'), + 'value': custom_field_record.get('custom_field_value') + } + ) + + for hdd_detail_record in d42_device_hdd_details: + device_id = hdd_detail_record.get('device_pk') + + if device_id in d42_inventory: + d42_inventory[device_id]['hdd_details'].append({ + "raid_group": hdd_detail_record.get('raid_group'), + "serial_no": hdd_detail_record.get('hdd_serial_no'), + "raid_type": hdd_detail_record.get('hdd_raid_type'), + "hdd": { + "description": hdd_detail_record.get('hdd_description'), + "partno": hdd_detail_record.get('partno'), + "rpm": { + "id": hdd_detail_record.get('hdd_rom_id'), + "name": hdd_detail_record.get('hdd_rpm') + }, + "notes": hdd_detail_record.get('hdd_notes'), + "bytes": hdd_detail_record.get('hdd_bytes'), + "location": hdd_detail_record.get('hdd_location'), + "hd_id": hdd_detail_record.get('hdd_id'), + "manufacturer_id": hdd_detail_record.get('manufactuerer_id'), + "type": { + "id": hdd_detail_record.get('hdd_type_id'), + "name": hdd_detail_record.get('hdd_type_name') + }, + "size": hdd_detail_record.get('size') + }, + "part_id": hdd_detail_record.get('part_id'), + "hddcount": hdd_detail_record.get('hddcount'), + "description": hdd_detail_record.get('description') + }) + + for device_external_link_record in d42_device_external_links: + device_id = device_external_link_record.get('device_pk') + + if device_id in d42_inventory: + d42_inventory[device_id]['device_external_links'].append( + { + 'notes': device_external_link_record.get('device_url_notes'), + 'link': device_external_link_record.get('device_url') + } + ) + + for device_mac_address_record in d42_device_mac_addresses: + + device_id = device_mac_address_record.get('device_pk') + + if device_id in d42_inventory: + d42_inventory[device_id]['mac_addresses'].append( + { + "mac": device_mac_address_record.get('mac'), + "vlan": device_mac_address_record.get('vlan'), + "port_name": device_mac_address_record.get('port_name'), + "port": device_mac_address_record.get('port') + } + ) + + for device_entitlement_record in d42_device_entitlements: + + device_id = device_entitlement_record.get('device_pk') + + if device_id in d42_inventory: + d42_inventory[device_id]['device_purchase_line_items'].append( + { + "purchase_id": device_entitlement_record.get('purchase_id'), + "line_start_date": device_entitlement_record.get('line_start_date'), + "line_no": device_entitlement_record.get('line_no'), + "line_renew_date": device_entitlement_record.get('line_renew_date'), + "line_notes": device_entitlement_record.get('line_notes'), + "line_quantity": device_entitlement_record.get('line_quantity'), + "line_frequency": device_entitlement_record.get('line_frequency'), + "line_cost": device_entitlement_record.get('line_cost'), + "line_cancel_policy": device_entitlement_record.get('line_cancel_policy'), + "line_item_type": device_entitlement_record.get('line_item_type'), + "purchase_order_no": device_entitlement_record.get('purchase_order_no'), + "line_type": device_entitlement_record.get('line_type'), + "line_service_type": device_entitlement_record.get('line_service_type'), + "line_end_date": device_entitlement_record.get('line_end_date'), + "line_contract_type": device_entitlement_record.get('line_contract_type') + } + ) + + for device_ip_record in d42_device_ips: + device_id = device_ip_record.get('device_pk') + + if device_id in d42_inventory: + d42_inventory[device_id]['ip_addresses'].append( + { + "subnet_id": device_ip_record.get('subnet_id'), + "ip": device_ip_record.get('ip'), + "label": device_ip_record.get('label'), + "type": device_ip_record.get('type'), + "subnet": device_ip_record.get('subnet') + } + ) + else: + print(device_id + ' Not found in d42_inventory') + + # now that the dictionary has been constructed, remove device id keys and keep value + # add additional imformation like total_count + + total_count = 0 + for key, value in d42_inventory.items(): + inventory['Devices'].append(value) + total_count += 1 + + # update with the final device count + inventory['total_count'] = total_count + + return inventory + + + def get_devices(self): + device_query = """ + select + view_device_v1.*, + view_device_v1.customer_fk as customer_id, + customer.name as customer, + t_cost.cost, + view_vendor_v1.name as manufacturer, + view_device_v1.type as hw_model, + view_hardware_v1.hardware_pk as hw_model_id, + vendor.name as manufacturer, + view_hardware_v1.is_it_switch, + view_hardware_v1.is_it_blade_host, + view_hardware_v1.depth as hw_depth, + view_hardware_v1.size as hw_size, + vhost.name as virtual_host_name, + nonauthoritativealias.alias_names as nonauthoritativealiases, + alias.alias_names as aliases, + obj_category.name as category, + case + WHEN view_device_v1.hz = 'MHz' and view_device_v1.cpupower is not null + THEN view_device_v1.cpupower / 1000 + ELSE + view_device_v1.cpupower + END as cpuspeed, + case + WHEN view_device_v1.ram_size_type = 'MB' and view_device_v1.ram is not null + THEN view_device_v1.ram / 1024 + WHEN view_device_v1.ram_size_type = 'TB' and view_device_v1.ram is not null + THEN view_device_v1.ram * 1024 + ELSE + view_device_v1.ram + END as ramsize, + network_ip.ip_address, network_hw.hwaddress, + CEIL(COALESCE(p.total_part_disk_size, + CASE + WHEN view_device_v1.hard_disk_count IS NOT NULL AND + view_device_v1.hard_disk_size IS NOT NULL AND + view_device_v1.hard_disk_size_type IS NOT NULL THEN + view_device_v1.hard_disk_count * + CASE + WHEN view_device_v1.hard_disk_size_type = 'GB' + THEN view_device_v1.hard_disk_size + WHEN view_device_v1.hard_disk_size_type = 'TB' + THEN view_device_v1.hard_disk_size * 1024 + WHEN view_device_v1.hard_disk_size_type = 'PB' + THEN view_device_v1.hard_disk_size * 1024 * 1024 + ELSE + NULL + END + ELSE NULL + END)) AS disk_size + from view_device_v1 + left join view_hardware_v1 + on view_device_v1.hardware_fk = view_hardware_v1.hardware_pk + left join (select view_ipaddress_v1.device_fk, string_agg(host(view_ipaddress_v1.ip_address)::character varying, ', ' order by view_ipaddress_v1.ip_address) as ip_address from view_ipaddress_v1 group by view_ipaddress_v1.device_fk) network_ip + on view_device_v1.device_pk=network_ip.device_fk + left join (select device_fk, string_agg(view_netport_v1.hwaddress :: macaddr::character varying, ', ' order by view_netport_v1.hwaddress) as hwaddress from view_netport_v1 where view_netport_v1.hwaddress is null or (view_netport_v1.hwaddress is not null and LENGTH(view_netport_v1.hwaddress)=12) group by view_netport_v1.device_fk) network_hw + on view_device_v1.device_pk=network_hw.device_fk + left join (select device_fk, sum(cost) as cost from view_purchaselineitem_v1 left join view_purchaselineitems_to_devices_v1 on purchaselineitem_fk = purchaselineitem_pk group by device_fk) t_cost + on t_cost.device_fk=device_pk + left join view_vendor_v1 + on vendor_pk=vendor_fk + left join (select device_fk, string_agg(view_devicenonauthoritativealias_v1.alias_name, ', ' order by view_devicenonauthoritativealias_v1.alias_name) as alias_names from view_devicenonauthoritativealias_v1 group by view_devicenonauthoritativealias_v1.device_fk) nonauthoritativealias + on view_device_v1.device_pk=nonauthoritativealias.device_fk + left join (select device_fk, string_agg(view_devicealias_v1.alias_name, ', ' order by view_devicealias_v1.alias_name) as alias_names from view_devicealias_v1 group by view_devicealias_v1.device_fk) alias + on view_device_v1.device_pk=alias.device_fk + left join (select device_pk, name from view_device_v1) vhost + on view_device_v1.virtual_host_device_fk=vhost.device_pk + left join (select objectcategory_pk, name from view_objectcategory_v1) obj_category + on view_device_v1.objectcategory_fk = obj_category.objectcategory_pk + left join (select vendor_pk, name from view_vendor_v1) vendor + on view_hardware_v1.vendor_fk = vendor.vendor_pk + left join (select customer_pk, name from view_customer_v1) customer + on view_device_v1.customer_fk = customer.customer_pk + LEFT OUTER JOIN ( + SELECT p.device_fk, + SUM(p.pcount * + CASE + WHEN pm.hdsize_unit = 'GB' + THEN pm.hdsize + WHEN pm.hdsize_unit = 'TB' + THEN pm.hdsize * 1024 + WHEN pm.hdsize_unit = 'PB' + THEN pm.hdsize * 1024 * 1024 + WHEN pm.hdsize_unit = 'EB' + THEN pm.hdsize * 1024 * 1024 * 1024 + WHEN pm.hdsize_unit = 'ZB' + THEN pm.hdsize * 1024 * 1024 * 1024 * 1024 + WHEN pm.hdsize_unit = 'YB' + THEN pm.hdsize * 1024 * 1024 * 1024 * 1024 * 1024 + ELSE + NULL + END) AS total_part_disk_size + FROM view_part_v1 p + INNER JOIN view_partmodel_v1 pm ON p.partmodel_fk = pm.partmodel_pk + WHERE pm.type_id = 3 AND pm.hdsize IS NOT NULL AND p.pcount > 0 AND p.device_fk IS NOT NULL + GROUP BY p.device_fk + ) AS p ON view_device_v1.device_pk = p.device_fk + """ + + return self.get_doql_csv(device_query) + + + def get_custom_fields(self): + + custom_field_query = """ + select + view_device_v1.device_pk, + view_device_v1.name, + custom_field.key as custom_field_key, + custom_field.value as custom_field_value, + custom_field.notes as custom_field_notes + from view_device_v1 + inner join (select device_fk, type, notes, key, + (CASE + WHEN view_device_custom_fields_v1.type = 'Related Field' + THEN (CASE + WHEN view_device_custom_fields_v1.related_model_name = 'endusers' + THEN (SELECT name FROM view_enduser_v1 WHERE view_enduser_v1.enduser_pk = view_device_custom_fields_v1.value :: int) + ELSE 'not available' + END) + ELSE view_device_custom_fields_v1.value + END) as value + from view_device_custom_fields_v1) custom_field + on view_device_v1.device_pk = custom_field.device_fk + """ + + return self.get_doql_csv(custom_field_query) + + + def get_external_links(self): + external_links_query = """ + select + view_device_v1.device_pk, + view_device_v1.name as device_name, + external_links.device_url, + external_links.device_url_notes + from view_device_v1 + inner join (select device_fk, device_url, notes as device_url_notes from view_deviceurl_v1) external_links + on view_device_v1.device_pk=external_links.device_fk + """ + return self.get_doql_csv(external_links_query) + + + def get_hdd_details(self): + hdd_details_query = """ + select + view_device_v1.device_pk, + view_device_v1.name as device_name, + hdd_details.raid_group, + hdd_details.hdd_serial_no, + hdd_details.hdd_raid_type, + hdd.hdd_description, + hdd.partno, + hdd.rpm as hdd_rpm, + hdd.rpm_id as hdd_rom_id, + hdd.notes as hdd_notes, + hdd.hdd_bytes, + hdd.location as hdd_location, + hdd.hdd_id, + hdd.manufactuerer_id, + hdd.hdd_type_id, + hdd.hdd_type_name, + hdd.size, + hdd_details.part_id, + hdd_details.hddcount, + hdd_details.description + from view_device_v1 + inner join (select device_fk, raid_group, serial_no as hdd_serial_no, raid_type_name as hdd_raid_type, part_pk as part_id, partmodel_fk as partmodel_id, pcount as hddcount, description from view_part_v1) hdd_details + on view_device_v1.device_pk = hdd_details.device_fk + inner join (select description as hdd_description, partno, hdrpm_name as rpm, hdrpm_id as rpm_id, notes, hdsize_unit as hdd_bytes, location, partmodel_pk as hdd_id, vendor_fk as manufactuerer_id, hddtype_id as hdd_type_id, hddtype_name as hdd_type_name, hdsize as size from view_partmodel_v1) hdd + on hdd_details.partmodel_id = hdd.hdd_id + """ + + return self.get_doql_csv(hdd_details_query) + + + def get_ip_addresses(self): + ip_address_query = """ + select + view_device_v1.device_pk, + view_device_v1.name, + ip_address.subnet_fk as subnet_id, + ip_address.ip_address as ip, + ip_address.label, + ip_address.type, + subnet.name as subnet + from view_device_v1 + left join (select device_fk, subnet_fk, ip_address, label, type from view_ipaddress_v1) ip_address + on ip_address.device_fk = view_device_v1.device_pk + inner join (select subnet_pk, name from view_subnet_v1) subnet + on ip_address.subnet_fk = subnet.subnet_pk + """ + return self.get_doql_csv(ip_address_query) + + def get_mac_addresses(self): + mac_addresses_query = """ + select + view_device_v1.device_pk, + view_device_v1.name as device_name, + mac_address.hw_address as mac, + mac_address.port_name, + mac_address.port, + vlan.name as vlan + from view_device_v1 + inner join (select device_fk, (hwaddress :: macaddr::character varying) as hw_address, name as port_name, port, primary_vlan_fk from view_netport_v1) mac_address + ON view_device_v1.device_pk = mac_address.device_fk + full join (select vlan_pk, name from view_vlan_v1) vlan + ON vlan.vlan_pk = mac_address.primary_vlan_fk + """ + + return self.get_doql_csv(mac_addresses_query) + + + def get_device_entitlements(self): + # todo change parse method so it doesnt fail on non valid mac addresses, did this for FS mapping query + device_entitlements_query = """ + Select + view_device_v1.device_pk, + view_device_v1.name as device_name, + line_items.purchase_fk as purhase_id, + line_items.start_date as line_start_date, + line_items.line_no, + line_items.renew_date as line_renew_date, + line_items.notes as line_notes, + line_items.quantity as line_quantity, + line_items.frequency as line_frequency, + line_items.cost as line_cost, + line_items.cancel_policy as line_cancel_policy, + line_items.item_type as line_item_type, + purchase.order_no as purchase_order_no, + line_items.line_type as line_type, + line_items.service_type_name as line_service_type, + line_items.end_date as line_end_date, + line_items.contract_type_name as line_contract_type + from view_device_v1 + inner join (select device_fk, purchaselineitem_fk from view_purchaselineitems_to_devices_v1) purchase_lineitem + on view_device_v1.device_pk=purchase_lineitem.device_fk + inner join (select purchaselineitem_pk, + purchase_fk, + start_date, + line_no, + renew_date, + notes, + quantity, + frequency, + cost, + cancel_policy, + item_type, + line_type, + service_type_name, + end_date, + contract_type_name + from view_purchaselineitem_v1) line_items + on line_items.purchaselineitem_pk = purchase_lineitem.purchaselineitem_fk + inner join (select purchase_pk, order_no from view_purchase_v1) purchase + on purchase.purchase_pk = line_items.purchase_fk + """ + + return self.get_doql_csv(device_entitlements_query) + + +if __name__ == "__main__": + + + From e105d06d549ffac6be6832c0b7580671d944638d Mon Sep 17 00:00:00 2001 From: Omar Sanchez Date: Fri, 10 Jul 2020 18:05:55 -0400 Subject: [PATCH 02/13] removed os import as it was unused and removed main method used for testing --- plugins/inventory/d42.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/plugins/inventory/d42.py b/plugins/inventory/d42.py index 18cefa4..b0310a1 100644 --- a/plugins/inventory/d42.py +++ b/plugins/inventory/d42.py @@ -5,7 +5,6 @@ import io import csv import json -import sys __metaclass__ = type @@ -618,8 +617,3 @@ def get_device_entitlements(self): return self.get_doql_csv(device_entitlements_query) - -if __name__ == "__main__": - - - From 3be79ccf67dea2fb80eb2ee74c4f6155dd1364b4 Mon Sep 17 00:00:00 2001 From: Omar Sanchez Date: Fri, 10 Jul 2020 18:08:06 -0400 Subject: [PATCH 03/13] wrong function call --- plugins/inventory/d42.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/inventory/d42.py b/plugins/inventory/d42.py index b0310a1..7ad238c 100644 --- a/plugins/inventory/d42.py +++ b/plugins/inventory/d42.py @@ -89,7 +89,7 @@ def parse(self, inventory, loader, path, cache=False): try: objects = [] - json_response = self.get_devices() + json_response = self.get_d42_inventory() if 'Devices' in json_response: objects = json_response['Devices'] From 7820282dd89ec2dfdd0ab971221feabcb3a7cdc6 Mon Sep 17 00:00:00 2001 From: Omar Sanchez Date: Mon, 13 Jul 2020 10:19:46 -0400 Subject: [PATCH 04/13] upped the minor version --- galaxy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galaxy.yml b/galaxy.yml index 03d93b5..e26d537 100644 --- a/galaxy.yml +++ b/galaxy.yml @@ -1,6 +1,6 @@ namespace: "device42" name: "d42" -version: "1.0.2" +version: "1.1.2" readme: "README.md" authors: - "Will Tome (@willtome)" From 8839dfcf3239c1b668b1b721a96a258965604bad Mon Sep 17 00:00:00 2001 From: Omar Sanchez Date: Mon, 13 Jul 2020 10:50:35 -0400 Subject: [PATCH 05/13] get doql csv json alternate method --- plugins/inventory/d42.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/plugins/inventory/d42.py b/plugins/inventory/d42.py index 7ad238c..e9e5519 100644 --- a/plugins/inventory/d42.py +++ b/plugins/inventory/d42.py @@ -1,6 +1,5 @@ -# from __future__ import (absolute_import, division, print_function) - -# from ansible.plugins.inventory import BaseInventoryPlugin, Constructable, Cacheable, to_safe_group_name +from __future__ import (absolute_import, division, print_function) +from ansible.plugins.inventory import BaseInventoryPlugin, Constructable, Cacheable, to_safe_group_name import requests import io import csv @@ -122,14 +121,15 @@ def get_doql_csv(self, query): unformatted_d42_inventory = [] try: - response = requests.get(base_url + "/services/data/v1.0/query/", params=data, + response = requests.post(base_url + "/services/data/v1.0/query/", params=data, auth=(username, password), verify=ssl_check) # csv response to json object - doql_csv = response.text - reader = csv.DictReader(io.StringIO(doql_csv)) - unformatted_d42_inventory = json.dumps(list(reader)) - unformatted_d42_inventory = json.loads(unformatted_d42_inventory) + # doql_csv = response.text + unformatted_d42_inventory = response.json() + # reader = csv.DictReader(io.StringIO(doql_csv)) + #unformatted_d42_inventory = json.dumps(list(reader)) + #unformatted_d42_inventory = json.loads(unformatted_d42_inventory) except Exception as e: print(e) return [] From 83117cc1b4f71b853c21206ed3b0cc8b1d9181dc Mon Sep 17 00:00:00 2001 From: Omar Sanchez Date: Mon, 13 Jul 2020 11:56:59 -0400 Subject: [PATCH 06/13] simplified doql csv call, removed uneeded imports --- plugins/inventory/d42.py | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/plugins/inventory/d42.py b/plugins/inventory/d42.py index 7ad238c..eb53372 100644 --- a/plugins/inventory/d42.py +++ b/plugins/inventory/d42.py @@ -1,10 +1,6 @@ -# from __future__ import (absolute_import, division, print_function) - -# from ansible.plugins.inventory import BaseInventoryPlugin, Constructable, Cacheable, to_safe_group_name +from __future__ import (absolute_import, division, print_function) +from ansible.plugins.inventory import BaseInventoryPlugin, Constructable, Cacheable, to_safe_group_name import requests -import io -import csv -import json __metaclass__ = type @@ -50,6 +46,12 @@ default: true env: - name: D42_SSL_CHECK + debug: + description: Debug option. + type: boolean + default: false + env: + - name: D42_DEBUG ''' EXAMPLES = r''' @@ -58,6 +60,7 @@ username: admin password: password ssl_check: False +debug: False keyed_groups: - key: d42_service_level prefix: '' @@ -118,18 +121,16 @@ def get_doql_csv(self, query): password = self.get_option('password') ssl_check = self.get_option('ssl_check') - data = {'header': 'yes', 'query': query} + data = {'output_type': 'json', 'header': 'yes', 'query': query} unformatted_d42_inventory = [] try: - response = requests.get(base_url + "/services/data/v1.0/query/", params=data, + response = requests.post(base_url + "/services/data/v1.0/query/", params=data, auth=(username, password), verify=ssl_check) # csv response to json object - doql_csv = response.text - reader = csv.DictReader(io.StringIO(doql_csv)) - unformatted_d42_inventory = json.dumps(list(reader)) - unformatted_d42_inventory = json.loads(unformatted_d42_inventory) + unformatted_d42_inventory = response.json() + except Exception as e: print(e) return [] @@ -137,6 +138,7 @@ def get_doql_csv(self, query): return unformatted_d42_inventory def get_d42_inventory(self): + # the final inventory that will be returned inventory = { 'total_count': None, @@ -356,6 +358,7 @@ def get_d42_inventory(self): def get_devices(self): + device_query = """ select view_device_v1.*, From 38f0def2192c5d16ff2a0b752228c01dbcdd7e3a Mon Sep 17 00:00:00 2001 From: Omar Sanchez Date: Mon, 13 Jul 2020 12:12:42 -0400 Subject: [PATCH 07/13] added debug option and statements --- plugins/inventory/d42.py | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/plugins/inventory/d42.py b/plugins/inventory/d42.py index eb53372..04ca44c 100644 --- a/plugins/inventory/d42.py +++ b/plugins/inventory/d42.py @@ -120,25 +120,34 @@ def get_doql_csv(self, query): username = self.get_option('username') password = self.get_option('password') ssl_check = self.get_option('ssl_check') + debug = self.get_option('debug') data = {'output_type': 'json', 'header': 'yes', 'query': query} unformatted_d42_inventory = [] try: + # while there should be no timeout, ansible seems to get stuck sending requests without timeouts response = requests.post(base_url + "/services/data/v1.0/query/", params=data, - auth=(username, password), verify=ssl_check) + auth=(username, password), verify=ssl_check, timeout=30) + + if debug: + status_code = response.status_code + print('Response Status: ' + status_code) # csv response to json object unformatted_d42_inventory = response.json() except Exception as e: - print(e) + if debug: + print(e) return [] return unformatted_d42_inventory def get_d42_inventory(self): + debug = self.get_option('debug') + # the final inventory that will be returned inventory = { 'total_count': None, @@ -149,19 +158,30 @@ def get_d42_inventory(self): d42_inventory = {} # get all the items from device42 that will be used to build the json object + if debug: + print('Getting Devices') d42_devices = self.get_devices() + if debug: + print('Getting IPs') d42_device_ips = self.get_ip_addresses() + if debug: + print('Getting MAC Addresses') d42_device_mac_addresses = self.get_mac_addresses() + if debug: + print('Getting HDD Details') d42_device_hdd_details = self.get_hdd_details() + if debug: + print('Getting Device External Links') d42_device_external_links = self.get_external_links() + if debug: + print('Getting Device Entitlements') d42_device_entitlements = self.get_device_entitlements() + if debug: + print('Getting Custom Fields') d42_device_custom_fields = self.get_custom_fields() for device in d42_devices: - if device['device_pk'] in d42_inventory: - print('device already encountered, ensure that the doql for devices is working correctly') - device_record = {} device_record['datastores'] = device.get('datastores') From a5433a7c5eac9069c11ddf8b09d9b9b24ae38751 Mon Sep 17 00:00:00 2001 From: Omar Sanchez Date: Mon, 13 Jul 2020 15:13:19 -0400 Subject: [PATCH 08/13] changed method back to GET from POST --- plugins/inventory/d42.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/inventory/d42.py b/plugins/inventory/d42.py index 04ca44c..d7cb3b7 100644 --- a/plugins/inventory/d42.py +++ b/plugins/inventory/d42.py @@ -127,12 +127,12 @@ def get_doql_csv(self, query): try: # while there should be no timeout, ansible seems to get stuck sending requests without timeouts - response = requests.post(base_url + "/services/data/v1.0/query/", params=data, + response = requests.get(base_url + "/services/data/v1.0/query/", params=data, auth=(username, password), verify=ssl_check, timeout=30) if debug: status_code = response.status_code - print('Response Status: ' + status_code) + print('Response Status: ' + str(status_code)) # csv response to json object unformatted_d42_inventory = response.json() From c83ad3247d071f436d991f798450158a9ee82f21 Mon Sep 17 00:00:00 2001 From: Omar Sanchez Date: Mon, 13 Jul 2020 15:24:10 -0400 Subject: [PATCH 09/13] changed back to post method, data sent through body instead of params --- plugins/inventory/d42.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/inventory/d42.py b/plugins/inventory/d42.py index d7cb3b7..c69e71f 100644 --- a/plugins/inventory/d42.py +++ b/plugins/inventory/d42.py @@ -127,7 +127,7 @@ def get_doql_csv(self, query): try: # while there should be no timeout, ansible seems to get stuck sending requests without timeouts - response = requests.get(base_url + "/services/data/v1.0/query/", params=data, + response = requests.post(base_url + "/services/data/v1.0/query/", data=data, auth=(username, password), verify=ssl_check, timeout=30) if debug: From 0dd63476019f17a08796643d8c3bb353662b61cb Mon Sep 17 00:00:00 2001 From: Omar Sanchez Date: Mon, 13 Jul 2020 16:00:42 -0400 Subject: [PATCH 10/13] updated mac doql to account for devices with invalid mac addresses --- plugins/inventory/d42.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/plugins/inventory/d42.py b/plugins/inventory/d42.py index c69e71f..cb3e0d5 100644 --- a/plugins/inventory/d42.py +++ b/plugins/inventory/d42.py @@ -584,7 +584,15 @@ def get_mac_addresses(self): mac_address.port, vlan.name as vlan from view_device_v1 - inner join (select device_fk, (hwaddress :: macaddr::character varying) as hw_address, name as port_name, port, primary_vlan_fk from view_netport_v1) mac_address + inner join (select device_fk, ( + CASE + WHEN length(TRIM(hwaddress)) = 12 + THEN CONCAT_WS(':',left(TRIM(view_netport_v1.hwaddress),2), substring(TRIM(view_netport_v1.hwaddress),3,2), substring(TRIM(view_netport_v1.hwaddress),5,2), substring(TRIM(view_netport_v1.hwaddress),7,2), substring(TRIM(view_netport_v1.hwaddress),9,2), right(TRIM(view_netport_v1.hwaddress),2)) + ELSE + hwaddress + END + ) + as hw_address, name as port_name, port, primary_vlan_fk from view_netport_v1) mac_address ON view_device_v1.device_pk = mac_address.device_fk full join (select vlan_pk, name from view_vlan_v1) vlan ON vlan.vlan_pk = mac_address.primary_vlan_fk From 344be242179c2bc3988366a9fddee5f730dc905d Mon Sep 17 00:00:00 2001 From: Omar Sanchez Date: Mon, 13 Jul 2020 17:21:49 -0400 Subject: [PATCH 11/13] updated read me to include debug setting --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index f09e083..b47a2a5 100644 --- a/README.md +++ b/README.md @@ -95,6 +95,7 @@ url: https://10.10.10.10 username: admin password: password ssl_check: False +debug: False keyed_groups: - key: d42_service_level prefix: '' @@ -130,6 +131,7 @@ D42_USER = 'device42 user' D42_PWD = 'device42 password' D42_URL = 'https:// device42 server IP address' D42_SKIP_SSL_CHECK = False +D42_DEBUG = False ``` ### How to run @@ -165,6 +167,7 @@ D42_USER = admin D42_PWD = adm!nd42 D42_URL = https://10.10.10.10 D42_SKIP_SSL_CHECK = True +D42_DEBUG = FALSE # ====== Ansible settings ========= [DOQL] From 37385079a63926ae610fd1f4fda22d14654aa5cf Mon Sep 17 00:00:00 2001 From: Omar Sanchez Date: Mon, 13 Jul 2020 17:36:18 -0400 Subject: [PATCH 12/13] fixed version number --- galaxy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galaxy.yml b/galaxy.yml index e26d537..d03dc5d 100644 --- a/galaxy.yml +++ b/galaxy.yml @@ -1,6 +1,6 @@ namespace: "device42" name: "d42" -version: "1.1.2" +version: "1.1.0" readme: "README.md" authors: - "Will Tome (@willtome)" From 6282131a5c72e503813f50af3e5c67164272602a Mon Sep 17 00:00:00 2001 From: Omar Sanchez Date: Tue, 14 Jul 2020 16:01:29 -0400 Subject: [PATCH 13/13] code review --- plugins/inventory/d42.py | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/plugins/inventory/d42.py b/plugins/inventory/d42.py index cb3e0d5..6f900e7 100644 --- a/plugins/inventory/d42.py +++ b/plugins/inventory/d42.py @@ -115,14 +115,14 @@ def parse(self, inventory, loader, path, cache=False): except Exception as e: print(e) - def get_doql_csv(self, query): + def get_doql_json(self, query): base_url = self.get_option('url') username = self.get_option('username') password = self.get_option('password') ssl_check = self.get_option('ssl_check') debug = self.get_option('debug') - data = {'output_type': 'json', 'header': 'yes', 'query': query} + data = {'output_type': 'json', 'query': query} unformatted_d42_inventory = [] try: @@ -134,7 +134,7 @@ def get_doql_csv(self, query): status_code = response.status_code print('Response Status: ' + str(status_code)) - # csv response to json object + # json response to json object unformatted_d42_inventory = response.json() except Exception as e: @@ -189,7 +189,7 @@ def get_d42_inventory(self): device_record['ip_addresses'] = [] device_record['serial_no'] = device.get('serial_no') device_record['hw_depth'] = device.get('hw_depth') - device_record['nonauthoritativealiases'] = str(device.get('nonauthoritativealiases')).split(',') + device_record['nonauthoritativealiases'] = str(device.get('nonauthoritativealiases')).split(',') if device.get('nonauthoritativealiases') is not None else [] device_record['service_level'] = device.get('service_level') device_record['is_it_blade_host'] = 'no' if device.get('blade_chassis') == 'false' else 'yes' @@ -201,7 +201,7 @@ def get_d42_inventory(self): # custom field device_record['custom_fields'] = [] - device_record['aliases'] = str(device.get('aliases')).split(',') + device_record['aliases'] = str(device.get('aliases')).split(',') if device.get('aliases') is not None else [] device_record['category'] = device.get('category') # hdd details @@ -278,7 +278,7 @@ def get_d42_inventory(self): "description": hdd_detail_record.get('hdd_description'), "partno": hdd_detail_record.get('partno'), "rpm": { - "id": hdd_detail_record.get('hdd_rom_id'), + "id": hdd_detail_record.get('hdd_rpm_id'), "name": hdd_detail_record.get('hdd_rpm') }, "notes": hdd_detail_record.get('hdd_notes'), @@ -360,8 +360,6 @@ def get_d42_inventory(self): "subnet": device_ip_record.get('subnet') } ) - else: - print(device_id + ' Not found in d42_inventory') # now that the dictionary has been constructed, remove device id keys and keep value # add additional imformation like total_count @@ -479,7 +477,7 @@ def get_devices(self): ) AS p ON view_device_v1.device_pk = p.device_fk """ - return self.get_doql_csv(device_query) + return self.get_doql_json(device_query) def get_custom_fields(self): @@ -506,7 +504,7 @@ def get_custom_fields(self): on view_device_v1.device_pk = custom_field.device_fk """ - return self.get_doql_csv(custom_field_query) + return self.get_doql_json(custom_field_query) def get_external_links(self): @@ -520,7 +518,7 @@ def get_external_links(self): inner join (select device_fk, device_url, notes as device_url_notes from view_deviceurl_v1) external_links on view_device_v1.device_pk=external_links.device_fk """ - return self.get_doql_csv(external_links_query) + return self.get_doql_json(external_links_query) def get_hdd_details(self): @@ -534,7 +532,7 @@ def get_hdd_details(self): hdd.hdd_description, hdd.partno, hdd.rpm as hdd_rpm, - hdd.rpm_id as hdd_rom_id, + hdd.rpm_id as hdd_rpm_id, hdd.notes as hdd_notes, hdd.hdd_bytes, hdd.location as hdd_location, @@ -553,7 +551,7 @@ def get_hdd_details(self): on hdd_details.partmodel_id = hdd.hdd_id """ - return self.get_doql_csv(hdd_details_query) + return self.get_doql_json(hdd_details_query) def get_ip_addresses(self): @@ -572,7 +570,7 @@ def get_ip_addresses(self): inner join (select subnet_pk, name from view_subnet_v1) subnet on ip_address.subnet_fk = subnet.subnet_pk """ - return self.get_doql_csv(ip_address_query) + return self.get_doql_json(ip_address_query) def get_mac_addresses(self): mac_addresses_query = """ @@ -598,7 +596,7 @@ def get_mac_addresses(self): ON vlan.vlan_pk = mac_address.primary_vlan_fk """ - return self.get_doql_csv(mac_addresses_query) + return self.get_doql_json(mac_addresses_query) def get_device_entitlements(self): @@ -646,5 +644,5 @@ def get_device_entitlements(self): on purchase.purchase_pk = line_items.purchase_fk """ - return self.get_doql_csv(device_entitlements_query) + return self.get_doql_json(device_entitlements_query)