Merge 2a817e0 into 8728ab9

.github/workflows/coverage.yml

@@ -15,10 +15,10 @@ jobs:
 
     steps:
     - uses: actions/checkout@v2
-    - name: Set up Python 3.7
+    - name: Set up Python 3.8
       uses: actions/setup-python@v1
       with:
-        python-version: 3.7
+        python-version: 3.8
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip

.github/workflows/pythonapp.yml

@@ -16,10 +16,10 @@ jobs:
 
     steps:
     - uses: actions/checkout@v2
-    - name: Set up Python 3.7
+    - name: Set up Python 3.8
       uses: actions/setup-python@v1
       with:
-        python-version: 3.7
+        python-version: 3.8
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip

BaseCollector.py

@@ -24,7 +24,6 @@ def __init__(self):
         self.target = os.environ.get('TARGET')
         self.vrops = Vrops()
         self.name = self.__class__.__name__
-        self.wait_for_inventory_data()
         self.label_names = []
         self.project_ids = []
 
@@ -90,11 +89,13 @@ def get_iteration(self):
         return self.iteration
 
     def get_collection_times(self):
+        self.wait_for_inventory_data()
         request = requests.get(url="http://" + os.environ['INVENTORY'] + "/collection_times")
         self.collection_times = request.json()
         return self.collection_times
 
     def get_inventory_api_responses(self):
+        self.wait_for_inventory_data()
         request = requests.get(url="http://" + os.environ['INVENTORY'] + "/api_response_codes")
         self.api_responses = request.json()
         return self.api_responses

InventoryBuilder.py

@@ -210,7 +210,7 @@ def create_resource_objects(self, target: str, token: str) -> Vcenter:
         dss = [ds for ds in cluster_and_ds if ds.get('resourcekind') == "Datastore"]
 
         hosts = Vrops.get_hosts(vrops, target, token, [cl.get('uuid') for cl in cluster])
-        vms = Vrops.get_vms(vrops, target, token, [hs.get('uuid') for hs in hosts])
+        vms = Vrops.get_vms(vrops, target, token, [hs.get('uuid') for hs in hosts], vcenter_uuid)
 
         vcenter = Vcenter(target, token, vcenter_uuid, vcenter_name)
         for dc in datacenter:

tools/Vrops.py

@@ -141,8 +141,25 @@ def get_cluster_and_datastores(self, target, token, parent_uuids):
     def get_hosts(self, target, token, parent_uuids):
         return self.get_resources(target, token, parent_uuids, resourcekinds=["HostSystem"])
 
-    def get_vms(self, target, token, parent_uuids):
-        return self.get_resources(target, token, parent_uuids, resourcekinds=["VirtualMachine"], data_receiving=True)
+    def get_vms(self, target, token, parent_uuids, vcenter_uuid):
+        amount_vms, api_responding, _ = self.get_latest_stats_multiple(target, token, [vcenter_uuid],
+                                                                                   ['summary|total_number_vms'],
+                                                                                   'Inventory')
+        number_of_vms = amount_vms[0].get('stat-list', {}).get('stat', [])[0].get('data', [0])[0] if \
+            api_responding == 200 else 0
+
+        # vrops cannot handle more than 10000 resource requests
+        if number_of_vms > 10000:
+            uuids_chunked = list(chunk_list(parent_uuids, int(round(len(parent_uuids) / 2, 0))))
+            logger.debug(f'Chunking VM requests into {len(uuids_chunked)} chunks')
+            vms = list()
+            for uuid_list in uuids_chunked:
+                vms.extend(self.get_resources(target, token, uuid_list, resourcekinds=["VirtualMachine"],
+                                              data_receiving=True))
+            logger.debug(f'Number of VMs collected: {len(vms)}')
+            return vms
+        return self.get_resources(target, token, parent_uuids, resourcekinds=["VirtualMachine"],
+                                  data_receiving=True)
 
     def get_latest_values_multiple(self, target: str, token: str, uuids: list, keys: list, collector: str,
                                    kind: str = None) -> (list, int, float):