-
Notifications
You must be signed in to change notification settings - Fork 3.4k
/
inventory.py
1723 lines (1490 loc) · 67.8 KB
/
inventory.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
# Copyright (c) 2015 Ansible, Inc.
# All Rights Reserved.
# Python
import datetime
import time
import logging
import re
import copy
import os.path
from urllib.parse import urljoin
import yaml
import tempfile
import stat
# Django
from django.conf import settings
from django.db import models, connection
from django.utils.translation import gettext_lazy as _
from django.db import transaction
from django.core.exceptions import ValidationError
from django.urls import resolve
from django.utils.timezone import now
from django.db.models import Q
# REST Framework
from rest_framework.exceptions import ParseError
from ansible_base.lib.utils.models import prevent_search
# AWX
from awx.api.versioning import reverse
from awx.main.constants import CLOUD_PROVIDERS
from awx.main.consumers import emit_channel_notification
from awx.main.fields import (
ImplicitRoleField,
SmartFilterField,
OrderedManyToManyField,
)
from awx.main.managers import HostManager, HostMetricActiveManager
from awx.main.models.base import BaseModel, CommonModelNameNotUnique, VarsDictProperty, CLOUD_INVENTORY_SOURCES, accepts_json
from awx.main.models.events import InventoryUpdateEvent, UnpartitionedInventoryUpdateEvent
from awx.main.models.unified_jobs import UnifiedJob, UnifiedJobTemplate
from awx.main.models.mixins import (
ResourceMixin,
TaskManagerInventoryUpdateMixin,
RelatedJobsMixin,
CustomVirtualEnvMixin,
)
from awx.main.models.notifications import (
NotificationTemplate,
JobNotificationMixin,
)
from awx.main.models.credential.injectors import _openstack_data
from awx.main.utils import _inventory_updates
from awx.main.utils.safe_yaml import sanitize_jinja
from awx.main.utils.execution_environments import to_container_path, get_control_plane_execution_environment
from awx.main.utils.licensing import server_product_name
__all__ = ['Inventory', 'Host', 'Group', 'InventorySource', 'InventoryUpdate', 'SmartInventoryMembership', 'HostMetric', 'HostMetricSummaryMonthly']
logger = logging.getLogger('awx.main.models.inventory')
class InventoryConstructedInventoryMembership(models.Model):
constructed_inventory = models.ForeignKey('Inventory', on_delete=models.CASCADE, related_name='constructed_inventory_memberships')
input_inventory = models.ForeignKey('Inventory', on_delete=models.CASCADE)
position = models.PositiveIntegerField(
null=True,
default=None,
db_index=True,
)
class Inventory(CommonModelNameNotUnique, ResourceMixin, RelatedJobsMixin):
"""
an inventory source contains lists and hosts.
"""
FIELDS_TO_PRESERVE_AT_COPY = ['hosts', 'groups', 'instance_groups', 'prevent_instance_group_fallback']
KIND_CHOICES = [
('', _('Hosts have a direct link to this inventory.')),
('smart', _('Hosts for inventory generated using the host_filter property.')),
('constructed', _('Parse list of source inventories with the constructed inventory plugin.')),
]
class Meta:
app_label = 'main'
verbose_name_plural = _('inventories')
unique_together = [('name', 'organization')]
ordering = ('name',)
permissions = [
('use_inventory', 'Can use inventory in a job template'),
('adhoc_inventory', 'Can run ad hoc commands'),
('update_inventory', 'Can update inventory sources in inventory'),
]
organization = models.ForeignKey(
'Organization',
related_name='inventories',
help_text=_('Organization containing this inventory.'),
on_delete=models.SET_NULL,
null=True,
)
variables = accepts_json(
models.TextField(
blank=True,
default='',
help_text=_('Inventory variables in JSON or YAML format.'),
)
)
has_active_failures = models.BooleanField(
default=False,
editable=False,
help_text=_('This field is deprecated and will be removed in a future release. Flag indicating whether any hosts in this inventory have failed.'),
)
total_hosts = models.PositiveIntegerField(
default=0,
editable=False,
help_text=_('This field is deprecated and will be removed in a future release. Total number of hosts in this inventory.'),
)
hosts_with_active_failures = models.PositiveIntegerField(
default=0,
editable=False,
help_text=_('This field is deprecated and will be removed in a future release. Number of hosts in this inventory with active failures.'),
)
total_groups = models.PositiveIntegerField(
default=0,
editable=False,
help_text=_('This field is deprecated and will be removed in a future release. Total number of groups in this inventory.'),
)
has_inventory_sources = models.BooleanField(
default=False,
editable=False,
help_text=_(
'This field is deprecated and will be removed in a future release. Flag indicating whether this inventory has any external inventory sources.'
),
)
total_inventory_sources = models.PositiveIntegerField(
default=0,
editable=False,
help_text=_('Total number of external inventory sources configured within this inventory.'),
)
inventory_sources_with_failures = models.PositiveIntegerField(
default=0,
editable=False,
help_text=_('Number of external inventory sources in this inventory with failures.'),
)
kind = models.CharField(
max_length=32,
choices=KIND_CHOICES,
blank=True,
default='',
help_text=_('Kind of inventory being represented.'),
)
host_filter = SmartFilterField(
blank=True,
null=True,
default=None,
help_text=_('Filter that will be applied to the hosts of this inventory.'),
)
input_inventories = OrderedManyToManyField(
'Inventory',
blank=True,
through_fields=('constructed_inventory', 'input_inventory'),
related_name='destination_inventories',
help_text=_('Only valid for constructed inventories, this links to the inventories that will be used.'),
through='InventoryConstructedInventoryMembership',
)
instance_groups = OrderedManyToManyField(
'InstanceGroup',
blank=True,
through='InventoryInstanceGroupMembership',
)
admin_role = ImplicitRoleField(
parent_role='organization.inventory_admin_role',
)
update_role = ImplicitRoleField(
parent_role='admin_role',
)
adhoc_role = ImplicitRoleField(
parent_role='admin_role',
)
use_role = ImplicitRoleField(
parent_role='adhoc_role',
)
read_role = ImplicitRoleField(
parent_role=[
'organization.auditor_role',
'update_role',
'use_role',
'admin_role',
]
)
pending_deletion = models.BooleanField(
default=False,
editable=False,
help_text=_('Flag indicating the inventory is being deleted.'),
)
labels = models.ManyToManyField(
"Label",
blank=True,
related_name='inventory_labels',
help_text=_('Labels associated with this inventory.'),
)
prevent_instance_group_fallback = models.BooleanField(
default=False,
help_text=(
"If enabled, the inventory will prevent adding any organization "
"instance groups to the list of preferred instances groups to run "
"associated job templates on."
"If this setting is enabled and you provided an empty list, the global instance "
"groups will be applied."
),
)
def get_absolute_url(self, request=None):
if request is not None:
# circular import
from awx.api.urls.inventory import constructed_inventory_urls
route = resolve(request.path_info)
if any(route.url_name == url.name for url in constructed_inventory_urls):
return reverse('api:constructed_inventory_detail', kwargs={'pk': self.pk}, request=request)
return reverse('api:inventory_detail', kwargs={'pk': self.pk}, request=request)
variables_dict = VarsDictProperty('variables')
def get_group_hosts_map(self):
"""
Return dictionary mapping group_id to set of child host_id's.
"""
# FIXME: Cache this mapping?
group_hosts_kw = dict(group__inventory_id=self.pk, host__inventory_id=self.pk)
group_hosts_qs = Group.hosts.through.objects.filter(**group_hosts_kw)
group_hosts_qs = group_hosts_qs.values_list('group_id', 'host_id')
group_hosts_map = {}
for group_id, host_id in group_hosts_qs:
group_host_ids = group_hosts_map.setdefault(group_id, set())
group_host_ids.add(host_id)
return group_hosts_map
def get_group_parents_map(self):
"""
Return dictionary mapping group_id to set of parent group_id's.
"""
# FIXME: Cache this mapping?
group_parents_kw = dict(from_group__inventory_id=self.pk, to_group__inventory_id=self.pk)
group_parents_qs = Group.parents.through.objects.filter(**group_parents_kw)
group_parents_qs = group_parents_qs.values_list('from_group_id', 'to_group_id')
group_parents_map = {}
for from_group_id, to_group_id in group_parents_qs:
group_parents = group_parents_map.setdefault(from_group_id, set())
group_parents.add(to_group_id)
return group_parents_map
def get_group_children_map(self):
"""
Return dictionary mapping group_id to set of child group_id's.
"""
# FIXME: Cache this mapping?
group_parents_kw = dict(from_group__inventory_id=self.pk, to_group__inventory_id=self.pk)
group_parents_qs = Group.parents.through.objects.filter(**group_parents_kw)
group_parents_qs = group_parents_qs.values_list('from_group_id', 'to_group_id')
group_children_map = {}
for from_group_id, to_group_id in group_parents_qs:
group_children = group_children_map.setdefault(to_group_id, set())
group_children.add(from_group_id)
return group_children_map
@staticmethod
def parse_slice_params(slice_str):
m = re.match(r"slice(?P<number>\d+)of(?P<step>\d+)", slice_str)
if not m:
raise ParseError(_('Could not parse subset as slice specification.'))
number = int(m.group('number'))
step = int(m.group('step'))
if number > step:
raise ParseError(_('Slice number must be less than total number of slices.'))
elif number < 1:
raise ParseError(_('Slice number must be 1 or higher.'))
return (number, step)
def get_sliced_hosts(self, host_queryset, slice_number, slice_count):
"""
Returns a slice of Hosts given a slice number and total slice count, or
the original queryset if slicing is not requested.
NOTE: If slicing is performed, this will return a List[Host] with the
resulting slice. If slicing is not performed it will return the
original queryset (not evaluating it or forcing it to a list). This
puts the burden on the caller to check the resulting type. This is
non-ideal because it's easy to get wrong, but I think the only way
around it is to force the queryset which has memory implications for
large inventories.
"""
if slice_count > 1 and slice_number > 0:
offset = slice_number - 1
host_queryset = host_queryset[offset::slice_count]
return host_queryset
def get_script_data(self, hostvars=False, towervars=False, show_all=False, slice_number=1, slice_count=1):
hosts_kw = dict()
if not show_all:
hosts_kw['enabled'] = True
fetch_fields = ['name', 'id', 'variables', 'inventory_id']
if towervars:
fetch_fields.append('enabled')
host_queryset = self.hosts.filter(**hosts_kw).order_by('name').only(*fetch_fields)
hosts = self.get_sliced_hosts(host_queryset, slice_number, slice_count)
data = dict()
all_group = data.setdefault('all', dict())
all_hostnames = set(host.name for host in hosts)
if self.variables_dict:
all_group['vars'] = self.variables_dict
if self.kind == 'smart':
all_group['hosts'] = [host.name for host in hosts]
else:
# Keep track of hosts that are members of a group
grouped_hosts = set([])
# Build in-memory mapping of groups and their hosts.
group_hosts_qs = Group.hosts.through.objects.filter(group__inventory_id=self.id, host__inventory_id=self.id).values_list(
'group_id', 'host_id', 'host__name'
)
group_hosts_map = {}
for group_id, host_id, host_name in group_hosts_qs:
if host_name not in all_hostnames:
continue # host might not be in current shard
group_hostnames = group_hosts_map.setdefault(group_id, [])
group_hostnames.append(host_name)
grouped_hosts.add(host_name)
# Build in-memory mapping of groups and their children.
group_parents_qs = Group.parents.through.objects.filter(
from_group__inventory_id=self.id,
to_group__inventory_id=self.id,
).values_list('from_group_id', 'from_group__name', 'to_group_id')
group_children_map = {}
for from_group_id, from_group_name, to_group_id in group_parents_qs:
group_children = group_children_map.setdefault(to_group_id, [])
group_children.append(from_group_name)
# Add ungrouped hosts to all group
all_group['hosts'] = [host.name for host in hosts if host.name not in grouped_hosts]
# Now use in-memory maps to build up group info.
all_group_names = []
for group in self.groups.only('name', 'id', 'variables', 'inventory_id'):
group_info = dict()
if group.id in group_hosts_map:
group_info['hosts'] = group_hosts_map[group.id]
if group.id in group_children_map:
group_info['children'] = group_children_map[group.id]
group_vars = group.variables_dict
if group_vars:
group_info['vars'] = group_vars
if group_info:
data[group.name] = group_info
all_group_names.append(group.name)
# add all groups as children of all group, includes empty groups
if all_group_names:
all_group['children'] = all_group_names
if hostvars:
data.setdefault('_meta', dict())
data['_meta'].setdefault('hostvars', dict())
for host in hosts:
data['_meta']['hostvars'][host.name] = host.variables_dict
if towervars:
for prefix in ('host', 'tower'):
tower_dict = {
f'remote_{prefix}_enabled': str(host.enabled).lower(),
f'remote_{prefix}_id': host.id,
}
data['_meta']['hostvars'][host.name].update(tower_dict)
return data
def update_computed_fields(self):
"""
Update model fields that are computed from database relationships.
"""
logger.debug("Going to update inventory computed fields, pk={0}".format(self.pk))
start_time = time.time()
active_hosts = self.hosts
failed_hosts = active_hosts.filter(last_job_host_summary__failed=True)
active_groups = self.groups
if self.kind == 'smart':
active_groups = active_groups.none()
if self.kind == 'smart':
active_inventory_sources = self.inventory_sources.none()
else:
active_inventory_sources = self.inventory_sources.filter(source__in=CLOUD_INVENTORY_SOURCES)
failed_inventory_sources = active_inventory_sources.filter(last_job_failed=True)
total_hosts = active_hosts.count()
# if total_hosts has changed, set update_task_impact to True
update_task_impact = total_hosts != self.total_hosts
computed_fields = {
'has_active_failures': bool(failed_hosts.count()),
'total_hosts': total_hosts,
'hosts_with_active_failures': failed_hosts.count(),
'total_groups': active_groups.count(),
'has_inventory_sources': bool(active_inventory_sources.count()),
'total_inventory_sources': active_inventory_sources.count(),
'inventory_sources_with_failures': failed_inventory_sources.count(),
}
# CentOS python seems to have issues clobbering the inventory on poor timing during certain operations
iobj = Inventory.objects.get(id=self.id)
for field, value in list(computed_fields.items()):
if getattr(iobj, field) != value:
setattr(iobj, field, value)
# update in-memory object
setattr(self, field, value)
else:
computed_fields.pop(field)
if computed_fields:
iobj.save(update_fields=computed_fields.keys())
if update_task_impact:
# if total hosts count has changed, re-calculate task_impact for any
# job that is still in pending for this inventory, since task_impact
# is cached on task creation and used in task management system
tasks = self.jobs.filter(status="pending")
for t in tasks:
t.task_impact = t._get_task_impact()
UnifiedJob.objects.bulk_update(tasks, ['task_impact'])
logger.debug("Finished updating inventory computed fields, pk={0}, in {1:.3f} seconds".format(self.pk, time.time() - start_time))
def websocket_emit_status(self, status):
connection.on_commit(
lambda: emit_channel_notification('inventories-status_changed', {'group_name': 'inventories', 'inventory_id': self.id, 'status': status})
)
@property
def root_groups(self):
group_pks = self.groups.values_list('pk', flat=True)
return self.groups.exclude(parents__pk__in=group_pks).distinct()
@transaction.atomic
def schedule_deletion(self, user_id=None):
from awx.main.tasks.system import delete_inventory
from awx.main.signals import activity_stream_delete
if self.pending_deletion is True:
raise RuntimeError("Inventory is already pending deletion.")
self.pending_deletion = True
self.save(update_fields=['pending_deletion'])
self.jobtemplates.clear()
activity_stream_delete(Inventory, self, inventory_delete_flag=True)
self.websocket_emit_status('pending_deletion')
delete_inventory.delay(self.pk, user_id)
def _update_host_smart_inventory_memeberships(self):
if self.kind == 'smart' and settings.AWX_REBUILD_SMART_MEMBERSHIP:
def on_commit():
from awx.main.tasks.system import update_host_smart_inventory_memberships
update_host_smart_inventory_memberships.delay()
connection.on_commit(on_commit)
def _enforce_constructed_source(self):
"""
Constructed inventory should always have exactly 1 inventory source, constructed type
this enforces that requirement
"""
if self.kind == 'constructed':
if not self.inventory_sources.exists():
self.inventory_sources.create(
source='constructed', name=f'Auto-created source for: {self.name}'[:512], overwrite=True, overwrite_vars=True, update_on_launch=True
)
def save(self, *args, **kwargs):
self._update_host_smart_inventory_memeberships()
super(Inventory, self).save(*args, **kwargs)
if self.kind == 'smart' and 'host_filter' in kwargs.get('update_fields', ['host_filter']) and connection.vendor != 'sqlite':
# Minimal update of host_count for smart inventory host filter changes
self.update_computed_fields()
self._enforce_constructed_source()
def delete(self, *args, **kwargs):
self._update_host_smart_inventory_memeberships()
super(Inventory, self).delete(*args, **kwargs)
'''
RelatedJobsMixin
'''
def _get_related_jobs(self):
return UnifiedJob.objects.non_polymorphic().filter(Q(job__inventory=self) | Q(inventoryupdate__inventory=self) | Q(adhoccommand__inventory=self))
class SmartInventoryMembership(BaseModel):
"""
A lookup table for Host membership in Smart Inventory
"""
class Meta:
app_label = 'main'
unique_together = (('host', 'inventory'),)
inventory = models.ForeignKey('Inventory', related_name='+', on_delete=models.CASCADE)
host = models.ForeignKey('Host', related_name='+', on_delete=models.CASCADE)
class Host(CommonModelNameNotUnique, RelatedJobsMixin):
"""
A managed node
"""
FIELDS_TO_PRESERVE_AT_COPY = ['name', 'description', 'groups', 'inventory', 'enabled', 'instance_id', 'variables']
class Meta:
app_label = 'main'
unique_together = (("name", "inventory"),) # FIXME: Add ('instance_id', 'inventory') after migration.
ordering = ('name',)
inventory = models.ForeignKey(
'Inventory',
related_name='hosts',
on_delete=models.CASCADE,
)
smart_inventories = models.ManyToManyField(
'Inventory',
related_name='+',
through='SmartInventoryMembership',
)
enabled = models.BooleanField(
default=True,
help_text=_('Is this host online and available for running jobs?'),
)
instance_id = models.CharField(
max_length=1024,
blank=True,
default='',
help_text=_('The value used by the remote inventory source to uniquely identify the host'),
)
variables = accepts_json(
models.TextField(
blank=True,
default='',
help_text=_('Host variables in JSON or YAML format.'),
)
)
last_job = models.ForeignKey(
'Job',
related_name='hosts_as_last_job+',
null=True,
default=None,
editable=False,
on_delete=models.SET_NULL,
)
last_job_host_summary = models.ForeignKey(
'JobHostSummary',
related_name='hosts_as_last_job_summary+',
blank=True,
null=True,
default=None,
editable=False,
on_delete=models.SET_NULL,
)
inventory_sources = models.ManyToManyField(
'InventorySource',
related_name='hosts',
editable=False,
help_text=_('Inventory source(s) that created or modified this host.'),
)
ansible_facts = models.JSONField(
blank=True,
default=dict,
help_text=_('Arbitrary JSON structure of most recent ansible_facts, per-host.'),
)
ansible_facts_modified = models.DateTimeField(
default=None,
editable=False,
null=True,
help_text=_('The date and time ansible_facts was last modified.'),
)
objects = HostManager()
def get_absolute_url(self, request=None):
return reverse('api:host_detail', kwargs={'pk': self.pk}, request=request)
variables_dict = VarsDictProperty('variables')
@property
def all_groups(self):
"""
Return all groups of which this host is a member, avoiding infinite
recursion in the case of cyclical group relations.
"""
group_parents_map = self.inventory.get_group_parents_map()
group_pks = set(self.groups.values_list('pk', flat=True))
child_pks_to_check = set()
child_pks_to_check.update(group_pks)
child_pks_checked = set()
while child_pks_to_check:
for child_pk in list(child_pks_to_check):
p_ids = group_parents_map.get(child_pk, set())
group_pks.update(p_ids)
child_pks_to_check.remove(child_pk)
child_pks_checked.add(child_pk)
child_pks_to_check.update(p_ids - child_pks_checked)
return Group.objects.filter(pk__in=group_pks).distinct()
# Use .job_host_summaries.all() to get jobs affecting this host.
# Use .job_events.all() to get events affecting this host.
def get_effective_host_name(self):
"""
Return the name of the host that will be used in actual ansible
command run.
"""
host_name = self.name
if 'ansible_ssh_host' in self.variables_dict:
host_name = self.variables_dict['ansible_ssh_host']
if 'ansible_host' in self.variables_dict:
host_name = self.variables_dict['ansible_host']
return host_name
def _update_host_smart_inventory_memeberships(self):
if settings.AWX_REBUILD_SMART_MEMBERSHIP:
def on_commit():
from awx.main.tasks.system import update_host_smart_inventory_memberships
update_host_smart_inventory_memberships.delay()
connection.on_commit(on_commit)
def clean_name(self):
try:
sanitize_jinja(self.name)
except ValueError as e:
raise ValidationError(str(e) + ": {}".format(self.name))
return self.name
def save(self, *args, **kwargs):
self._update_host_smart_inventory_memeberships()
super(Host, self).save(*args, **kwargs)
def delete(self, *args, **kwargs):
self._update_host_smart_inventory_memeberships()
super(Host, self).delete(*args, **kwargs)
'''
RelatedJobsMixin
'''
def _get_related_jobs(self):
return self.inventory._get_related_jobs()
class Group(CommonModelNameNotUnique, RelatedJobsMixin):
"""
A group containing managed hosts. A group or host may belong to multiple
groups.
"""
FIELDS_TO_PRESERVE_AT_COPY = ['name', 'description', 'inventory', 'children', 'parents', 'hosts', 'variables']
class Meta:
app_label = 'main'
unique_together = (("name", "inventory"),)
ordering = ('name',)
inventory = models.ForeignKey(
'Inventory',
related_name='groups',
on_delete=models.CASCADE,
)
# Can also be thought of as: parents == member_of, children == members
parents = models.ManyToManyField(
'self',
symmetrical=False,
related_name='children',
blank=True,
)
variables = accepts_json(
models.TextField(
blank=True,
default='',
help_text=_('Group variables in JSON or YAML format.'),
)
)
hosts = models.ManyToManyField(
'Host',
related_name='groups',
blank=True,
help_text=_('Hosts associated directly with this group.'),
)
inventory_sources = models.ManyToManyField(
'InventorySource',
related_name='groups',
editable=False,
help_text=_('Inventory source(s) that created or modified this group.'),
)
def get_absolute_url(self, request=None):
return reverse('api:group_detail', kwargs={'pk': self.pk}, request=request)
@transaction.atomic
def delete_recursive(self):
from awx.main.utils import ignore_inventory_computed_fields
from awx.main.tasks.system import update_inventory_computed_fields
from awx.main.signals import disable_activity_stream, activity_stream_delete
def mark_actual():
all_group_hosts = Group.hosts.through.objects.select_related("host", "group").filter(group__inventory=self.inventory)
group_hosts = {'groups': {}, 'hosts': {}}
all_group_parents = Group.parents.through.objects.select_related("from_group", "to_group").filter(from_group__inventory=self.inventory)
group_children = {}
group_parents = {}
marked_hosts = []
marked_groups = [self.id]
for pairing in all_group_hosts:
if pairing.group_id not in group_hosts['groups']:
group_hosts['groups'][pairing.group_id] = []
if pairing.host_id not in group_hosts['hosts']:
group_hosts['hosts'][pairing.host_id] = []
group_hosts['groups'][pairing.group_id].append(pairing.host_id)
group_hosts['hosts'][pairing.host_id].append(pairing.group_id)
for pairing in all_group_parents:
if pairing.to_group_id not in group_children:
group_children[pairing.to_group_id] = []
if pairing.from_group_id not in group_parents:
group_parents[pairing.from_group_id] = []
group_children[pairing.to_group_id].append(pairing.from_group_id)
group_parents[pairing.from_group_id].append(pairing.to_group_id)
linked_children = [(self.id, g) for g in group_children[self.id]] if self.id in group_children else []
if self.id in group_hosts['groups']:
for host in copy.copy(group_hosts['groups'][self.id]):
group_hosts['hosts'][host].remove(self.id)
group_hosts['groups'][self.id].remove(host)
if len(group_hosts['hosts'][host]) < 1:
marked_hosts.append(host)
for subgroup in linked_children:
parent, group = subgroup
group_parents[group].remove(parent)
group_children[parent].remove(group)
if len(group_parents[group]) > 0:
continue
for host in copy.copy(group_hosts['groups'].get(group, [])):
group_hosts['hosts'][host].remove(group)
group_hosts['groups'][group].remove(host)
if len(group_hosts['hosts'][host]) < 1:
marked_hosts.append(host)
if group in group_children:
for direct_child in group_children[group]:
linked_children.append((group, direct_child))
marked_groups.append(group)
Group.objects.filter(id__in=marked_groups).delete()
Host.objects.filter(id__in=marked_hosts).delete()
update_inventory_computed_fields.delay(self.inventory.id)
with ignore_inventory_computed_fields():
with disable_activity_stream():
mark_actual()
activity_stream_delete(None, self)
variables_dict = VarsDictProperty('variables')
def get_all_parents(self, except_pks=None):
"""
Return all parents of this group recursively. The group itself will
be excluded unless there is a cycle leading back to it.
"""
group_parents_map = self.inventory.get_group_parents_map()
child_pks_to_check = set([self.pk])
child_pks_checked = set()
parent_pks = set()
while child_pks_to_check:
for child_pk in list(child_pks_to_check):
p_ids = group_parents_map.get(child_pk, set())
parent_pks.update(p_ids)
child_pks_to_check.remove(child_pk)
child_pks_checked.add(child_pk)
child_pks_to_check.update(p_ids - child_pks_checked)
return Group.objects.filter(pk__in=parent_pks).distinct()
@property
def all_parents(self):
return self.get_all_parents()
def get_all_children(self, except_pks=None):
"""
Return all children of this group recursively. The group itself will
be excluded unless there is a cycle leading back to it.
"""
group_children_map = self.inventory.get_group_children_map()
parent_pks_to_check = set([self.pk])
parent_pks_checked = set()
child_pks = set()
while parent_pks_to_check:
for parent_pk in list(parent_pks_to_check):
c_ids = group_children_map.get(parent_pk, set())
child_pks.update(c_ids)
parent_pks_to_check.remove(parent_pk)
parent_pks_checked.add(parent_pk)
parent_pks_to_check.update(c_ids - parent_pks_checked)
return Group.objects.filter(pk__in=child_pks).distinct()
@property
def all_children(self):
return self.get_all_children()
def get_all_hosts(self, except_group_pks=None):
"""
Return all hosts associated with this group or any of its children.
"""
group_children_map = self.inventory.get_group_children_map()
group_hosts_map = self.inventory.get_group_hosts_map()
parent_pks_to_check = set([self.pk])
parent_pks_checked = set()
host_pks = set()
while parent_pks_to_check:
for parent_pk in list(parent_pks_to_check):
c_ids = group_children_map.get(parent_pk, set())
parent_pks_to_check.remove(parent_pk)
parent_pks_checked.add(parent_pk)
parent_pks_to_check.update(c_ids - parent_pks_checked)
h_ids = group_hosts_map.get(parent_pk, set())
host_pks.update(h_ids)
return Host.objects.filter(pk__in=host_pks).distinct()
@property
def all_hosts(self):
return self.get_all_hosts()
@property
def job_host_summaries(self):
from awx.main.models.jobs import JobHostSummary
return JobHostSummary.objects.filter(host__in=self.all_hosts)
@property
def job_events(self):
from awx.main.models.jobs import JobEvent
return JobEvent.objects.filter(host__in=self.all_hosts)
@property
def ad_hoc_commands(self):
from awx.main.models.ad_hoc_commands import AdHocCommand
return AdHocCommand.objects.filter(hosts__in=self.all_hosts)
'''
RelatedJobsMixin
'''
def _get_related_jobs(self):
return UnifiedJob.objects.non_polymorphic().filter(Q(job__inventory=self.inventory) | Q(inventoryupdate__inventory_source__groups=self))
class HostMetric(models.Model):
hostname = models.CharField(unique=True, max_length=512)
first_automation = models.DateTimeField(auto_now_add=True, null=False, db_index=True, help_text=_('When the host was first automated against'))
last_automation = models.DateTimeField(db_index=True, help_text=_('When the host was last automated against'))
last_deleted = models.DateTimeField(null=True, db_index=True, help_text=_('When the host was last deleted'))
automated_counter = models.BigIntegerField(default=0, help_text=_('How many times was the host automated'))
deleted_counter = models.IntegerField(default=0, help_text=_('How many times was the host deleted'))
deleted = models.BooleanField(
default=False, help_text=_('Boolean flag saying whether the host is deleted and therefore not counted into the subscription consumption')
)
used_in_inventories = models.IntegerField(null=True, help_text=_('How many inventories contain this host'))
objects = models.Manager()
active_objects = HostMetricActiveManager()
def get_absolute_url(self, request=None):
return reverse('api:host_metric_detail', kwargs={'pk': self.pk}, request=request)
def soft_delete(self):
if not self.deleted:
self.deleted_counter = (self.deleted_counter or 0) + 1
self.last_deleted = now()
self.deleted = True
self.save(update_fields=['deleted', 'deleted_counter', 'last_deleted'])
def soft_restore(self):
if self.deleted:
self.deleted = False
self.save(update_fields=['deleted'])
class HostMetricSummaryMonthly(models.Model):
"""
HostMetric summaries computed by scheduled task 'awx.main.tasks.system.host_metric_summary_monthly' monthly
"""
date = models.DateField(unique=True)
license_consumed = models.BigIntegerField(default=0, help_text=_("How many unique hosts are consumed from the license"))
license_capacity = models.BigIntegerField(default=0, help_text=_("'License capacity as max. number of unique hosts"))
hosts_added = models.IntegerField(default=0, help_text=_("How many hosts were added in the associated month, consuming more license capacity"))
hosts_deleted = models.IntegerField(default=0, help_text=_("How many hosts were deleted in the associated month, freeing the license capacity"))
indirectly_managed_hosts = models.IntegerField(default=0, help_text=("Manually entered number indirectly managed hosts for a certain month"))
class InventorySourceOptions(BaseModel):
"""
Common fields for InventorySource and InventoryUpdate.
"""
injectors = dict()
SOURCE_CHOICES = [
('file', _('File, Directory or Script')),
('constructed', _('Template additional groups and hostvars at runtime')),
('scm', _('Sourced from a Project')),
('ec2', _('Amazon EC2')),
('gce', _('Google Compute Engine')),
('azure_rm', _('Microsoft Azure Resource Manager')),
('vmware', _('VMware vCenter')),
('satellite6', _('Red Hat Satellite 6')),
('openstack', _('OpenStack')),
('rhv', _('Red Hat Virtualization')),
('controller', _('Red Hat Ansible Automation Platform')),
('insights', _('Red Hat Insights')),
('terraform', _('Terraform State')),
('openshift_virtualization', _('OpenShift Virtualization')),
]
# From the options of the Django management base command
INVENTORY_UPDATE_VERBOSITY_CHOICES = [
(0, '0 (WARNING)'),
(1, '1 (INFO)'),
(2, '2 (DEBUG)'),
]
class Meta:
abstract = True
source = models.CharField(
max_length=32,
choices=SOURCE_CHOICES,
blank=False,
default=None,
)
source_path = models.CharField(
max_length=1024,
blank=True,
default='',
)
source_vars = models.TextField(
blank=True,
default='',
help_text=_('Inventory source variables in YAML or JSON format.'),
)
scm_branch = models.CharField(
max_length=1024,
default='',
blank=True,
help_text=_('Inventory source SCM branch. Project default used if blank. Only allowed if project allow_override field is set to true.'),
)
enabled_var = models.TextField(
blank=True,
default='',
help_text=_(
'Retrieve the enabled state from the given dict of host '
'variables. The enabled variable may be specified as "foo.bar", '
'in which case the lookup will traverse into nested dicts, '
'equivalent to: from_dict.get("foo", {}).get("bar", default)'
),
)
enabled_value = models.TextField(
blank=True,
default='',
help_text=_(
'Only used when enabled_var is set. Value when the host is '
'considered enabled. For example if enabled_var="status.power_state"'
'and enabled_value="powered_on" with host variables:'
'{'
' "status": {'
' "power_state": "powered_on",'
' "created": "2020-08-04T18:13:04+00:00",'
' "healthy": true'
' },'
' "name": "foobar",'
' "ip_address": "192.168.2.1"'
'}'
'The host would be marked enabled. If power_state where any '
'value other than powered_on then the host would be disabled '
'when imported. If the key is not found then the '
'host will be enabled'