-
Notifications
You must be signed in to change notification settings - Fork 5.5k
/
gitfs.py
2521 lines (2336 loc) · 92.9 KB
/
gitfs.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
# -*- coding: utf-8 -*-
# Import python libs
from __future__ import absolute_import
import copy
import distutils.version # pylint: disable=import-error,no-name-in-module
import errno
import fnmatch
import glob
import hashlib
import logging
import os
import re
import shlex
import shutil
import stat
import subprocess
from datetime import datetime
VALID_PROVIDERS = ('gitpython', 'pygit2', 'dulwich')
# Optional per-remote params that can only be used on a per-remote basis, and
# thus do not have defaults in salt/config.py.
PER_REMOTE_ONLY = ('name',)
SYMLINK_RECURSE_DEPTH = 100
# Auth support (auth params can be global or per-remote, too)
AUTH_PROVIDERS = ('pygit2',)
AUTH_PARAMS = ('user', 'password', 'pubkey', 'privkey', 'passphrase',
'insecure_auth')
_RECOMMEND_GITPYTHON = (
'GitPython is installed, you may wish to set {0}_provider to '
'\'gitpython\' in the master config file to use GitPython for {0} '
'support.'
)
_RECOMMEND_PYGIT2 = (
'pygit2 is installed, you may wish to set {0}_provider to '
'\'pygit2\' in the master config file to use pygit2 for for {0} '
'support.'
)
_RECOMMEND_DULWICH = (
'Dulwich is installed, you may wish to set {0}_provider to '
'\'dulwich\' in the master config file to use Dulwich for {0} '
'support.'
)
_INVALID_REPO = (
'Cache path {0} (corresponding remote: {1}) exists but is not a valid '
'git repository. You will need to manually delete this directory on the '
'master to continue to use this {2} remote.'
)
# Import salt libs
import salt.utils
import salt.utils.itertools
import salt.utils.url
import salt.fileserver
from salt.exceptions import FileserverConfigError
from salt.utils.event import tagify
# Import third party libs
import salt.ext.six as six
# pylint: disable=import-error
try:
import git
import gitdb
HAS_GITPYTHON = True
except ImportError:
HAS_GITPYTHON = False
try:
import pygit2
HAS_PYGIT2 = True
try:
GitError = pygit2.errors.GitError
except AttributeError:
GitError = Exception
except ImportError:
HAS_PYGIT2 = False
try:
import dulwich.errors
import dulwich.repo
import dulwich.client
import dulwich.config
import dulwich.objects
HAS_DULWICH = True
except ImportError:
HAS_DULWICH = False
# pylint: enable=import-error
log = logging.getLogger(__name__)
# Minimum versions for backend providers
GITPYTHON_MINVER = '0.3'
PYGIT2_MINVER = '0.20.3'
LIBGIT2_MINVER = '0.20.0'
# dulwich.__version__ is a versioninfotuple so we can compare tuples
# instead of using distutils.version.LooseVersion
DULWICH_MINVER = (0, 9, 4)
def failhard(role):
'''
Fatal configuration issue, raise an exception
'''
raise FileserverConfigError('Failed to load {0}'.format(role))
class GitProvider(object):
'''
Base class for gitfs/git_pillar provider classes Should never be used
directly.
self.provider should be set in the sub-class' __init__ function before
invoking GitProvider.__init__().
'''
def __init__(self, opts, remote, per_remote_defaults,
override_params, cache_root, role='gitfs'):
self.opts = opts
self.role = role
self.env_blacklist = self.opts.get(
'{0}_env_blacklist'.format(self.role), [])
self.env_whitelist = self.opts.get(
'{0}_env_whitelist'.format(self.role), [])
repo_conf = copy.deepcopy(per_remote_defaults)
per_remote_collisions = [x for x in override_params
if x in PER_REMOTE_ONLY]
if per_remote_collisions:
log.critical(
'The following parameter names are restricted to per-remote '
'use only: {0}. This is a bug, please report it.'.format(
', '.join(per_remote_collisions)
)
)
try:
valid_per_remote_params = override_params + PER_REMOTE_ONLY
except TypeError:
valid_per_remote_params = \
list(override_params) + list(PER_REMOTE_ONLY)
if isinstance(remote, dict):
self.id = next(iter(remote))
self.get_url()
per_remote_conf = dict(
[(key, six.text_type(val)) for key, val in
six.iteritems(salt.utils.repack_dictlist(remote[self.id]))]
)
if not per_remote_conf:
log.critical(
'Invalid per-remote configuration for {0} remote \'{1}\'. '
'If no per-remote parameters are being specified, there '
'may be a trailing colon after the URL, which should be '
'removed. Check the master configuration file.'
.format(self.role, self.id)
)
failhard(self.role)
# Separate the per-remote-only (non-global) parameters
per_remote_only = {}
for param in PER_REMOTE_ONLY:
if param in per_remote_conf:
per_remote_only[param] = per_remote_conf.pop(param)
per_remote_errors = False
for param in (x for x in per_remote_conf
if x not in valid_per_remote_params):
if param in AUTH_PARAMS \
and self.provider not in AUTH_PROVIDERS:
msg = (
'{0} authentication parameter \'{1}\' (from remote '
'\'{2}\') is only supported by the following '
'provider(s): {3}. Current {0}_provider is \'{4}\'.'
.format(
self.role,
param,
self.id,
', '.join(AUTH_PROVIDERS),
self.provider
)
)
if self.role == 'gitfs':
msg += (
'See the GitFS Walkthrough in the Salt '
'documentation for further information.'
)
log.critical(msg)
else:
msg = (
'Invalid {0} configuration parameter \'{1}\' in '
'remote {2}. Valid parameters are: {3}.'.format(
self.role,
param,
self.url,
', '.join(valid_per_remote_params)
)
)
if self.role == 'gitfs':
msg += (
' See the GitFS Walkthrough in the Salt '
'documentation for further information.'
)
log.critical(msg)
per_remote_errors = True
if per_remote_errors:
failhard(self.role)
repo_conf.update(per_remote_conf)
repo_conf.update(per_remote_only)
else:
self.id = remote
self.get_url()
# Winrepo doesn't support the 'root' option, but it still must be part
# of the GitProvider object because other code depends on it. Add it as
# an empty string.
if 'root' not in repo_conf:
repo_conf['root'] = ''
if self.role == 'winrepo' and 'name' not in repo_conf:
# Ensure that winrepo has the 'name' parameter set if it wasn't
# provided. Default to the last part of the URL, minus the .git if
# it is present.
repo_conf['name'] = self.url.rsplit('/', 1)[-1]
# Remove trailing .git from name
if repo_conf['name'].lower().endswith('.git'):
repo_conf['name'] = repo_conf['name'][:-4]
# Set all repo config params as attributes
for key, val in six.iteritems(repo_conf):
setattr(self, key, val)
if hasattr(self, 'mountpoint'):
self.mountpoint = salt.utils.url.strip_proto(self.mountpoint)
else:
# For providers which do not use a mountpoint, assume the
# filesystem is mounted at the root of the fileserver.
self.mountpoint = ''
if not isinstance(self.url, six.string_types):
log.critical(
'Invalid {0} remote \'{1}\'. Remotes must be strings, you '
'may need to enclose the URL in quotes'.format(
self.role,
self.id
)
)
failhard(self.role)
hash_type = getattr(hashlib, self.opts.get('hash_type', 'md5'))
self.hash = hash_type(self.id).hexdigest()
self.cachedir_basename = getattr(self, 'name', self.hash)
self.cachedir = os.path.join(cache_root, self.cachedir_basename)
if not os.path.isdir(self.cachedir):
os.makedirs(self.cachedir)
try:
self.new = self.init_remote()
except Exception as exc:
msg = ('Exception caught while initializing {0} remote \'{1}\': '
'{2}'.format(self.role, self.id, exc))
if isinstance(self, GitPython):
msg += ' Perhaps git is not available.'
log.critical(msg, exc_info_on_loglevel=logging.DEBUG)
failhard(self.role)
def _get_envs_from_ref_paths(self, refs):
'''
Return the names of remote refs (stripped of the remote name) and tags
which are exposed as environments. If a branch or tag matches
'''
def _check_ref(env_set, base_ref, rname):
'''
Check the ref and resolve it as the base_ref if it matches. If the
resulting env is exposed via whitelist/blacklist, add it to the
env_set.
'''
if base_ref is not None and base_ref == rname:
rname = 'base'
if self.env_is_exposed(rname):
env_set.add(rname)
ret = set()
base_ref = getattr(self, 'base', None)
for ref in refs:
ref = re.sub('^refs/', '', ref)
rtype, rname = ref.split('/', 1)
if rtype == 'remotes':
parted = rname.partition('/')
rname = parted[2] if parted[2] else parted[0]
_check_ref(ret, base_ref, rname)
elif rtype == 'tags':
_check_ref(ret, base_ref, rname)
return ret
def check_lock(self):
'''
Used by the provider-specific fetch() function to check the existence
of an update lock, and set the lock if not present. If the lock exists
already, or if there was a problem setting the lock, this function
returns False. If the lock was successfully set, return True.
'''
if os.path.exists(self.lockfile):
log.warning(
'Update lockfile is present for {0} remote \'{1}\', '
'skipping. If this warning persists, it is possible that the '
'update process was interrupted. Removing {2} or running '
'\'salt-run cache.clear_git_lock {0}\' will allow updates to '
'continue for this remote.'
.format(self.role, self.id, self.lockfile)
)
return False
errors = self.lock()[-1]
if errors:
log.error('Unable to set update lock for {0} remote \'{1}\', '
'skipping.'.format(self.role, self.id))
return False
return True
def check_root(self):
'''
Check if the relative root path exists in the checked-out copy of the
remote. Return the full path to that relative root if it does exist,
otherwise return None.
'''
root_dir = os.path.join(self.cachedir, self.root).rstrip(os.sep)
if os.path.isdir(root_dir):
return root_dir
log.error(
'Root path \'{0}\' not present in {1} remote \'{2}\', '
'skipping.'.format(self.root, self.role, self.id)
)
return None
def clean_stale_refs(self):
'''
Not all providers need stale refs to be cleaned manually. Override this
function in a sub-class if this is needed.
'''
return []
def clear_lock(self):
'''
Clear update.lk
'''
def _add_error(errlist, exc):
msg = ('Unable to remove update lock for {0} ({1}): {2} '
.format(self.url, self.lockfile, exc))
log.debug(msg)
errlist.append(msg)
success = []
failed = []
if os.path.exists(self.lockfile):
try:
os.remove(self.lockfile)
except OSError as exc:
if exc.errno == errno.EISDIR:
# Somehow this path is a directory. Should never happen
# unless some wiseguy manually creates a directory at this
# path, but just in case, handle it.
try:
shutil.rmtree(self.lockfile)
except OSError as exc:
_add_error(failed, exc)
else:
_add_error(failed, exc)
else:
msg = 'Removed lock for {0} remote \'{1}\''.format(
self.role,
self.id
)
log.debug(msg)
success.append(msg)
return success, failed
def lock(self):
'''
Place an update.lk
'''
success = []
failed = []
if not os.path.exists(self.lockfile):
try:
with salt.utils.fopen(self.lockfile, 'w+') as fp_:
fp_.write('')
except (IOError, OSError) as exc:
msg = ('Unable to set update lock for {0} ({1}): {2} '
.format(self.url, self.lockfile, exc))
log.error(msg)
failed.append(msg)
else:
msg = 'Set lock for {0} remote \'{1}\''.format(
self.role,
self.id
)
log.debug(msg)
success.append(msg)
return success, failed
def init_remote(self):
'''
This function must be overridden in a sub-class
'''
raise NotImplementedError()
def checkout(self):
'''
This function must be overridden in a sub-class
'''
raise NotImplementedError()
def dir_list(self, tgt_env):
'''
This function must be overridden in a sub-class
'''
raise NotImplementedError()
def env_is_exposed(self, tgt_env):
'''
Check if an environment is exposed by comparing it against a whitelist
and blacklist.
'''
return salt.utils.check_whitelist_blacklist(
tgt_env,
whitelist=self.env_whitelist,
blacklist=self.env_blacklist
)
def envs(self):
'''
This function must be overridden in a sub-class
'''
raise NotImplementedError()
def fetch(self):
'''
This function must be overridden in a sub-class
'''
raise NotImplementedError()
def file_list(self, tgt_env):
'''
This function must be overridden in a sub-class
'''
raise NotImplementedError()
def find_file(self, path, tgt_env):
'''
This function must be overridden in a sub-class
'''
raise NotImplementedError()
def get_tree(self, tgt_env):
'''
This function must be overridden in a sub-class
'''
raise NotImplementedError()
def get_url(self):
'''
Examine self.id and assign self.url (and self.branch, for git_pillar)
'''
if self.role in ('git_pillar', 'winrepo'):
# With winrepo and git_pillar, the remote is specified in the
# format '<branch> <url>', so that we can get a unique identifier
# to hash for each remote.
try:
self.branch, self.url = self.id.split(None, 1)
except ValueError:
self.branch = self.opts['{0}_branch'.format(self.role)]
self.url = self.id
else:
self.url = self.id
def verify_auth(self):
'''
Override this function in a sub-class to implement auth checking.
'''
self.credentials = None
return True
def write_file(self, blob, dest):
'''
This function must be overridden in a sub-class
'''
raise NotImplementedError()
class GitPython(GitProvider):
'''
Interface to GitPython
'''
def __init__(self, opts, remote, per_remote_defaults,
override_params, cache_root, role='gitfs'):
self.provider = 'gitpython'
GitProvider.__init__(self, opts, remote, per_remote_defaults,
override_params, cache_root, role)
def checkout(self):
'''
Checkout the configured branch/tag
'''
for ref in ('origin/' + self.branch, self.branch):
try:
self.repo.git.checkout(ref)
except Exception:
continue
return self.check_root()
log.error(
'Failed to checkout {0} from {1} remote \'{2}\': remote ref does '
'not exist'.format(self.branch, self.role, self.id)
)
return None
def clean_stale_refs(self):
'''
Clean stale local refs so they don't appear as fileserver environments
'''
cleaned = []
for ref in self.repo.remotes[0].stale_refs:
if ref.name.startswith('refs/tags/'):
# Work around GitPython bug affecting removal of tags
# https://github.com/gitpython-developers/GitPython/issues/260
self.repo.git.tag('-d', ref.name[10:])
else:
ref.delete(self.repo, ref)
cleaned.append(ref)
if cleaned:
log.debug('{0} cleaned the following stale refs: {1}'
.format(self.role, cleaned))
return cleaned
def init_remote(self):
'''
Initialize/attach to a remote using GitPython. Return a boolean
which will let the calling function know whether or not a new repo was
initialized by this function.
'''
new = False
if not os.listdir(self.cachedir):
# Repo cachedir is empty, initialize a new repo there
self.repo = git.Repo.init(self.cachedir)
new = True
else:
# Repo cachedir exists, try to attach
try:
self.repo = git.Repo(self.cachedir)
except git.exc.InvalidGitRepositoryError:
log.error(_INVALID_REPO.format(self.cachedir, self.url))
return new
self.lockfile = os.path.join(self.repo.working_dir, 'update.lk')
if not self.repo.remotes:
try:
self.repo.create_remote('origin', self.url)
# Ensure tags are also fetched
self.repo.git.config('--add',
'remote.origin.fetch',
'+refs/tags/*:refs/tags/*')
self.repo.git.config('http.sslVerify', self.ssl_verify)
except os.error:
# This exception occurs when two processes are trying to write
# to the git config at once, go ahead and pass over it since
# this is the only write. This should place a lock down.
pass
else:
new = True
return new
def dir_list(self, tgt_env):
'''
Get list of directories for the target environment using GitPython
'''
ret = set()
tree = self.get_tree(tgt_env)
if not tree:
return ret
if self.root:
try:
tree = tree / self.root
except KeyError:
return ret
relpath = lambda path: os.path.relpath(path, self.root)
else:
relpath = lambda path: path
add_mountpoint = lambda path: os.path.join(self.mountpoint, path)
for blob in tree.traverse():
if isinstance(blob, git.Tree):
ret.add(add_mountpoint(relpath(blob.path)))
if self.mountpoint:
ret.add(self.mountpoint)
return ret
def envs(self):
'''
Check the refs and return a list of the ones which can be used as salt
environments.
'''
ref_paths = [x.path for x in self.repo.refs]
return self._get_envs_from_ref_paths(ref_paths)
def fetch(self):
'''
Fetch the repo. If the local copy was updated, return True. If the
local copy was already up-to-date, return False.
'''
if not self.check_lock():
return False
origin = self.repo.remotes[0]
try:
fetch_results = origin.fetch()
except AssertionError:
fetch_results = origin.fetch()
new_objs = False
for fetchinfo in fetch_results:
if fetchinfo.old_commit is not None:
log.debug(
'{0} has updated \'{1}\' for remote \'{2}\' '
'from {3} to {4}'.format(
self.role,
fetchinfo.name,
self.id,
fetchinfo.old_commit.hexsha[:7],
fetchinfo.commit.hexsha[:7]
)
)
new_objs = True
elif fetchinfo.flags in (fetchinfo.NEW_TAG,
fetchinfo.NEW_HEAD):
log.debug(
'{0} has fetched new {1} \'{2}\' for remote \'{3}\' '
.format(
self.role,
'tag' if fetchinfo.flags == fetchinfo.NEW_TAG
else 'head',
fetchinfo.name,
self.id
)
)
new_objs = True
cleaned = self.clean_stale_refs()
return bool(new_objs or cleaned)
def file_list(self, tgt_env):
'''
Get file list for the target environment using GitPython
'''
files = set()
symlinks = {}
tree = self.get_tree(tgt_env)
if not tree:
# Not found, return empty objects
return files, symlinks
if self.root:
try:
tree = tree / self.root
except KeyError:
return files, symlinks
relpath = lambda path: os.path.relpath(path, self.root)
else:
relpath = lambda path: path
add_mountpoint = lambda path: os.path.join(self.mountpoint, path)
for file_blob in tree.traverse():
if not isinstance(file_blob, git.Blob):
continue
file_path = add_mountpoint(relpath(file_blob.path))
files.add(file_path)
if stat.S_ISLNK(file_blob.mode):
stream = six.StringIO()
file_blob.stream_data(stream)
stream.seek(0)
link_tgt = stream.read()
stream.close()
symlinks[file_path] = link_tgt
return files, symlinks
def find_file(self, path, tgt_env):
'''
Find the specified file in the specified environment
'''
tree = self.get_tree(tgt_env)
if not tree:
# Branch/tag/SHA not found
return None, None
blob = None
depth = 0
while True:
depth += 1
if depth > SYMLINK_RECURSE_DEPTH:
break
try:
file_blob = tree / path
if stat.S_ISLNK(file_blob.mode):
# Path is a symlink. The blob data corresponding to
# this path's object ID will be the target of the
# symlink. Follow the symlink and set path to the
# location indicated in the blob data.
stream = six.StringIO()
file_blob.stream_data(stream)
stream.seek(0)
link_tgt = stream.read()
stream.close()
path = os.path.normpath(
os.path.join(os.path.dirname(path), link_tgt)
)
else:
blob = file_blob
break
except KeyError:
# File not found or repo_path points to a directory
break
return blob, blob.hexsha if blob is not None else blob
def get_tree(self, tgt_env):
'''
Return a git.Tree object if the branch/tag/SHA is found, otherwise None
'''
if tgt_env == 'base':
tgt_ref = self.base
else:
tgt_ref = tgt_env
for ref in self.repo.refs:
if isinstance(ref, (git.RemoteReference, git.TagReference)):
parted = ref.name.partition('/')
rspec = parted[2] if parted[2] else parted[0]
if rspec == tgt_ref:
return ref.commit.tree
# Branch or tag not matched, check if 'tgt_env' is a commit
if not self.env_is_exposed(tgt_env):
return None
try:
commit = self.repo.rev_parse(tgt_ref)
return commit.tree
except gitdb.exc.ODBError:
return None
def write_file(self, blob, dest):
'''
Using the blob object, write the file to the destination path
'''
with salt.utils.fopen(dest, 'w+') as fp_:
blob.stream_data(fp_)
class Pygit2(GitProvider):
'''
Interface to Pygit2
'''
def __init__(self, opts, remote, per_remote_defaults,
override_params, cache_root, role='gitfs'):
self.provider = 'pygit2'
self.use_callback = \
distutils.version.LooseVersion(pygit2.__version__) >= \
distutils.version.LooseVersion('0.23.2')
GitProvider.__init__(self, opts, remote, per_remote_defaults,
override_params, cache_root, role)
def checkout(self):
'''
Checkout the configured branch/tag
'''
local_ref = 'refs/heads/' + self.branch
remote_ref = 'refs/remotes/origin/' + self.branch
tag_ref = 'refs/tags/' + self.branch
refs = self.repo.listall_references()
try:
if remote_ref in refs:
# Get commit id for the remote ref
oid = self.repo.lookup_reference(remote_ref).get_object().id
if local_ref not in refs:
# No local branch for this remote, so create one and point
# it at the commit id of the remote ref
self.repo.create_reference(local_ref, oid)
# Check HEAD ref existence (checking out local_ref when HEAD
# ref doesn't exist will raise an exception in pygit2 >= 0.21),
# and create the HEAD ref if it is missing.
head_ref = self.repo.lookup_reference('HEAD').target
if head_ref not in refs and head_ref != local_ref:
branch_name = head_ref.partition('refs/heads/')[-1]
if not branch_name:
# Shouldn't happen, but log an error if it does
log.error(
'pygit2 was unable to resolve branch name from '
'HEAD ref \'{0}\' in {1} remote \'{2}\''.format(
head_ref, self.role, self.id
)
)
return None
remote_head = 'refs/remotes/origin/' + branch_name
if remote_head not in refs:
log.error(
'Unable to find remote ref \'{0}\' in {1} remote '
'\'{2}\''.format(head_ref, self.role, self.id)
)
return None
self.repo.create_reference(
head_ref,
self.repo.lookup_reference(remote_head).target
)
# Point HEAD at the local ref
self.repo.checkout(local_ref)
# Reset HEAD to the commit id of the remote ref
self.repo.reset(oid, pygit2.GIT_RESET_HARD)
return self.check_root()
elif tag_ref in refs:
self.repo.checkout(tag_ref)
return self.check_root()
except Exception as exc:
log.error(
'Failed to checkout {0} from {1} remote \'{2}\': {3}'.format(
self.branch,
self.role,
self.id,
exc
),
exc_info_on_loglevel=logging.DEBUG
)
return None
log.error(
'Failed to checkout {0} from {1} remote \'{2}\': remote ref '
'does not exist'.format(self.branch, self.role, self.id)
)
return None
def clean_stale_refs(self, local_refs=None): # pylint: disable=arguments-differ
'''
Clean stale local refs so they don't appear as fileserver environments
'''
if self.credentials is not None:
log.debug(
'pygit2 does not support detecting stale refs for '
'authenticated remotes, saltenvs will not reflect '
'branches/tags removed from remote \'{0}\''
.format(self.id)
)
return []
if local_refs is None:
local_refs = self.repo.listall_references()
remote_refs = []
cmd_str = 'git ls-remote origin'
cmd = subprocess.Popen(
shlex.split(cmd_str),
close_fds=not salt.utils.is_windows(),
cwd=self.repo.workdir,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
output = cmd.communicate()[0]
if cmd.returncode != 0:
log.warning(
'Failed to list remote references for {0} remote \'{1}\'. '
'Output from \'{2}\' follows:\n{3}'.format(
self.role,
self.id,
cmd_str,
output
)
)
return []
for line in salt.utils.itertools.split(output, '\n'):
try:
# Rename heads to match the remote ref names from
# pygit2.Repository.listall_references()
remote_refs.append(
line.split()[-1].replace(b'refs/heads/',
b'refs/remotes/origin/')
)
except IndexError:
continue
cleaned = []
if remote_refs:
for ref in local_refs:
if ref.startswith('refs/heads/'):
# Local head, ignore it
continue
elif ref not in remote_refs:
self.repo.lookup_reference(ref).delete()
cleaned.append(ref)
if cleaned:
log.debug('{0} cleaned the following stale refs: {1}'
.format(self.role, cleaned))
return cleaned
def init_remote(self):
'''
Initialize/attach to a remote using pygit2. Return a boolean which
will let the calling function know whether or not a new repo was
initialized by this function.
'''
new = False
if not os.listdir(self.cachedir):
# Repo cachedir is empty, initialize a new repo there
self.repo = pygit2.init_repository(self.cachedir)
new = True
else:
# Repo cachedir exists, try to attach
try:
try:
self.repo = pygit2.Repository(self.cachedir)
except pygit2.GitError as exc:
import pwd
# https://github.com/libgit2/pygit2/issues/339
# https://github.com/libgit2/libgit2/issues/2122
if "Error stat'ing config file" not in str(exc):
raise
home = pwd.getpwnam(salt.utils.get_user).pw_dir
pygit2.settings.search_path[pygit2.GIT_CONFIG_LEVEL_GLOBAL] = home
self.repo = pygit2.Repository(self.cachedir)
except KeyError:
log.error(_INVALID_REPO.format(self.cachedir, self.url))
return new
self.lockfile = os.path.join(self.repo.workdir, 'update.lk')
if not self.repo.remotes:
try:
self.repo.create_remote('origin', self.url)
# Ensure tags are also fetched
self.repo.config.set_multivar(
'remote.origin.fetch',
'FOO',
'+refs/tags/*:refs/tags/*'
)
self.repo.config.set_multivar(
'http.sslVerify',
'',
self.ssl_verify
)
except os.error:
# This exception occurs when two processes are trying to write
# to the git config at once, go ahead and pass over it since
# this is the only write. This should place a lock down.
pass
else:
new = True
return new
def dir_list(self, tgt_env):
'''
Get a list of directories for the target environment using pygit2
'''
def _traverse(tree, blobs, prefix):
'''
Traverse through a pygit2 Tree object recursively, accumulating all
the empty directories within it in the "blobs" list
'''
for entry in iter(tree):
if entry.oid not in self.repo:
# Entry is a submodule, skip it
continue
blob = self.repo[entry.oid]
if not isinstance(blob, pygit2.Tree):
continue
blobs.append(os.path.join(prefix, entry.name))
if len(blob):
_traverse(blob, blobs, os.path.join(prefix, entry.name))
ret = set()
tree = self.get_tree(tgt_env)
if not tree:
return ret
if self.root:
try:
oid = tree[self.root].oid
tree = self.repo[oid]
except KeyError:
return ret
if not isinstance(tree, pygit2.Tree):
return ret
relpath = lambda path: os.path.relpath(path, self.root)
else:
relpath = lambda path: path
blobs = []
if len(tree):
_traverse(tree, blobs, self.root)
add_mountpoint = lambda path: os.path.join(self.mountpoint, path)
for blob in blobs:
ret.add(add_mountpoint(relpath(blob)))
if self.mountpoint:
ret.add(self.mountpoint)
return ret
def envs(self):
'''
Check the refs and return a list of the ones which can be used as salt
environments.
'''