/
operations.py
1079 lines (900 loc) · 42.4 KB
/
operations.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
"""
Functions to be used in fabfiles and other non-core code, such as run()/sudo().
"""
from __future__ import with_statement
import os
import os.path
import re
import stat
import subprocess
import sys
import time
from glob import glob
from traceback import format_exc
from contextlib import closing
from fabric.context_managers import settings, char_buffered
from fabric.io import output_loop, input_loop
from fabric.network import needs_host
from fabric.sftp import SFTP
from fabric.state import (env, connections, output, win32, default_channel,
io_sleep)
from fabric.thread_handling import ThreadHandler
from fabric.utils import abort, indent, warn, puts, handle_prompt_abort
# For terminal size logic below
if not win32:
import fcntl
import termios
import struct
def _pty_size():
"""
Obtain (rows, cols) tuple for sizing a pty on the remote end.
Defaults to 80x24 (which is also the 'ssh' lib's default) but will detect
local (stdout-based) terminal window size on non-Windows platforms.
"""
rows, cols = 24, 80
if not win32 and sys.stdout.isatty():
# We want two short unsigned integers (rows, cols)
fmt = 'HH'
# Create an empty (zeroed) buffer for ioctl to map onto. Yay for C!
buffer = struct.pack(fmt, 0, 0)
# Call TIOCGWINSZ to get window size of stdout, returns our filled
# buffer
try:
result = fcntl.ioctl(sys.stdout.fileno(), termios.TIOCGWINSZ,
buffer)
# Unpack buffer back into Python data types
rows, cols = struct.unpack(fmt, result)
# Deal with e.g. sys.stdout being monkeypatched, such as in testing.
# Or termios not having a TIOCGWINSZ.
except AttributeError:
pass
return rows, cols
def _handle_failure(message, exception=None):
"""
Call `abort` or `warn` with the given message.
The value of ``env.warn_only`` determines which method is called.
If ``exception`` is given, it is inspected to get a string message, which
is printed alongside the user-generated ``message``.
"""
func = env.warn_only and warn or abort
# If debug printing is on, append a traceback to the message
if output.debug:
message += "\n\n" + format_exc()
# Otherwise, if we were given an exception, append its contents.
elif exception is not None:
# Figure out how to get a string out of the exception; EnvironmentError
# subclasses, for example, "are" integers and .strerror is the string.
# Others "are" strings themselves. May have to expand this further for
# other error types.
if hasattr(exception, 'strerror') and exception.strerror is not None:
underlying = exception.strerror
else:
underlying = exception
message += "\n\nUnderlying exception message:\n" + indent(underlying)
return func(message)
def _shell_escape(string):
"""
Escape double quotes, backticks and dollar signs in given ``string``.
For example::
>>> _shell_escape('abc$')
'abc\\\\$'
>>> _shell_escape('"')
'\\\\"'
"""
for char in ('"', '$', '`'):
string = string.replace(char, '\%s' % char)
return string
class _AttributeString(str):
"""
Simple string subclass to allow arbitrary attribute access.
"""
@property
def stdout(self):
return str(self)
class _AttributeList(list):
"""
Like _AttributeString, but for lists.
"""
pass
# Can't wait till Python versions supporting 'def func(*args, foo=bar)' become
# widespread :(
def require(*keys, **kwargs):
"""
Check for given keys in the shared environment dict and abort if not found.
Positional arguments should be strings signifying what env vars should be
checked for. If any of the given arguments do not exist, Fabric will abort
execution and print the names of the missing keys.
The optional keyword argument ``used_for`` may be a string, which will be
printed in the error output to inform users why this requirement is in
place. ``used_for`` is printed as part of a string similar to::
"Th(is|ese) variable(s) (are|is) used for %s"
so format it appropriately.
The optional keyword argument ``provided_by`` may be a list of functions or
function names or a single function or function name which the user should
be able to execute in order to set the key or keys; it will be included in
the error output if requirements are not met.
Note: it is assumed that the keyword arguments apply to all given keys as a
group. If you feel the need to specify more than one ``used_for``, for
example, you should break your logic into multiple calls to ``require()``.
.. versionchanged:: 1.1
Allow iterable ``provided_by`` values instead of just single values.
"""
# If all keys exist, we're good, so keep going.
missing_keys = filter(lambda x: x not in env, keys)
if not missing_keys:
return
# Pluralization
if len(missing_keys) > 1:
variable = "variables were"
used = "These variables are"
else:
variable = "variable was"
used = "This variable is"
# Regardless of kwargs, print what was missing. (Be graceful if used outside
# of a command.)
if 'command' in env:
prefix = "The command '%s' failed because the " % env.command
else:
prefix = "The "
msg = "%sfollowing required environment %s not defined:\n%s" % (
prefix, variable, indent(missing_keys)
)
# Print used_for if given
if 'used_for' in kwargs:
msg += "\n\n%s used for %s" % (used, kwargs['used_for'])
# And print provided_by if given
if 'provided_by' in kwargs:
funcs = kwargs['provided_by']
# non-iterable is given, treat it as a list of this single item
if not hasattr(funcs, '__iter__'):
funcs = [funcs]
if len(funcs) > 1:
command = "one of the following commands"
else:
command = "the following command"
to_s = lambda obj: getattr(obj, '__name__', str(obj))
provided_by = [to_s(obj) for obj in funcs]
msg += "\n\nTry running %s prior to this one, to fix the problem:\n%s"\
% (command, indent(provided_by))
abort(msg)
def prompt(text, key=None, default='', validate=None):
"""
Prompt user with ``text`` and return the input (like ``raw_input``).
A single space character will be appended for convenience, but nothing
else. Thus, you may want to end your prompt text with a question mark or a
colon, e.g. ``prompt("What hostname?")``.
If ``key`` is given, the user's input will be stored as ``env.<key>`` in
addition to being returned by `prompt`. If the key already existed in
``env``, its value will be overwritten and a warning printed to the user.
If ``default`` is given, it is displayed in square brackets and used if the
user enters nothing (i.e. presses Enter without entering any text).
``default`` defaults to the empty string. If non-empty, a space will be
appended, so that a call such as ``prompt("What hostname?",
default="foo")`` would result in a prompt of ``What hostname? [foo]`` (with
a trailing space after the ``[foo]``.)
The optional keyword argument ``validate`` may be a callable or a string:
* If a callable, it is called with the user's input, and should return the
value to be stored on success. On failure, it should raise an exception
with an exception message, which will be printed to the user.
* If a string, the value passed to ``validate`` is used as a regular
expression. It is thus recommended to use raw strings in this case. Note
that the regular expression, if it is not fully matching (bounded by
``^`` and ``$``) it will be made so. In other words, the input must fully
match the regex.
Either way, `prompt` will re-prompt until validation passes (or the user
hits ``Ctrl-C``).
.. note::
`~fabric.operations.prompt` honors :ref:`env.abort_on_prompts
<abort-on-prompts>` and will call `~fabric.utils.abort` instead of
prompting if that flag is set to ``True``. If you want to block on user
input regardless, try wrapping with
`~fabric.context_managers.settings`.
Examples::
# Simplest form:
environment = prompt('Please specify target environment: ')
# With default, and storing as env.dish:
prompt('Specify favorite dish: ', 'dish', default='spam & eggs')
# With validation, i.e. requiring integer input:
prompt('Please specify process nice level: ', key='nice', validate=int)
# With validation against a regular expression:
release = prompt('Please supply a release name',
validate=r'^\w+-\d+(\.\d+)?$')
# Prompt regardless of the global abort-on-prompts setting:
with settings(abort_on_prompts=False):
prompt('I seriously need an answer on this! ')
"""
handle_prompt_abort()
# Store previous env value for later display, if necessary
if key:
previous_value = env.get(key)
# Set up default display
default_str = ""
if default != '':
default_str = " [%s] " % str(default).strip()
else:
default_str = " "
# Construct full prompt string
prompt_str = text.strip() + default_str
# Loop until we pass validation
value = None
while value is None:
# Get input
value = raw_input(prompt_str) or default
# Handle validation
if validate:
# Callable
if callable(validate):
# Callable validate() must raise an exception if validation
# fails.
try:
value = validate(value)
except Exception, e:
# Reset value so we stay in the loop
value = None
print("Validation failed for the following reason:")
print(indent(e.message) + "\n")
# String / regex must match and will be empty if validation fails.
else:
# Need to transform regex into full-matching one if it's not.
if not validate.startswith('^'):
validate = r'^' + validate
if not validate.endswith('$'):
validate += r'$'
result = re.findall(validate, value)
if not result:
print("Regular expression validation failed: '%s' does not match '%s'\n" % (value, validate))
# Reset value so we stay in the loop
value = None
# At this point, value must be valid, so update env if necessary
if key:
env[key] = value
# Print warning if we overwrote some other value
if key and previous_value is not None and previous_value != value:
warn("overwrote previous env variable '%s'; used to be '%s', is now '%s'." % (
key, previous_value, value
))
# And return the value, too, just in case someone finds that useful.
return value
@needs_host
def put(local_path=None, remote_path=None, use_sudo=False,
mirror_local_mode=False, mode=None):
"""
Upload one or more files to a remote host.
`~fabric.operations.put` returns an iterable containing the absolute file
paths of all remote files uploaded. This iterable also exhibits a
``.failed`` attribute containing any local file paths which failed to
upload (and may thus be used as a boolean test.) You may also check
``.succeeded`` which is equivalent to ``not .failed``.
``local_path`` may be a relative or absolute local file or directory path,
and may contain shell-style wildcards, as understood by the Python ``glob``
module. Tilde expansion (as implemented by ``os.path.expanduser``) is also
performed.
``local_path`` may alternately be a file-like object, such as the result of
``open('path')`` or a ``StringIO`` instance.
.. note::
In this case, `~fabric.operations.put` will attempt to read the entire
contents of the file-like object by rewinding it using ``seek`` (and
will use ``tell`` afterwards to preserve the previous file position).
.. note::
Use of a file-like object in `~fabric.operations.put`'s ``local_path``
argument will cause a temporary file to be utilized due to limitations
in our SSH layer's API.
``remote_path`` may also be a relative or absolute location, but applied to
the remote host. Relative paths are relative to the remote user's home
directory, but tilde expansion (e.g. ``~/.ssh/``) will also be performed if
necessary.
An empty string, in either path argument, will be replaced by the
appropriate end's current working directory.
While the SFTP protocol (which `put` uses) has no direct ability to upload
files to locations not owned by the connecting user, you may specify
``use_sudo=True`` to work around this. When set, this setting causes `put`
to upload the local files to a temporary location on the remote end, and
then use `sudo` to move them to ``remote_path``.
In some use cases, it is desirable to force a newly uploaded file to match
the mode of its local counterpart (such as when uploading executable
scripts). To do this, specify ``mirror_local_mode=True``.
Alternately, you may use the ``mode`` kwarg to specify an exact mode, in
the same vein as ``os.chmod`` or the Unix ``chmod`` command.
`~fabric.operations.put` will honor `~fabric.context_managers.cd`, so
relative values in ``remote_path`` will be prepended by the current remote
working directory, if applicable. Thus, for example, the below snippet
would attempt to upload to ``/tmp/files/test.txt`` instead of
``~/files/test.txt``::
with cd('/tmp'):
put('/path/to/local/test.txt', 'files')
Use of `~fabric.context_managers.lcd` will affect ``local_path`` in the
same manner.
Examples::
put('bin/project.zip', '/tmp/project.zip')
put('*.py', 'cgi-bin/')
put('index.html', 'index.html', mode=0755)
.. versionchanged:: 1.0
Now honors the remote working directory as manipulated by
`~fabric.context_managers.cd`, and the local working directory as
manipulated by `~fabric.context_managers.lcd`.
.. versionchanged:: 1.0
Now allows file-like objects in the ``local_path`` argument.
.. versionchanged:: 1.0
Directories may be specified in the ``local_path`` argument and will
trigger recursive uploads.
.. versionchanged:: 1.0
Return value is now an iterable of uploaded remote file paths which
also exhibits the ``.failed`` and ``.succeeded`` attributes.
"""
# Handle empty local path
local_path = local_path or os.getcwd()
# Test whether local_path is a path or a file-like object
local_is_path = not (hasattr(local_path, 'read') \
and callable(local_path.read))
ftp = SFTP(env.host_string)
with closing(ftp) as ftp:
home = ftp.normalize('.')
# Empty remote path implies cwd
remote_path = remote_path or home
# Expand tildes
if remote_path.startswith('~'):
remote_path = remote_path.replace('~', home, 1)
# Honor cd() (assumes Unix style file paths on remote end)
if not os.path.isabs(remote_path) and env.get('cwd'):
remote_path = env.cwd.rstrip('/') + '/' + remote_path
if local_is_path:
# Expand local paths
local_path = os.path.expanduser(local_path)
# Honor lcd() where it makes sense
if not os.path.isabs(local_path) and env.lcwd:
local_path = os.path.join(env.lcwd, local_path)
# Glob local path
names = glob(local_path)
else:
names = [local_path]
# Make sure local arg exists
if local_is_path and not names:
err = "'%s' is not a valid local path or glob." % local_path
raise ValueError(err)
# Sanity check and wierd cases
if ftp.exists(remote_path):
if local_is_path and len(names) != 1 and not ftp.isdir(remote_path):
raise ValueError("'%s' is not a directory" % remote_path)
# Iterate over all given local files
remote_paths = []
failed_local_paths = []
for lpath in names:
try:
if local_is_path and os.path.isdir(lpath):
p = ftp.put_dir(lpath, remote_path, use_sudo,
mirror_local_mode, mode)
remote_paths.extend(p)
else:
p = ftp.put(lpath, remote_path, use_sudo, mirror_local_mode,
mode, local_is_path)
remote_paths.append(p)
except Exception, e:
msg = "put() encountered an exception while uploading '%s'"
failure = lpath if local_is_path else "<StringIO>"
failed_local_paths.append(failure)
_handle_failure(message=msg % lpath, exception=e)
ret = _AttributeList(remote_paths)
ret.failed = failed_local_paths
ret.succeeded = not ret.failed
return ret
@needs_host
def get(remote_path, local_path=None):
"""
Download one or more files from a remote host.
`~fabric.operations.get` returns an iterable containing the absolute paths
to all local files downloaded, which will be empty if ``local_path`` was a
StringIO object (see below for more on using StringIO). This object will
also exhibit a ``.failed`` attribute containing any remote file paths which
failed to download, and a ``.succeeded`` attribute equivalent to ``not
.failed``.
``remote_path`` is the remote file or directory path to download, which may
contain shell glob syntax, e.g. ``"/var/log/apache2/*.log"``, and will have
tildes replaced by the remote home directory. Relative paths will be
considered relative to the remote user's home directory, or the current
remote working directory as manipulated by `~fabric.context_managers.cd`.
If the remote path points to a directory, that directory will be downloaded
recursively.
``local_path`` is the local file path where the downloaded file or files
will be stored. If relative, it will honor the local current working
directory as manipulated by `~fabric.context_managers.lcd`. It may be
interpolated, using standard Python dict-based interpolation, with the
following variables:
* ``host``: The value of ``env.host_string``, eg ``myhostname`` or
``user@myhostname-222`` (the colon between hostname and port is turned
into a dash to maximize filesystem compatibility)
* ``dirname``: The directory part of the remote file path, e.g. the
``src/projectname`` in ``src/projectname/utils.py``.
* ``basename``: The filename part of the remote file path, e.g. the
``utils.py`` in ``src/projectname/utils.py``
* ``path``: The full remote path, e.g. ``src/projectname/utils.py``.
.. note::
When ``remote_path`` is an absolute directory path, only the inner
directories will be recreated locally and passed into the above
variables. So for example, ``get('/var/log', '%(path)s')`` would start
writing out files like ``apache2/access.log``,
``postgresql/8.4/postgresql.log``, etc, in the local working directory.
It would **not** write out e.g. ``var/log/apache2/access.log``.
Additionally, when downloading a single file, ``%(dirname)s`` and
``%(path)s`` do not make as much sense and will be empty and equivalent
to ``%(basename)s``, respectively. Thus a call like
``get('/var/log/apache2/access.log', '%(path)s')`` will save a local
file named ``access.log``, not ``var/log/apache2/access.log``.
This behavior is intended to be consistent with the command-line
``scp`` program.
If left blank, ``local_path`` defaults to ``"%(host)s/%(path)s"`` in order
to be safe for multi-host invocations.
.. warning::
If your ``local_path`` argument does not contain ``%(host)s`` and your
`~fabric.operations.get` call runs against multiple hosts, your local
files will be overwritten on each successive run!
If ``local_path`` does not make use of the above variables (i.e. if it is a
simple, explicit file path) it will act similar to ``scp`` or ``cp``,
overwriting pre-existing files if necessary, downloading into a directory
if given (e.g. ``get('/path/to/remote_file.txt', 'local_directory')`` will
create ``local_directory/remote_file.txt``) and so forth.
``local_path`` may alternately be a file-like object, such as the result of
``open('path', 'w')`` or a ``StringIO`` instance.
.. note::
Attempting to `get` a directory into a file-like object is not valid
and will result in an error.
.. note::
This function will use ``seek`` and ``tell`` to overwrite the entire
contents of the file-like object, in order to be consistent with the
behavior of `~fabric.operations.put` (which also considers the entire
file). However, unlike `~fabric.operations.put`, the file pointer will
not be restored to its previous location, as that doesn't make as much
sense here and/or may not even be possible.
.. note::
Due to how our SSH layer works, a temporary file will still be written
to your hard disk even if you specify a file-like object such as a
StringIO for the ``local_path`` argument. Cleanup is performed,
however -- we just note this for users expecting straight-to-memory
transfers. (We hope to patch our SSH layer in the future to enable true
straight-to-memory downloads.)
.. versionchanged:: 1.0
Now honors the remote working directory as manipulated by
`~fabric.context_managers.cd`, and the local working directory as
manipulated by `~fabric.context_managers.lcd`.
.. versionchanged:: 1.0
Now allows file-like objects in the ``local_path`` argument.
.. versionchanged:: 1.0
``local_path`` may now contain interpolated path- and host-related
variables.
.. versionchanged:: 1.0
Directories may be specified in the ``remote_path`` argument and will
trigger recursive downloads.
.. versionchanged:: 1.0
Return value is now an iterable of downloaded local file paths, which
also exhibits the ``.failed`` and ``.succeeded`` attributes.
"""
# Handle empty local path / default kwarg value
local_path = local_path or "%(host)s/%(path)s"
# Test whether local_path is a path or a file-like object
local_is_path = not (hasattr(local_path, 'write') \
and callable(local_path.write))
# Honor lcd() where it makes sense
if local_is_path and not os.path.isabs(local_path) and env.lcwd:
local_path = os.path.join(env.lcwd, local_path)
ftp = SFTP(env.host_string)
with closing(ftp) as ftp:
home = ftp.normalize('.')
# Expand home directory markers (tildes, etc)
if remote_path.startswith('~'):
remote_path = remote_path.replace('~', home, 1)
if local_is_path:
local_path = os.path.expanduser(local_path)
# Honor cd() (assumes Unix style file paths on remote end)
if not os.path.isabs(remote_path):
# Honor cwd if it's set (usually by with cd():)
if env.get('cwd'):
remote_path = env.cwd.rstrip('/') + '/' + remote_path
# Otherwise, be relative to remote home directory (SFTP server's
# '.')
else:
remote_path = os.path.join(home, remote_path)
# Track final local destination files so we can return a list
local_files = []
failed_remote_files = []
try:
# Glob remote path
names = ftp.glob(remote_path)
# Handle invalid local-file-object situations
if not local_is_path:
if len(names) > 1 or ftp.isdir(names[0]):
_handle_failure("[%s] %s is a glob or directory, but local_path is a file object!" % (env.host_string, remote_path))
for remote_path in names:
if ftp.isdir(remote_path):
result = ftp.get_dir(remote_path, local_path)
local_files.extend(result)
else:
# Result here can be file contents (if not local_is_path)
# or final resultant file path (if local_is_path)
result = ftp.get(remote_path, local_path, local_is_path,
os.path.basename(remote_path))
if not local_is_path:
# Overwrite entire contents of local_path
local_path.seek(0)
local_path.write(result)
else:
local_files.append(result)
except Exception, e:
failed_remote_files.append(remote_path)
msg = "get() encountered an exception while downloading '%s'"
_handle_failure(message=msg % remote_path, exception=e)
ret = _AttributeList(local_files if local_is_path else [])
ret.failed = failed_remote_files
ret.succeeded = not ret.failed
return ret
def _sudo_prefix(user):
"""
Return ``env.sudo_prefix`` with ``user`` inserted if necessary.
"""
# Insert env.sudo_prompt into env.sudo_prefix
prefix = env.sudo_prefix % env.sudo_prompt
if user is not None:
if str(user).isdigit():
user = "#%s" % user
return "%s -u \"%s\" " % (prefix, user)
return prefix
def _shell_wrap(command, shell=True, sudo_prefix=None):
"""
Conditionally wrap given command in env.shell (while honoring sudo.)
"""
# Honor env.shell, while allowing the 'shell' kwarg to override it (at
# least in terms of turning it off.)
if shell and not env.use_shell:
shell = False
# Sudo plus space, or empty string
if sudo_prefix is None:
sudo_prefix = ""
else:
sudo_prefix += " "
# If we're shell wrapping, prefix shell and space, escape the command and
# then quote it. Otherwise, empty string.
if shell:
shell = env.shell + " "
command = '"%s"' % _shell_escape(command)
else:
shell = ""
# Resulting string should now have correct formatting
return sudo_prefix + shell + command
def _prefix_commands(command, which):
"""
Prefixes ``command`` with all prefixes found in ``env.command_prefixes``.
``env.command_prefixes`` is a list of strings which is modified by the
`~fabric.context_managers.prefix` context manager.
This function also handles a special-case prefix, ``cwd``, used by
`~fabric.context_managers.cd`. The ``which`` kwarg should be a string,
``"local"`` or ``"remote"``, which will determine whether ``cwd`` or
``lcwd`` is used.
"""
# Local prefix list (to hold env.command_prefixes + any special cases)
prefixes = list(env.command_prefixes)
# Handle current working directory, which gets its own special case due to
# being a path string that gets grown/shrunk, instead of just a single
# string or lack thereof.
# Also place it at the front of the list, in case user is expecting another
# prefixed command to be "in" the current working directory.
cwd = env.cwd if which == 'remote' else env.lcwd
if cwd:
prefixes.insert(0, 'cd %s' % cwd)
glue = " && "
prefix = (glue.join(prefixes) + glue) if prefixes else ""
return prefix + command
def _prefix_env_vars(command):
"""
Prefixes ``command`` with any shell environment vars, e.g. ``PATH=foo ``.
Currently, this only applies the PATH updating implemented in
`~fabric.context_managers.path`.
"""
# path(): local shell env var update, appending/prepending/replacing $PATH
path = env.path
if path:
if env.path_behavior == 'append':
path = 'PATH=$PATH:\"%s\" ' % path
elif env.path_behavior == 'prepend':
path = 'PATH=\"%s\":$PATH ' % path
elif env.path_behavior == 'replace':
path = 'PATH=\"%s\" ' % path
else:
path = ''
return path + command
def _execute(channel, command, pty=True, combine_stderr=None,
invoke_shell=False):
"""
Execute ``command`` over ``channel``.
``pty`` controls whether a pseudo-terminal is created.
``combine_stderr`` controls whether we call ``channel.set_combine_stderr``.
By default, the global setting for this behavior (:ref:`env.combine_stderr
<combine-stderr>`) is consulted, but you may specify ``True`` or ``False``
here to override it.
``invoke_shell`` controls whether we use ``exec_command`` or
``invoke_shell`` (plus a handful of other things, such as always forcing a
pty.)
Returns a three-tuple of (``stdout``, ``stderr``, ``status``), where
``stdout``/``stderr`` are captured output strings and ``status`` is the
program's return code, if applicable.
"""
with char_buffered(sys.stdin):
# Combine stdout and stderr to get around oddball mixing issues
if combine_stderr is None:
combine_stderr = env.combine_stderr
channel.set_combine_stderr(combine_stderr)
# Assume pty use, and allow overriding of this either via kwarg or env
# var. (invoke_shell always wants a pty no matter what.)
using_pty = True
if not invoke_shell and (not pty or not env.always_use_pty):
using_pty = False
# Request pty with size params (default to 80x24, obtain real
# parameters if on POSIX platform)
if using_pty:
rows, cols = _pty_size()
channel.get_pty(width=cols, height=rows)
# Kick off remote command
if invoke_shell:
channel.invoke_shell()
if command:
channel.sendall(command + "\n")
else:
channel.exec_command(command)
# Init stdout, stderr capturing. Must use lists instead of strings as
# strings are immutable and we're using these as pass-by-reference
stdout, stderr = [], []
if invoke_shell:
stdout = stderr = None
workers = (
ThreadHandler('out', output_loop, channel, "recv", stdout),
ThreadHandler('err', output_loop, channel, "recv_stderr", stderr),
ThreadHandler('in', input_loop, channel, using_pty)
)
while True:
if channel.exit_status_ready():
break
else:
for worker in workers:
e = worker.exception
if e:
raise e[0], e[1], e[2]
time.sleep(io_sleep)
# Obtain exit code of remote program now that we're done.
status = channel.recv_exit_status()
# Wait for threads to exit so we aren't left with stale threads
for worker in workers:
worker.thread.join()
# Close channel
channel.close()
# Update stdout/stderr with captured values if applicable
if not invoke_shell:
stdout = ''.join(stdout).strip()
stderr = ''.join(stderr).strip()
# Tie off "loose" output by printing a newline. Helps to ensure any
# following print()s aren't on the same line as a trailing line prefix
# or similar. However, don't add an extra newline if we've already
# ended up with one, as that adds a entire blank line instead.
if output.running \
and (output.stdout and stdout and not stdout.endswith("\n")) \
or (output.stderr and stderr and not stderr.endswith("\n")):
print("")
return stdout, stderr, status
@needs_host
def open_shell(command=None):
"""
Invoke a fully interactive shell on the remote end.
If ``command`` is given, it will be sent down the pipe before handing
control over to the invoking user.
This function is most useful for when you need to interact with a heavily
shell-based command or series of commands, such as when debugging or when
fully interactive recovery is required upon remote program failure.
It should be considered an easy way to work an interactive shell session
into the middle of a Fabric script and is *not* a drop-in replacement for
`~fabric.operations.run`, which is also capable of interacting with the
remote end (albeit only while its given command is executing) and has much
stronger programmatic abilities such as error handling and stdout/stderr
capture.
Specifically, `~fabric.operations.open_shell` provides a better interactive
experience than `~fabric.operations.run`, but use of a full remote shell
prevents Fabric from determining whether programs run within the shell have
failed, and pollutes the stdout/stderr stream with shell output such as
login banners, prompts and echoed stdin.
Thus, this function does not have a return value and will not trigger
Fabric's failure handling if any remote programs result in errors.
.. versionadded:: 1.0
"""
_execute(default_channel(), command, True, True, True)
def _run_command(command, shell=True, pty=True, combine_stderr=True,
sudo=False, user=None):
"""
Underpinnings of `run` and `sudo`. See their docstrings for more info.
"""
# Set up new var so original argument can be displayed verbatim later.
given_command = command
# Handle context manager modifications, and shell wrapping
wrapped_command = _shell_wrap(
_prefix_commands(_prefix_env_vars(command), 'remote'),
shell,
_sudo_prefix(user) if sudo else None
)
# Execute info line
which = 'sudo' if sudo else 'run'
if output.debug:
print("[%s] %s: %s" % (env.host_string, which, wrapped_command))
elif output.running:
print("[%s] %s: %s" % (env.host_string, which, given_command))
# Actual execution, stdin/stdout/stderr handling, and termination
stdout, stderr, status = _execute(default_channel(), wrapped_command, pty,
combine_stderr)
# Assemble output string
out = _AttributeString(stdout)
err = _AttributeString(stderr)
# Error handling
out.failed = False
if status != 0:
out.failed = True
msg = "%s() encountered an error (return code %s) while executing '%s'" % (which, status, command)
_handle_failure(message=msg)
# Attach return code to output string so users who have set things to
# warn only, can inspect the error code.
out.return_code = status
# Convenience mirror of .failed
out.succeeded = not out.failed
# Attach stderr for anyone interested in that.
out.stderr = err
return out
@needs_host
def run(command, shell=True, pty=True, combine_stderr=None):
"""
Run a shell command on a remote host.
If ``shell`` is True (the default), `run` will execute the given command
string via a shell interpreter, the value of which may be controlled by
setting ``env.shell`` (defaulting to something similar to ``/bin/bash -l -c
"<command>"``.) Any double-quote (``"``) or dollar-sign (``$``) characters
in ``command`` will be automatically escaped when ``shell`` is True.
`run` will return the result of the remote program's stdout as a single
(likely multiline) string. This string will exhibit ``failed`` and
``succeeded`` boolean attributes specifying whether the command failed or
succeeded, and will also include the return code as the ``return_code``
attribute.
Any text entered in your local terminal will be forwarded to the remote
program as it runs, thus allowing you to interact with password or other
prompts naturally. For more on how this works, see
:doc:`/usage/interactivity`.
You may pass ``pty=False`` to forego creation of a pseudo-terminal on the
remote end in case the presence of one causes problems for the command in
question. However, this will force Fabric itself to echo any and all input
you type while the command is running, including sensitive passwords. (With
``pty=True``, the remote pseudo-terminal will echo for you, and will
intelligently handle password-style prompts.) See :ref:`pseudottys` for
details.
Similarly, if you need to programmatically examine the stderr stream of the
remote program (exhibited as the ``stderr`` attribute on this function's
return value), you may set ``combine_stderr=False``. Doing so has a high
chance of causing garbled output to appear on your terminal (though the
resulting strings returned by `~fabric.operations.run` will be properly
separated). For more info, please read :ref:`combine_streams`.
Examples::
run("ls /var/www/")
run("ls /home/myuser", shell=False)
output = run('ls /var/www/site1')
.. versionadded:: 1.0
The ``succeeded`` and ``stderr`` return value attributes, the
``combine_stderr`` kwarg, and interactive behavior.
.. versionchanged:: 1.0
The default value of ``pty`` is now ``True``.
.. versionchanged:: 1.0.2
The default value of ``combine_stderr`` is now ``None`` instead of
``True``. However, the default *behavior* is unchanged, as the global
setting is still ``True``.
"""
return _run_command(command, shell, pty, combine_stderr)
@needs_host
def sudo(command, shell=True, pty=True, combine_stderr=None, user=None):
"""
Run a shell command on a remote host, with superuser privileges.
`sudo` is identical in every way to `run`, except that it will always wrap
the given ``command`` in a call to the ``sudo`` program to provide
superuser privileges.
`sudo` accepts an additional ``user`` argument, which is passed to ``sudo``
and allows you to run as some user other than root. On most systems, the
``sudo`` program can take a string username or an integer userid (uid);
``user`` may likewise be a string or an int.
Examples::
sudo("~/install_script.py")
sudo("mkdir /var/www/new_docroot", user="www-data")
sudo("ls /home/jdoe", user=1001)
result = sudo("ls /tmp/")
.. versionchanged:: 1.0
See the changed and added notes for `~fabric.operations.run`.
"""
return _run_command(command, shell, pty, combine_stderr, sudo=True,
user=user)
def local(command, capture=False):
"""
Run a command on the local system.
`local` is simply a convenience wrapper around the use of the builtin
Python ``subprocess`` module with ``shell=True`` activated. If you need to
do anything special, consider using the ``subprocess`` module directly.
`local` is not currently capable of simultaneously printing and
capturing output, as `~fabric.operations.run`/`~fabric.operations.sudo`
do. The ``capture`` kwarg allows you to switch between printing and
capturing as necessary, and defaults to ``False``.
When ``capture=False``, the local subprocess' stdout and stderr streams are
hooked up directly to your terminal, though you may use the global
:doc:`output controls </usage/output_controls>` ``output.stdout`` and
``output.stderr`` to hide one or both if desired. In this mode,
`~fabric.operations.local` returns None.
When ``capture=True``, this function will return the contents of the
command's stdout as a string-like object; as with `~fabric.operations.run`
and `~fabric.operations.sudo`, this return value exhibits the