file.py

# -*- coding: utf-8 -*-
'''
Manage information about regular files, directories,
and special files on the minion, set/read user,
group, mode, and data
'''

# TODO: We should add the capability to do u+r type operations here
# some time in the future

from __future__ import absolute_import, print_function, unicode_literals

# Import python libs
import datetime
import difflib
import errno
import fnmatch
import io
import itertools
import logging
import operator
import os
import re
import shutil
import stat
import string
import sys
import tempfile
import time
import glob
import hashlib
import mmap
from collections import Iterable, Mapping
from functools import reduce  # pylint: disable=redefined-builtin

# pylint: disable=import-error,no-name-in-module,redefined-builtin
from salt.ext import six
from salt.ext.six.moves import range, zip
from salt.ext.six.moves.urllib.parse import urlparse as _urlparse
# pylint: enable=import-error,no-name-in-module,redefined-builtin

try:
    import grp
    import pwd
except ImportError:
    pass

# Import salt libs
import salt.utils.args
import salt.utils.atomicfile
import salt.utils.filebuffer
import salt.utils.files
import salt.utils.find
import salt.utils.functools
import salt.utils.hashutils
import salt.utils.itertools
import salt.utils.locales
import salt.utils.path
import salt.utils.platform
import salt.utils.stringutils
import salt.utils.templates
import salt.utils.url
import salt.utils.user
import salt.utils.data
from salt.exceptions import CommandExecutionError, MinionError, SaltInvocationError, get_error_message as _get_error_message
from salt.utils.files import HASHES, HASHES_REVMAP

log = logging.getLogger(__name__)

__func_alias__ = {
    'makedirs_': 'makedirs'
}


def __virtual__():
    '''
    Only work on POSIX-like systems
    '''
    # win_file takes care of windows
    if salt.utils.platform.is_windows():
        return (
            False,
            'The file execution module cannot be loaded: only available on '
            'non-Windows systems - use win_file instead.'
        )
    return True


def __clean_tmp(sfn):
    '''
    Clean out a template temp file
    '''
    if sfn.startswith(os.path.join(tempfile.gettempdir(),
                                   salt.utils.files.TEMPFILE_PREFIX)):
        # Don't remove if it exists in file_roots (any saltenv)
        all_roots = itertools.chain.from_iterable(
                six.itervalues(__opts__['file_roots']))
        in_roots = any(sfn.startswith(root) for root in all_roots)
        # Only clean up files that exist
        if os.path.exists(sfn) and not in_roots:
            os.remove(sfn)


def _error(ret, err_msg):
    '''
    Common function for setting error information for return dicts
    '''
    ret['result'] = False
    ret['comment'] = err_msg
    return ret


def _binary_replace(old, new):
    '''
    This function does NOT do any diffing, it just checks the old and new files
    to see if either is binary, and provides an appropriate string noting the
    difference between the two files. If neither file is binary, an empty
    string is returned.

    This function should only be run AFTER it has been determined that the
    files differ.
    '''
    old_isbin = not __utils__['files.is_text'](old)
    new_isbin = not __utils__['files.is_text'](new)
    if any((old_isbin, new_isbin)):
        if all((old_isbin, new_isbin)):
            return 'Replace binary file'
        elif old_isbin:
            return 'Replace binary file with text file'
        elif new_isbin:
            return 'Replace text file with binary file'
    return ''


def _get_bkroot():
    '''
    Get the location of the backup dir in the minion cache
    '''
    # Get the cachedir from the minion config
    return os.path.join(__salt__['config.get']('cachedir'), 'file_backup')


def _splitlines_preserving_trailing_newline(str):
    '''
    Returns a list of the lines in the string, breaking at line boundaries and
    preserving a trailing newline (if present).

    Essentially, this works like ``str.striplines(False)`` but preserves an
    empty line at the end. This is equivalent to the following code:

    .. code-block:: python

        lines = str.splitlines()
        if str.endswith('\n') or str.endswith('\r'):
            lines.append('')
    '''
    lines = str.splitlines()
    if str.endswith('\n') or str.endswith('\r'):
        lines.append('')
    return lines


def gid_to_group(gid):
    '''
    Convert the group id to the group name on this system

    gid
        gid to convert to a group name

    CLI Example:

    .. code-block:: bash

        salt '*' file.gid_to_group 0
    '''
    try:
        gid = int(gid)
    except ValueError:
        # This is not an integer, maybe it's already the group name?
        gid = group_to_gid(gid)

    if gid == '':
        # Don't even bother to feed it to grp
        return ''

    try:
        return grp.getgrgid(gid).gr_name
    except (KeyError, NameError):
        # If group is not present, fall back to the gid.
        return gid


def group_to_gid(group):
    '''
    Convert the group to the gid on this system

    group
        group to convert to its gid

    CLI Example:

    .. code-block:: bash

        salt '*' file.group_to_gid root
    '''
    if group is None:
        return ''
    try:
        if isinstance(group, int):
            return group
        return grp.getgrnam(group).gr_gid
    except KeyError:
        return ''


def get_gid(path, follow_symlinks=True):
    '''
    Return the id of the group that owns a given file

    path
        file or directory of which to get the gid

    follow_symlinks
        indicated if symlinks should be followed


    CLI Example:

    .. code-block:: bash

        salt '*' file.get_gid /etc/passwd

    .. versionchanged:: 0.16.4
        ``follow_symlinks`` option added
    '''
    return stats(os.path.expanduser(path), follow_symlinks=follow_symlinks).get('gid', -1)


def get_group(path, follow_symlinks=True):
    '''
    Return the group that owns a given file

    path
        file or directory of which to get the group

    follow_symlinks
        indicated if symlinks should be followed

    CLI Example:

    .. code-block:: bash

        salt '*' file.get_group /etc/passwd

    .. versionchanged:: 0.16.4
        ``follow_symlinks`` option added
    '''
    return stats(os.path.expanduser(path), follow_symlinks=follow_symlinks).get('group', False)


def uid_to_user(uid):
    '''
    Convert a uid to a user name

    uid
        uid to convert to a username

    CLI Example:

    .. code-block:: bash

        salt '*' file.uid_to_user 0
    '''
    try:
        return pwd.getpwuid(uid).pw_name
    except (KeyError, NameError):
        # If user is not present, fall back to the uid.
        return uid


def user_to_uid(user):
    '''
    Convert user name to a uid

    user
        user name to convert to its uid

    CLI Example:

    .. code-block:: bash

        salt '*' file.user_to_uid root
    '''
    if user is None:
        user = salt.utils.user.get_user()
    try:
        if isinstance(user, int):
            return user
        return pwd.getpwnam(user).pw_uid
    except KeyError:
        return ''


def get_uid(path, follow_symlinks=True):
    '''
    Return the id of the user that owns a given file

    path
        file or directory of which to get the uid

    follow_symlinks
        indicated if symlinks should be followed

    CLI Example:

    .. code-block:: bash

        salt '*' file.get_uid /etc/passwd

    .. versionchanged:: 0.16.4
        ``follow_symlinks`` option added
    '''
    return stats(os.path.expanduser(path), follow_symlinks=follow_symlinks).get('uid', -1)


def get_user(path, follow_symlinks=True):
    '''
    Return the user that owns a given file

    path
        file or directory of which to get the user

    follow_symlinks
        indicated if symlinks should be followed

    CLI Example:

    .. code-block:: bash

        salt '*' file.get_user /etc/passwd

    .. versionchanged:: 0.16.4
        ``follow_symlinks`` option added
    '''
    return stats(os.path.expanduser(path), follow_symlinks=follow_symlinks).get('user', False)


def get_mode(path, follow_symlinks=True):
    '''
    Return the mode of a file

    path
        file or directory of which to get the mode

    follow_symlinks
        indicated if symlinks should be followed

    CLI Example:

    .. code-block:: bash

        salt '*' file.get_mode /etc/passwd

    .. versionchanged:: 2014.1.0
        ``follow_symlinks`` option added
    '''
    return stats(os.path.expanduser(path), follow_symlinks=follow_symlinks).get('mode', '')


def set_mode(path, mode):
    '''
    Set the mode of a file

    path
        file or directory of which to set the mode

    mode
        mode to set the path to

    CLI Example:

    .. code-block:: bash

        salt '*' file.set_mode /etc/passwd 0644
    '''
    path = os.path.expanduser(path)

    mode = six.text_type(mode).lstrip('0Oo')
    if not mode:
        mode = '0'
    if not os.path.exists(path):
        raise CommandExecutionError('{0}: File not found'.format(path))
    try:
        os.chmod(path, int(mode, 8))
    except Exception:
        return 'Invalid Mode ' + mode
    return get_mode(path)


def lchown(path, user, group):
    '''
    Chown a file, pass the file the desired user and group without following
    symlinks.

    path
        path to the file or directory

    user
        user owner

    group
        group owner

    CLI Example:

    .. code-block:: bash

        salt '*' file.chown /etc/passwd root root
    '''
    path = os.path.expanduser(path)

    uid = user_to_uid(user)
    gid = group_to_gid(group)
    err = ''
    if uid == '':
        if user:
            err += 'User does not exist\n'
        else:
            uid = -1
    if gid == '':
        if group:
            err += 'Group does not exist\n'
        else:
            gid = -1

    return os.lchown(path, uid, gid)


def chown(path, user, group):
    '''
    Chown a file, pass the file the desired user and group

    path
        path to the file or directory

    user
        user owner

    group
        group owner

    CLI Example:

    .. code-block:: bash

        salt '*' file.chown /etc/passwd root root
    '''
    path = os.path.expanduser(path)

    uid = user_to_uid(user)
    gid = group_to_gid(group)
    err = ''
    if uid == '':
        if user:
            err += 'User does not exist\n'
        else:
            uid = -1
    if gid == '':
        if group:
            err += 'Group does not exist\n'
        else:
            gid = -1
    if not os.path.exists(path):
        try:
            # Broken symlinks will return false, but still need to be chowned
            return os.lchown(path, uid, gid)
        except OSError:
            pass
        err += 'File not found'
    if err:
        return err
    return os.chown(path, uid, gid)


def chgrp(path, group):
    '''
    Change the group of a file

    path
        path to the file or directory

    group
        group owner

    CLI Example:

    .. code-block:: bash

        salt '*' file.chgrp /etc/passwd root
    '''
    path = os.path.expanduser(path)

    user = get_user(path)
    return chown(path, user, group)


def _cmp_attrs(path, attrs):
    '''
    .. versionadded:: 2018.3.0

    Compare attributes of a given file to given attributes.
    Returns a pair (list) where first item are attributes to
    add and second item are to be removed.

    Please take into account when using this function that some minions will
    not have lsattr installed.

    path
        path to file to compare attributes with.

    attrs
        string of attributes to compare against a given file
    '''
    diff = [None, None]

    try:
        lattrs = lsattr(path).get(path, '')
    except AttributeError:
        # lsattr not installed
        return None

    old = [chr for chr in lattrs if chr not in attrs]
    if len(old) > 0:
        diff[1] = ''.join(old)

    new = [chr for chr in attrs if chr not in lattrs]
    if len(new) > 0:
        diff[0] = ''.join(new)

    return diff


def lsattr(path):
    '''
    .. versionadded:: 2018.3.0
    .. versionchanged:: 2018.3.1
        If ``lsattr`` is not installed on the system, ``None`` is returned.

    Obtain the modifiable attributes of the given file. If path
    is to a directory, an empty list is returned.

    path
        path to file to obtain attributes of. File/directory must exist.

    CLI Example:

    .. code-block:: bash

        salt '*' file.lsattr foo1.txt
    '''
    if not salt.utils.path.which('lsattr'):
        return None

    if not os.path.exists(path):
        raise SaltInvocationError("File or directory does not exist.")

    cmd = ['lsattr', path]
    result = __salt__['cmd.run'](cmd, ignore_retcode=True, python_shell=False)

    results = {}
    for line in result.splitlines():
        if not line.startswith('lsattr: '):
            vals = line.split(None, 1)
            results[vals[1]] = re.findall(r"[acdijstuADST]", vals[0])

    return results


def chattr(*files, **kwargs):
    '''
    .. versionadded:: 2018.3.0

    Change the attributes of files. This function accepts one or more files and
    the following options:

    operator
        Can be wither ``add`` or ``remove``. Determines whether attributes
        should be added or removed from files

    attributes
        One or more of the following characters: ``acdijstuADST``, representing
        attributes to add to/remove from files

    version
        a version number to assign to the file(s)

    flags
        One or more of the following characters: ``RVf``, representing
        flags to assign to chattr (recurse, verbose, suppress most errors)

    CLI Example:

    .. code-block:: bash

        salt '*' file.chattr foo1.txt foo2.txt operator=add attributes=ai
        salt '*' file.chattr foo3.txt operator=remove attributes=i version=2
    '''
    operator = kwargs.pop('operator', None)
    attributes = kwargs.pop('attributes', None)
    flags = kwargs.pop('flags', None)
    version = kwargs.pop('version', None)

    if (operator is None) or (operator not in ('add', 'remove')):
        raise SaltInvocationError(
            "Need an operator: 'add' or 'remove' to modify attributes.")
    if attributes is None:
        raise SaltInvocationError("Need attributes: [AacDdijsTtSu]")

    cmd = ['chattr']

    if operator == "add":
        attrs = '+{0}'.format(attributes)
    elif operator == "remove":
        attrs = '-{0}'.format(attributes)

    cmd.append(attrs)

    if flags is not None:
        cmd.append('-{0}'.format(flags))

    if version is not None:
        cmd.extend(['-v', version])

    cmd.extend(files)

    result = __salt__['cmd.run'](cmd, python_shell=False)

    if bool(result):
        raise CommandExecutionError(
            "chattr failed to run, possibly due to bad parameters.")

    return True


def get_sum(path, form='sha256'):
    '''
    Return the checksum for the given file. The following checksum algorithms
    are supported:

    * md5
    * sha1
    * sha224
    * sha256 **(default)**
    * sha384
    * sha512

    path
        path to the file or directory

    form
        desired sum format

    CLI Example:

    .. code-block:: bash

        salt '*' file.get_sum /etc/passwd sha512
    '''
    path = os.path.expanduser(path)

    if not os.path.isfile(path):
        return 'File not found'
    return salt.utils.hashutils.get_hash(path, form, 4096)


def get_hash(path, form='sha256', chunk_size=65536):
    '''
    Get the hash sum of a file

    This is better than ``get_sum`` for the following reasons:
        - It does not read the entire file into memory.
        - It does not return a string on error. The returned value of
            ``get_sum`` cannot really be trusted since it is vulnerable to
            collisions: ``get_sum(..., 'xyz') == 'Hash xyz not supported'``

    path
        path to the file or directory

    form
        desired sum format

    chunk_size
        amount to sum at once

    CLI Example:

    .. code-block:: bash

        salt '*' file.get_hash /etc/shadow
    '''
    return salt.utils.hashutils.get_hash(os.path.expanduser(path), form, chunk_size)


def get_source_sum(file_name='',
                   source='',
                   source_hash=None,
                   source_hash_name=None,
                   saltenv='base'):
    '''
    .. versionadded:: 2016.11.0

    Used by :py:func:`file.get_managed <salt.modules.file.get_managed>` to
    obtain the hash and hash type from the parameters specified below.

    file_name
        Optional file name being managed, for matching with
        :py:func:`file.extract_hash <salt.modules.file.extract_hash>`.

    source
        Source file, as used in :py:mod:`file <salt.states.file>` and other
        states. If ``source_hash`` refers to a file containing hashes, then
        this filename will be used to match a filename in that file. If the
        ``source_hash`` is a hash expression, then this argument will be
        ignored.

    source_hash
        Hash file/expression, as used in :py:mod:`file <salt.states.file>` and
        other states. If this value refers to a remote URL or absolute path to
        a local file, it will be cached and :py:func:`file.extract_hash
        <salt.modules.file.extract_hash>` will be used to obtain a hash from
        it.

    source_hash_name
        Specific file name to look for when ``source_hash`` refers to a remote
        file, used to disambiguate ambiguous matches.

    saltenv : base
        Salt fileserver environment from which to retrieve the source_hash. This
        value will only be used when ``source_hash`` refers to a file on the
        Salt fileserver (i.e. one beginning with ``salt://``).

    CLI Example:

    .. code-block:: bash

        salt '*' file.get_source_sum /tmp/foo.tar.gz source=http://mydomain.tld/foo.tar.gz source_hash=499ae16dcae71eeb7c3a30c75ea7a1a6
        salt '*' file.get_source_sum /tmp/foo.tar.gz source=http://mydomain.tld/foo.tar.gz source_hash=https://mydomain.tld/hashes.md5
        salt '*' file.get_source_sum /tmp/foo.tar.gz source=http://mydomain.tld/foo.tar.gz source_hash=https://mydomain.tld/hashes.md5 source_hash_name=./dir2/foo.tar.gz
    '''
    def _invalid_source_hash_format():
        '''
        DRY helper for reporting invalid source_hash input
        '''
        raise CommandExecutionError(
            'Source hash {0} format is invalid. The supported formats are: '
            '1) a hash, 2) an expression in the format <hash_type>=<hash>, or '
            '3) either a path to a local file containing hashes, or a URI of '
            'a remote hash file. Supported protocols for remote hash files '
            'are: {1}. The hash may also not be of a valid length, the '
            'following are supported hash types and lengths: {2}.'.format(
                source_hash,
                ', '.join(salt.utils.files.VALID_PROTOS),
                ', '.join(
                    ['{0} ({1})'.format(HASHES_REVMAP[x], x)
                     for x in sorted(HASHES_REVMAP)]
                ),
            )
        )

    hash_fn = None
    if os.path.isabs(source_hash):
        hash_fn = source_hash
    else:
        try:
            proto = _urlparse(source_hash).scheme
            if proto in salt.utils.files.VALID_PROTOS:
                hash_fn = __salt__['cp.cache_file'](source_hash, saltenv)
                if not hash_fn:
                    raise CommandExecutionError(
                        'Source hash file {0} not found'.format(source_hash)
                    )
            else:
                if proto != '':
                    # Some unsupported protocol (e.g. foo://) is being used.
                    # We'll get into this else block if a hash expression
                    # (like md5=<md5 checksum here>), but in those cases, the
                    # protocol will be an empty string, in which case we avoid
                    # this error condition.
                    _invalid_source_hash_format()
        except (AttributeError, TypeError):
            _invalid_source_hash_format()

    if hash_fn is not None:
        ret = extract_hash(hash_fn, '', file_name, source, source_hash_name)
        if ret is None:
            _invalid_source_hash_format()
        return ret
    else:
        # The source_hash is a hash expression
        ret = {}
        try:
            ret['hash_type'], ret['hsum'] = \
                [x.strip() for x in source_hash.split('=', 1)]
        except AttributeError:
            _invalid_source_hash_format()
        except ValueError:
            # No hash type, try to figure out by hash length
            if not re.match('^[{0}]+$'.format(string.hexdigits), source_hash):
                _invalid_source_hash_format()
            ret['hsum'] = source_hash
            source_hash_len = len(source_hash)
            if source_hash_len in HASHES_REVMAP:
                ret['hash_type'] = HASHES_REVMAP[source_hash_len]
            else:
                _invalid_source_hash_format()

        if ret['hash_type'] not in HASHES:
            raise CommandExecutionError(
                'Invalid hash type \'{0}\'. Supported hash types are: {1}. '
                'Either remove the hash type and simply use \'{2}\' as the '
                'source_hash, or change the hash type to a supported type.'
                .format(ret['hash_type'], ', '.join(HASHES), ret['hsum'])
            )
        else:
            hsum_len = len(ret['hsum'])
            if hsum_len not in HASHES_REVMAP:
                _invalid_source_hash_format()
            elif hsum_len != HASHES[ret['hash_type']]:
                raise CommandExecutionError(
                    'Invalid length ({0}) for hash type \'{1}\'. Either '
                    'remove the hash type and simply use \'{2}\' as the '
                    'source_hash, or change the hash type to \'{3}\''.format(
                        hsum_len,
                        ret['hash_type'],
                        ret['hsum'],
                        HASHES_REVMAP[hsum_len],
                    )
                )

        return ret


def check_hash(path, file_hash):
    '''
    Check if a file matches the given hash string

    Returns ``True`` if the hash matches, otherwise ``False``.

    path
        Path to a file local to the minion.

    hash
        The hash to check against the file specified in the ``path`` argument.

        .. versionchanged:: 2016.11.4

        For this and newer versions the hash can be specified without an
        accompanying hash type (e.g. ``e138491e9d5b97023cea823fe17bac22``),
        but for earlier releases it is necessary to also specify the hash type
        in the format ``<hash_type>=<hash_value>`` (e.g.
        ``md5=e138491e9d5b97023cea823fe17bac22``).

    CLI Example:

    .. code-block:: bash

        salt '*' file.check_hash /etc/fstab e138491e9d5b97023cea823fe17bac22
        salt '*' file.check_hash /etc/fstab md5=e138491e9d5b97023cea823fe17bac22
    '''
    path = os.path.expanduser(path)

    if not isinstance(file_hash, six.string_types):
        raise SaltInvocationError('hash must be a string')

    for sep in (':', '='):
        if sep in file_hash:
            hash_type, hash_value = file_hash.split(sep, 1)
            break
    else:
        hash_value = file_hash
        hash_len = len(file_hash)
        hash_type = HASHES_REVMAP.get(hash_len)
        if hash_type is None:
            raise SaltInvocationError(
                'Hash {0} (length: {1}) could not be matched to a supported '
                'hash type. The supported hash types and lengths are: '
                '{2}'.format(
                    file_hash,
                    hash_len,
                    ', '.join(
                        ['{0} ({1})'.format(HASHES_REVMAP[x], x)
                         for x in sorted(HASHES_REVMAP)]
                    ),
                )
            )

    return get_hash(path, hash_type) == hash_value


def find(path, *args, **kwargs):
    '''
    Approximate the Unix ``find(1)`` command and return a list of paths that
    meet the specified criteria.

    The options include match criteria:

    .. code-block:: text

        name    = path-glob                 # case sensitive
        iname   = path-glob                 # case insensitive
        regex   = path-regex                # case sensitive
        iregex  = path-regex                # case insensitive
        type    = file-types                # match any listed type
        user    = users                     # match any listed user
        group   = groups                    # match any listed group
        size    = [+-]number[size-unit]     # default unit = byte
        mtime   = interval                  # modified since date
        grep    = regex                     # search file contents

    and/or actions:

    .. code-block:: text

        delete [= file-types]               # default type = 'f'
        exec    = command [arg ...]         # where {} is replaced by pathname
        print  [= print-opts]

    and/or depth criteria:

    .. code-block:: text

        maxdepth = maximum depth to transverse in path
        mindepth = minimum depth to transverse before checking files or directories

    The default action is ``print=path``

    ``path-glob``:

    .. code-block:: text

        *                = match zero or more chars
        ?                = match any char
        [abc]            = match a, b, or c
        [!abc] or [^abc] = match anything except a, b, and c
        [x-y]            = match chars x through y
        [!x-y] or [^x-y] = match anything except chars x through y
        {a,b,c}          = match a or b or c

    ``path-regex``: a Python Regex (regular expression) pattern to match pathnames

    ``file-types``: a string of one or more of the following:

    .. code-block:: text

        a: all file types
        b: block device
        c: character device
        d: directory
        p: FIFO (named pipe)
        f: plain file
        l: symlink
        s: socket

    ``users``: a space and/or comma separated list of user names and/or uids

    ``groups``: a space and/or comma separated list of group names and/or gids

    ``size-unit``:

    .. code-block:: text

        b: bytes
        k: kilobytes
        m: megabytes
        g: gigabytes
        t: terabytes

    interval:

    .. code-block:: text

        [<num>w] [<num>d] [<num>h] [<num>m] [<num>s]

        where:
            w: week
            d: day
            h: hour
            m: minute
            s: second

    print-opts: a comma and/or space separated list of one or more of the
    following:

    .. code-block:: text

        group: group name
        md5:   MD5 digest of file contents
        mode:  file permissions (as integer)
        mtime: last modification time (as time_t)
        name:  file basename
        path:  file absolute path
        size:  file size in bytes
        type:  file type
        user:  user name

    CLI Examples:

    .. code-block:: bash

        salt '*' file.find / type=f name=\\*.bak size=+10m
        salt '*' file.find /var mtime=+30d size=+10m print=path,size,mtime
        salt '*' file.find /var/log name=\\*.[0-9] mtime=+30d size=+10m delete
    '''
    if 'delete' in args:
        kwargs['delete'] = 'f'
    elif 'print' in args:
        kwargs['print'] = 'path'

    try:
        finder = salt.utils.find.Finder(kwargs)
    except ValueError as ex:
        return 'error: {0}'.format(ex)

    ret = [item for i in [finder.find(p) for p in glob.glob(os.path.expanduser(path))] for item in i]
    ret.sort()
    return ret


def _sed_esc(string, escape_all=False):
    '''
    Escape single quotes and forward slashes
    '''
    special_chars = "^.[$()|*+?{"
    string = string.replace("'", "'\"'\"'").replace("/", "\\/")
    if escape_all is True:
        for char in special_chars:
            string = string.replace(char, "\\" + char)
    return string


def sed(path,
        before,
        after,
        limit='',
        backup='.bak',
        options='-r -e',
        flags='g',
        escape_all=False,
        negate_match=False):
    '''
    .. deprecated:: 0.17.0
       Use :py:func:`~salt.modules.file.replace` instead.

    Make a simple edit to a file

    Equivalent to:

    .. code-block:: bash

        sed <backup> <options> "/<limit>/ s/<before>/<after>/<flags> <file>"

    path
        The full path to the file to be edited
    before
        A pattern to find in order to replace with ``after``
    after
        Text that will replace ``before``
    limit : ``''``
        An initial pattern to search for before searching for ``before``
    backup : ``.bak``
        The file will be backed up before edit with this file extension;
        **WARNING:** each time ``sed``/``comment``/``uncomment`` is called will
        overwrite this backup
    options : ``-r -e``
        Options to pass to sed
    flags : ``g``
        Flags to modify the sed search; e.g., ``i`` for case-insensitive pattern
        matching
    negate_match : False
        Negate the search command (``!``)

        .. versionadded:: 0.17.0

    Forward slashes and single quotes will be escaped automatically in the
    ``before`` and ``after`` patterns.

    CLI Example:

    .. code-block:: bash

        salt '*' file.sed /etc/httpd/httpd.conf 'LogLevel warn' 'LogLevel info'
    '''
    # Largely inspired by Fabric's contrib.files.sed()
    # XXX:dc: Do we really want to always force escaping?
    #
    path = os.path.expanduser(path)

    if not os.path.exists(path):
        return False

    # Mandate that before and after are strings
    before = six.text_type(before)
    after = six.text_type(after)
    before = _sed_esc(before, escape_all)
    after = _sed_esc(after, escape_all)
    limit = _sed_esc(limit, escape_all)
    if sys.platform == 'darwin':
        options = options.replace('-r', '-E')

    cmd = ['sed']
    cmd.append('-i{0}'.format(backup) if backup else '-i')
    cmd.extend(salt.utils.args.shlex_split(options))
    cmd.append(
        r'{limit}{negate_match}s/{before}/{after}/{flags}'.format(
            limit='/{0}/ '.format(limit) if limit else '',
            negate_match='!' if negate_match else '',
            before=before,
            after=after,
            flags=flags
        )
    )
    cmd.append(path)

    return __salt__['cmd.run_all'](cmd, python_shell=False)


def sed_contains(path,
                 text,
                 limit='',
                 flags='g'):
    '''
    .. deprecated:: 0.17.0
       Use :func:`search` instead.

    Return True if the file at ``path`` contains ``text``. Utilizes sed to
    perform the search (line-wise search).

    Note: the ``p`` flag will be added to any flags you pass in.

    CLI Example:

    .. code-block:: bash

        salt '*' file.contains /etc/crontab 'mymaintenance.sh'
    '''
    # Largely inspired by Fabric's contrib.files.contains()
    path = os.path.expanduser(path)

    if not os.path.exists(path):
        return False

    before = _sed_esc(six.text_type(text), False)
    limit = _sed_esc(six.text_type(limit), False)
    options = '-n -r -e'
    if sys.platform == 'darwin':
        options = options.replace('-r', '-E')

    cmd = ['sed']
    cmd.extend(salt.utils.args.shlex_split(options))
    cmd.append(
        r'{limit}s/{before}/$/{flags}'.format(
            limit='/{0}/ '.format(limit) if limit else '',
            before=before,
            flags='p{0}'.format(flags)
        )
    )
    cmd.append(path)

    result = __salt__['cmd.run'](cmd, python_shell=False)

    return bool(result)


def psed(path,
         before,
         after,
         limit='',
         backup='.bak',
         flags='gMS',
         escape_all=False,
         multi=False):
    '''
    .. deprecated:: 0.17.0
       Use :py:func:`~salt.modules.file.replace` instead.

    Make a simple edit to a file (pure Python version)

    Equivalent to:

    .. code-block:: bash

        sed <backup> <options> "/<limit>/ s/<before>/<after>/<flags> <file>"

    path
        The full path to the file to be edited
    before
        A pattern to find in order to replace with ``after``
    after
        Text that will replace ``before``
    limit : ``''``
        An initial pattern to search for before searching for ``before``
    backup : ``.bak``
        The file will be backed up before edit with this file extension;
        **WARNING:** each time ``sed``/``comment``/``uncomment`` is called will
        overwrite this backup
    flags : ``gMS``
        Flags to modify the search. Valid values are:
          - ``g``: Replace all occurrences of the pattern, not just the first.
          - ``I``: Ignore case.
          - ``L``: Make ``\\w``, ``\\W``, ``\\b``, ``\\B``, ``\\s`` and ``\\S``
            dependent on the locale.
          - ``M``: Treat multiple lines as a single line.
          - ``S``: Make `.` match all characters, including newlines.
          - ``U``: Make ``\\w``, ``\\W``, ``\\b``, ``\\B``, ``\\d``, ``\\D``,
            ``\\s`` and ``\\S`` dependent on Unicode.
          - ``X``: Verbose (whitespace is ignored).
    multi: ``False``
        If True, treat the entire file as a single line

    Forward slashes and single quotes will be escaped automatically in the
    ``before`` and ``after`` patterns.

    CLI Example:

    .. code-block:: bash

        salt '*' file.sed /etc/httpd/httpd.conf 'LogLevel warn' 'LogLevel info'
    '''
    # Largely inspired by Fabric's contrib.files.sed()
    # XXX:dc: Do we really want to always force escaping?
    #
    # Mandate that before and after are strings
    path = os.path.expanduser(path)

    multi = bool(multi)

    before = six.text_type(before)
    after = six.text_type(after)
    before = _sed_esc(before, escape_all)
    # The pattern to replace with does not need to be escaped!!!
    #after = _sed_esc(after, escape_all)
    limit = _sed_esc(limit, escape_all)

    shutil.copy2(path, '{0}{1}'.format(path, backup))

    with salt.utils.files.fopen(path, 'w') as ofile:
        with salt.utils.files.fopen('{0}{1}'.format(path, backup), 'r') as ifile:
            if multi is True:
                for line in ifile.readline():
                    ofile.write(
                        salt.utils.stringutils.to_str(
                            _psed(
                                salt.utils.stringutils.to_unicode(line),
                                before,
                                after,
                                limit,
                                flags
                            )
                        )
                    )
            else:
                ofile.write(
                    salt.utils.stringutils.to_str(
                        _psed(
                            salt.utils.stringutils.to_unicode(ifile.read()),
                            before,
                            after,
                            limit,
                            flags
                        )
                    )
                )


RE_FLAG_TABLE = {'I': re.I,
                 'L': re.L,
                 'M': re.M,
                 'S': re.S,
                 'U': re.U,
                 'X': re.X}


def _psed(text,
          before,
          after,
          limit,
          flags):
    '''
    Does the actual work for file.psed, so that single lines can be passed in
    '''
    atext = text
    if limit:
        limit = re.compile(limit)
        comps = text.split(limit)
        atext = ''.join(comps[1:])

    count = 1
    if 'g' in flags:
        count = 0
        flags = flags.replace('g', '')

    aflags = 0
    for flag in flags:
        aflags |= RE_FLAG_TABLE[flag]

    before = re.compile(before, flags=aflags)
    text = re.sub(before, after, atext, count=count)

    return text


def uncomment(path,
              regex,
              char='#',
              backup='.bak'):
    '''
    .. deprecated:: 0.17.0
       Use :py:func:`~salt.modules.file.replace` instead.

    Uncomment specified commented lines in a file

    path
        The full path to the file to be edited
    regex
        A regular expression used to find the lines that are to be uncommented.
        This regex should not include the comment character. A leading ``^``
        character will be stripped for convenience (for easily switching
        between comment() and uncomment()).
    char : ``#``
        The character to remove in order to uncomment a line
    backup : ``.bak``
        The file will be backed up before edit with this file extension;
        **WARNING:** each time ``sed``/``comment``/``uncomment`` is called will
        overwrite this backup

    CLI Example:

    .. code-block:: bash

        salt '*' file.uncomment /etc/hosts.deny 'ALL: PARANOID'
    '''
    return comment_line(path=path,
                        regex=regex,
                        char=char,
                        cmnt=False,
                        backup=backup)


def comment(path,
            regex,
            char='#',
            backup='.bak'):
    '''
    .. deprecated:: 0.17.0
       Use :py:func:`~salt.modules.file.replace` instead.

    Comment out specified lines in a file

    path
        The full path to the file to be edited
    regex
        A regular expression used to find the lines that are to be commented;
        this pattern will be wrapped in parenthesis and will move any
        preceding/trailing ``^`` or ``$`` characters outside the parenthesis
        (e.g., the pattern ``^foo$`` will be rewritten as ``^(foo)$``)
    char : ``#``
        The character to be inserted at the beginning of a line in order to
        comment it out
    backup : ``.bak``
        The file will be backed up before edit with this file extension

        .. warning::

            This backup will be overwritten each time ``sed`` / ``comment`` /
            ``uncomment`` is called. Meaning the backup will only be useful
            after the first invocation.

    CLI Example:

    .. code-block:: bash

        salt '*' file.comment /etc/modules pcspkr
    '''
    return comment_line(path=path,
                        regex=regex,
                        char=char,
                        cmnt=True,
                        backup=backup)


def comment_line(path,
                 regex,
                 char='#',
                 cmnt=True,
                 backup='.bak'):
    r'''
    Comment or Uncomment a line in a text file.

    :param path: string
        The full path to the text file.

    :param regex: string
        A regex expression that begins with ``^`` that will find the line you wish
        to comment. Can be as simple as ``^color =``

    :param char: string
        The character used to comment a line in the type of file you're referencing.
        Default is ``#``

    :param cmnt: boolean
        True to comment the line. False to uncomment the line. Default is True.

    :param backup: string
        The file extension to give the backup file. Default is ``.bak``
        Set to False/None to not keep a backup.

    :return: boolean
        Returns True if successful, False if not

    CLI Example:

    The following example will comment out the ``pcspkr`` line in the
    ``/etc/modules`` file using the default ``#`` character and create a backup
    file named ``modules.bak``

    .. code-block:: bash

        salt '*' file.comment_line '/etc/modules' '^pcspkr'


    CLI Example:

    The following example will uncomment the ``log_level`` setting in ``minion``
    config file if it is set to either ``warning``, ``info``, or ``debug`` using
    the ``#`` character and create a backup file named ``minion.bk``

    .. code-block:: bash

        salt '*' file.comment_line 'C:\salt\conf\minion' '^log_level: (warning|info|debug)' '#' False '.bk'
    '''
    # Get the regex for comment or uncomment
    if cmnt:
        regex = '{0}({1}){2}'.format(
                '^' if regex.startswith('^') else '',
                regex.lstrip('^').rstrip('$'),
                '$' if regex.endswith('$') else '')
    else:
        regex = r'^{0}\s*({1}){2}'.format(
                char,
                regex.lstrip('^').rstrip('$'),
                '$' if regex.endswith('$') else '')

    # Load the real path to the file
    path = os.path.realpath(os.path.expanduser(path))

    # Make sure the file exists
    if not os.path.isfile(path):
        raise SaltInvocationError('File not found: {0}'.format(path))

    # Make sure it is a text file
    if not __utils__['files.is_text'](path):
        raise SaltInvocationError(
            'Cannot perform string replacements on a binary file: {0}'.format(path))

    # First check the whole file, determine whether to make the replacement
    # Searching first avoids modifying the time stamp if there are no changes
    found = False
    # Dictionaries for comparing changes
    orig_file = []
    new_file = []
    # Buffer size for fopen
    bufsize = os.path.getsize(path)
    try:
        # Use a read-only handle to open the file
        with salt.utils.files.fopen(path,
                              mode='rb',
                              buffering=bufsize) as r_file:
            # Loop through each line of the file and look for a match
            for line in r_file:
                # Is it in this line
                line = salt.utils.stringutils.to_unicode(line)
                if re.match(regex, line):
                    # Load lines into dictionaries, set found to True
                    orig_file.append(line)
                    if cmnt:
                        new_file.append('{0}{1}'.format(char, line))
                    else:
                        new_file.append(line.lstrip(char))
                    found = True
    except (OSError, IOError) as exc:
        raise CommandExecutionError(
            "Unable to open file '{0}'. "
            "Exception: {1}".format(path, exc)
        )

    # We've searched the whole file. If we didn't find anything, return False
    if not found:
        return False

    if not salt.utils.platform.is_windows():
        pre_user = get_user(path)
        pre_group = get_group(path)
        pre_mode = salt.utils.files.normalize_mode(get_mode(path))

    # Create a copy to read from and to use as a backup later
    try:
        temp_file = _mkstemp_copy(path=path, preserve_inode=False)
    except (OSError, IOError) as exc:
        raise CommandExecutionError("Exception: {0}".format(exc))

    try:
        # Open the file in write mode
        mode = 'wb' if six.PY2 and salt.utils.platform.is_windows() else 'w'
        with salt.utils.files.fopen(path,
                              mode=mode,
                              buffering=bufsize) as w_file:
            try:
                # Open the temp file in read mode
                with salt.utils.files.fopen(temp_file,
                                      mode='rb',
                                      buffering=bufsize) as r_file:
                    # Loop through each line of the file and look for a match
                    for line in r_file:
                        line = salt.utils.stringutils.to_unicode(line)
                        try:
                            # Is it in this line
                            if re.match(regex, line):
                                # Write the new line
                                if cmnt:
                                    wline = '{0}{1}'.format(char, line)
                                else:
                                    wline = line.lstrip(char)
                            else:
                                # Write the existing line (no change)
                                wline = line
                            wline = salt.utils.stringutils.to_bytes(wline) \
                                if six.PY2 and salt.utils.platform.is_windows() \
                                else salt.utils.stringutils.to_str(wline)
                            w_file.write(wline)
                        except (OSError, IOError) as exc:
                            raise CommandExecutionError(
                                "Unable to write file '{0}'. Contents may "
                                "be truncated. Temporary file contains copy "
                                "at '{1}'. "
                                "Exception: {2}".format(path, temp_file, exc)
                            )
            except (OSError, IOError) as exc:
                raise CommandExecutionError("Exception: {0}".format(exc))
    except (OSError, IOError) as exc:
        raise CommandExecutionError("Exception: {0}".format(exc))

    if backup:
        # Move the backup file to the original directory
        backup_name = '{0}{1}'.format(path, backup)
        try:
            shutil.move(temp_file, backup_name)
        except (OSError, IOError) as exc:
            raise CommandExecutionError(
                "Unable to move the temp file '{0}' to the "
                "backup file '{1}'. "
                "Exception: {2}".format(path, temp_file, exc)
            )
    else:
        os.remove(temp_file)

    if not salt.utils.platform.is_windows():
        check_perms(path, None, pre_user, pre_group, pre_mode)

    # Return a diff using the two dictionaries
    return ''.join(difflib.unified_diff(orig_file, new_file))


def _get_flags(flags):
    '''
    Return an integer appropriate for use as a flag for the re module from a
    list of human-readable strings

    .. code-block:: python

        >>> _get_flags(['MULTILINE', 'IGNORECASE'])
        10
        >>> _get_flags('MULTILINE')
        8
        >>> _get_flags(2)
        2
    '''
    if isinstance(flags, six.string_types):
        flags = [flags]

    if isinstance(flags, Iterable) and not isinstance(flags, Mapping):
        _flags_acc = []
        for flag in flags:
            _flag = getattr(re, six.text_type(flag).upper())

            if not isinstance(_flag, six.integer_types):
                raise SaltInvocationError(
                    'Invalid re flag given: {0}'.format(flag)
                )

            _flags_acc.append(_flag)

        return reduce(operator.__or__, _flags_acc)
    elif isinstance(flags, six.integer_types):
        return flags
    else:
        raise SaltInvocationError(
            'Invalid re flags: "{0}", must be given either as a single flag '
            'string, a list of strings, or as an integer'.format(flags)
        )


def _add_flags(flags, new_flags):
    '''
    Combine ``flags`` and ``new_flags``
    '''
    flags = _get_flags(flags)
    new_flags = _get_flags(new_flags)
    return flags | new_flags


def _mkstemp_copy(path,
                  preserve_inode=True):
    '''
    Create a temp file and move/copy the contents of ``path`` to the temp file.
    Return the path to the temp file.

    path
        The full path to the file whose contents will be moved/copied to a temp file.
        Whether it's moved or copied depends on the value of ``preserve_inode``.
    preserve_inode
        Preserve the inode of the file, so that any hard links continue to share the
        inode with the original filename. This works by *copying* the file, reading
        from the copy, and writing to the file at the original inode. If ``False``, the
        file will be *moved* rather than copied, and a new file will be written to a
        new inode, but using the original filename. Hard links will then share an inode
        with the backup, instead (if using ``backup`` to create a backup copy).
        Default is ``True``.
    '''
    temp_file = None
    # Create the temp file
    try:
        temp_file = salt.utils.files.mkstemp(prefix=salt.utils.files.TEMPFILE_PREFIX)
    except (OSError, IOError) as exc:
        raise CommandExecutionError(
            "Unable to create temp file. "
            "Exception: {0}".format(exc)
            )
    # use `copy` to preserve the inode of the
    # original file, and thus preserve hardlinks
    # to the inode. otherwise, use `move` to
    # preserve prior behavior, which results in
    # writing the file to a new inode.
    if preserve_inode:
        try:
            shutil.copy2(path, temp_file)
        except (OSError, IOError) as exc:
            raise CommandExecutionError(
                "Unable to copy file '{0}' to the "
                "temp file '{1}'. "
                "Exception: {2}".format(path, temp_file, exc)
                )
    else:
        try:
            shutil.move(path, temp_file)
        except (OSError, IOError) as exc:
            raise CommandExecutionError(
                "Unable to move file '{0}' to the "
                "temp file '{1}'. "
                "Exception: {2}".format(path, temp_file, exc)
                )

    return temp_file


def _starts_till(src, probe, strip_comments=True):
    '''
    Returns True if src and probe at least matches at the beginning till some point.
    '''
    def _strip_comments(txt):
        '''
        Strip possible comments.
        Usually comments are one or two symbols at the beginning of the line, separated with space
        '''
        buff = txt.split(" ", 1)
        return len(buff) == 2 and len(buff[0]) < 2 and buff[1] or txt

    def _to_words(txt):
        '''
        Split by words
        '''
        return txt and [w for w in txt.strip().split(" ") if w.strip()] or txt

    no_match = -1
    equal = 0
    if not src or not probe:
        return no_match

    if src == probe:
        return equal

    src = _to_words(strip_comments and _strip_comments(src) or src)
    probe = _to_words(strip_comments and _strip_comments(probe) or probe)

    a_buff, b_buff = len(src) < len(probe) and (src, probe) or (probe, src)
    b_buff = ' '.join(b_buff)
    for idx in range(len(a_buff)):
        prb = ' '.join(a_buff[:-(idx + 1)])
        if prb and b_buff.startswith(prb):
            return idx

    return no_match


def _regex_to_static(src, regex):
    '''
    Expand regular expression to static match.
    '''
    if not src or not regex:
        return None

    try:
        compiled = re.compile(regex, re.DOTALL)
        src = [line for line in src if compiled.search(line) or line.count(regex)]
    except Exception as ex:
        raise CommandExecutionError("{0}: '{1}'".format(_get_error_message(ex), regex))

    return src and src or []


def _assert_occurrence(probe, target, amount=1):
    '''
    Raise an exception, if there are different amount of specified occurrences in src.
    '''
    occ = len(probe)
    if occ > amount:
        msg = 'more than'
    elif occ < amount:
        msg = 'less than'
    elif not occ:
        msg = 'no'
    else:
        msg = None

    if msg:
        raise CommandExecutionError('Found {0} expected occurrences in "{1}" expression'.format(msg, target))

    return occ


def _set_line_indent(src, line, indent):
    '''
    Indent the line with the source line.
    '''
    if not indent:
        return line

    idt = []
    for c in src:
        if c not in ['\t', ' ']:
            break
        idt.append(c)

    return ''.join(idt) + line.lstrip()


def _get_eol(line):
    match = re.search('((?<!\r)\n|\r(?!\n)|\r\n)$', line)
    return match and match.group() or ''


def _set_line_eol(src, line):
    '''
    Add line ending
    '''
    line_ending = _get_eol(src) or os.linesep
    return line.rstrip() + line_ending


def _insert_line_before(idx, body, content, indent):
    if not idx or (idx and _starts_till(body[idx - 1], content) < 0):
        cnd = _set_line_indent(body[idx], content, indent)
        body.insert(idx, cnd)
    return body


def _insert_line_after(idx, body, content, indent):
    # No duplicates or append, if "after" is the last line
    next_line = idx + 1 < len(body) and body[idx + 1] or None
    if next_line is None or _starts_till(next_line, content) < 0:
        cnd = _set_line_indent(body[idx], content, indent)
        body.insert(idx + 1, cnd)
    return body


def line(path, content=None, match=None, mode=None, location=None,
         before=None, after=None, show_changes=True, backup=False,
         quiet=False, indent=True):
    '''
    .. versionadded:: 2015.8.0

    Edit a line in the configuration file. The ``path`` and ``content``
    arguments are required, as well as passing in one of the ``mode``
    options.

    path
        Filesystem path to the file to be edited.

    content
        Content of the line. Allowed to be empty if mode=delete.

    match
        Match the target line for an action by
        a fragment of a string or regular expression.

        If neither ``before`` nor ``after`` are provided, and ``match``
        is also ``None``, match becomes the ``content`` value.

    mode
        Defines how to edit a line. One of the following options is
        required:

        - ensure
            If line does not exist, it will be added. This is based on the
            ``content`` argument.
        - replace
            If line already exists, it will be replaced.
        - delete
            Delete the line, once found.
        - insert
            Insert a line.

        .. note::

            If ``mode=insert`` is used, at least one of the following
            options must also be defined: ``location``, ``before``, or
            ``after``. If ``location`` is used, it takes precedence
            over the other two options.

    location
        Defines where to place content in the line. Note this option is only
        used when ``mode=insert`` is specified. If a location is passed in, it
        takes precedence over both the ``before`` and ``after`` kwargs. Valid
        locations are:

        - start
            Place the content at the beginning of the file.
        - end
            Place the content at the end of the file.

    before
        Regular expression or an exact case-sensitive fragment of the string.
        This option is only used when either the ``ensure`` or ``insert`` mode
        is defined.

    after
        Regular expression or an exact case-sensitive fragment of the string.
        This option is only used when either the ``ensure`` or ``insert`` mode
        is defined.

    show_changes
        Output a unified diff of the old file and the new file.
        If ``False`` return a boolean if any changes were made.
        Default is ``True``

        .. note::
            Using this option will store two copies of the file in-memory
            (the original version and the edited version) in order to generate the diff.

    backup
        Create a backup of the original file with the extension:
        "Year-Month-Day-Hour-Minutes-Seconds".

    quiet
        Do not raise any exceptions. E.g. ignore the fact that the file that is
        tried to be edited does not exist and nothing really happened.

    indent
        Keep indentation with the previous line. This option is not considered when
        the ``delete`` mode is specified.

    CLI Example:

    .. code-block:: bash

        salt '*' file.line /etc/nsswitch.conf "networks:\tfiles dns" after="hosts:.*?" mode='ensure'

    .. note::

        If an equal sign (``=``) appears in an argument to a Salt command, it is
        interpreted as a keyword argument in the format of ``key=val``. That
        processing can be bypassed in order to pass an equal sign through to the
        remote shell command by manually specifying the kwarg:

        .. code-block:: bash

            salt '*' file.line /path/to/file content="CREATEMAIL_SPOOL=no" match="CREATE_MAIL_SPOOL=yes" mode="replace"
    '''
    path = os.path.realpath(os.path.expanduser(path))
    if not os.path.isfile(path):
        if not quiet:
            raise CommandExecutionError('File "{0}" does not exists or is not a file.'.format(path))
        return False  # No changes had happened

    mode = mode and mode.lower() or mode
    if mode not in ['insert', 'ensure', 'delete', 'replace']:
        if mode is None:
            raise CommandExecutionError('Mode was not defined. How to process the file?')
        else:
            raise CommandExecutionError('Unknown mode: "{0}"'.format(mode))

    # We've set the content to be empty in the function params but we want to make sure
    # it gets passed when needed. Feature #37092
    empty_content_modes = ['delete']
    if mode not in empty_content_modes and content is None:
        raise CommandExecutionError('Content can only be empty if mode is "{0}"'.format(', '.join(empty_content_modes)))
    del empty_content_modes

    # Before/after has privilege. If nothing defined, match is used by content.
    if before is None and after is None and not match:
        match = content

    with salt.utils.files.fopen(path, mode='r') as fp_:
        body = salt.utils.data.decode_list(fp_.readlines())
    body_before = hashlib.sha256(salt.utils.stringutils.to_bytes(''.join(body))).hexdigest()
    # Add empty line at the end if last line ends with eol.
    # Allows simpler code
    if body and _get_eol(body[-1]):
        body.append('')

    after = _regex_to_static(body, after)
    before = _regex_to_static(body, before)
    match = _regex_to_static(body, match)

    if os.stat(path).st_size == 0 and mode in ('delete', 'replace'):
        log.warning('Cannot find text to {0}. File \'{1}\' is empty.'.format(mode, path))
        body = []
    elif mode == 'delete' and match:
        body = [line for line in body if line != match[0]]
    elif mode == 'replace' and match:
        idx = body.index(match[0])
        file_line = body.pop(idx)
        body.insert(idx, _set_line_indent(file_line, content, indent))
    elif mode == 'insert':
        if not location and not before and not after:
            raise CommandExecutionError('On insert must be defined either "location" or "before/after" conditions.')

        if not location:
            if before and after:
                _assert_occurrence(before, 'before')
                _assert_occurrence(after, 'after')

                out = []
                in_range = False
                for line in body:
                    if line == after[0]:
                        in_range = True
                    elif line == before[0] and in_range:
                        cnd = _set_line_indent(line, content, indent)
                        out.append(cnd)
                    out.append(line)
                body = out

            if before and not after:
                _assert_occurrence(before, 'before')

                idx = body.index(before[0])
                body = _insert_line_before(idx, body, content, indent)

            elif after and not before:
                _assert_occurrence(after, 'after')

                idx = body.index(after[0])
                body = _insert_line_after(idx, body, content, indent)

        else:
            if location == 'start':
                if body:
                    body.insert(0, _set_line_eol(body[0], content))
                else:
                    body.append(content + os.linesep)
            elif location == 'end':
                body.append(_set_line_indent(body[-1], content, indent) if body else content)

    elif mode == 'ensure':

        if before and after:
            _assert_occurrence(before, 'before')
            _assert_occurrence(after, 'after')

            is_there = bool([l for l in body if l.count(content)])
            if not is_there:
                idx = body.index(after[0])
                if idx < (len(body) - 1) and body[idx + 1] == before[0]:
                    cnd = _set_line_indent(body[idx], content, indent)
                    body.insert(idx + 1, cnd)
                else:
                    raise CommandExecutionError('Found more than one line between '
                                                'boundaries "before" and "after".')

        elif before and not after:
            _assert_occurrence(before, 'before')

            idx = body.index(before[0])
            body = _insert_line_before(idx, body, content, indent)

        elif not before and after:
            _assert_occurrence(after, 'after')

            idx = body.index(after[0])
            body = _insert_line_after(idx, body, content, indent)

        else:
            raise CommandExecutionError("Wrong conditions? "
                                        "Unable to ensure line without knowing "
                                        "where to put it before and/or after.")

    if body:
        for idx, line in enumerate(body):
            if not _get_eol(line) and idx+1 < len(body):
                prev = idx and idx-1 or 1
                body[idx] = _set_line_eol(body[prev], line)
        # We do not need empty line at the end anymore
        if '' == body[-1]:
            body.pop()

    changed = body_before != hashlib.sha256(salt.utils.stringutils.to_bytes(''.join(body))).hexdigest()

    if backup and changed and __opts__['test'] is False:
        try:
            temp_file = _mkstemp_copy(path=path, preserve_inode=True)
            shutil.move(temp_file, '{0}.{1}'.format(path, time.strftime('%Y-%m-%d-%H-%M-%S', time.localtime())))
        except (OSError, IOError) as exc:
            raise CommandExecutionError("Unable to create the backup file of {0}. Exception: {1}".format(path, exc))

    changes_diff = None

    if changed:
        if show_changes:
            with salt.utils.files.fopen(path, 'r') as fp_:
                path_content = salt.utils.data.decode_list(fp_.read().splitlines(True))
            changes_diff = ''.join(difflib.unified_diff(
                path_content, body
            ))
        if __opts__['test'] is False:
            fh_ = None
            try:
                # Make sure we match the file mode from salt.utils.files.fopen
                if six.PY2 and salt.utils.platform.is_windows():
                    mode = 'wb'
                    body = salt.utils.data.encode_list(body)
                else:
                    mode = 'w'
                    body = salt.utils.data.decode_list(body, to_str=True)
                fh_ = salt.utils.atomicfile.atomic_open(path, mode)
                fh_.writelines(body)
            finally:
                if fh_:
                    fh_.close()

    return show_changes and changes_diff or changed


def replace(path,
            pattern,
            repl,
            count=0,
            flags=8,
            bufsize=1,
            append_if_not_found=False,
            prepend_if_not_found=False,
            not_found_content=None,
            backup='.bak',
            dry_run=False,
            search_only=False,
            show_changes=True,
            ignore_if_missing=False,
            preserve_inode=True,
            backslash_literal=False,
        ):
    '''
    .. versionadded:: 0.17.0

    Replace occurrences of a pattern in a file. If ``show_changes`` is
    ``True``, then a diff of what changed will be returned, otherwise a
    ``True`` will be returned when changes are made, and ``False`` when
    no changes are made.

    This is a pure Python implementation that wraps Python's :py:func:`~re.sub`.

    path
        Filesystem path to the file to be edited. If a symlink is specified, it
        will be resolved to its target.

    pattern
        A regular expression, to be matched using Python's
        :py:func:`~re.search`.

    repl
        The replacement text

    count : 0
        Maximum number of pattern occurrences to be replaced. If count is a
        positive integer ``n``, only ``n`` occurrences will be replaced,
        otherwise all occurrences will be replaced.

    flags (list or int)
        A list of flags defined in the ``re`` module documentation from the
        Python standard library. Each list item should be a string that will
        correlate to the human-friendly flag name. E.g., ``['IGNORECASE',
        'MULTILINE']``. Optionally, ``flags`` may be an int, with a value
        corresponding to the XOR (``|``) of all the desired flags. Defaults to
        8 (which supports 'MULTILINE').

    bufsize (int or str)
        How much of the file to buffer into memory at once. The
        default value ``1`` processes one line at a time. The special value
        ``file`` may be specified which will read the entire file into memory
        before processing.

    append_if_not_found : False
        .. versionadded:: 2014.7.0

        If set to ``True``, and pattern is not found, then the content will be
        appended to the file.

    prepend_if_not_found : False
        .. versionadded:: 2014.7.0

        If set to ``True`` and pattern is not found, then the content will be
        prepended to the file.

    not_found_content
        .. versionadded:: 2014.7.0

        Content to use for append/prepend if not found. If None (default), uses
        ``repl``. Useful when ``repl`` uses references to group in pattern.

    backup : .bak
        The file extension to use for a backup of the file before editing. Set
        to ``False`` to skip making a backup.

    dry_run : False
        If set to ``True``, no changes will be made to the file, the function
        will just return the changes that would have been made (or a
        ``True``/``False`` value if ``show_changes`` is set to ``False``).

    search_only : False
        If set to true, this no changes will be performed on the file, and this
        function will simply return ``True`` if the pattern was matched, and
        ``False`` if not.

    show_changes : True
        If ``True``, return a diff of changes made. Otherwise, return ``True``
        if changes were made, and ``False`` if not.

        .. note::
            Using this option will store two copies of the file in memory (the
            original version and the edited version) in order to generate the
            diff. This may not normally be a concern, but could impact
            performance if used with large files.

    ignore_if_missing : False
        .. versionadded:: 2015.8.0

        If set to ``True``, this function will simply return ``False``
        if the file doesn't exist. Otherwise, an error will be thrown.

    preserve_inode : True
        .. versionadded:: 2015.8.0

        Preserve the inode of the file, so that any hard links continue to
        share the inode with the original filename. This works by *copying* the
        file, reading from the copy, and writing to the file at the original
        inode. If ``False``, the file will be *moved* rather than copied, and a
        new file will be written to a new inode, but using the original
        filename. Hard links will then share an inode with the backup, instead
        (if using ``backup`` to create a backup copy).

    backslash_literal : False
        .. versionadded:: 2016.11.7

        Interpret backslashes as literal backslashes for the repl and not
        escape characters.  This will help when using append/prepend so that
        the backslashes are not interpreted for the repl on the second run of
        the state.

    If an equal sign (``=``) appears in an argument to a Salt command it is
    interpreted as a keyword argument in the format ``key=val``. That
    processing can be bypassed in order to pass an equal sign through to the
    remote shell command by manually specifying the kwarg:

    .. code-block:: bash

        salt '*' file.replace /path/to/file pattern='=' repl=':'
        salt '*' file.replace /path/to/file pattern="bind-address\\s*=" repl='bind-address:'

    CLI Examples:

    .. code-block:: bash

        salt '*' file.replace /etc/httpd/httpd.conf pattern='LogLevel warn' repl='LogLevel info'
        salt '*' file.replace /some/file pattern='before' repl='after' flags='[MULTILINE, IGNORECASE]'
    '''
    symlink = False
    if is_link(path):
        symlink = True
        target_path = os.readlink(path)
        given_path = os.path.expanduser(path)

    path = os.path.realpath(os.path.expanduser(path))

    if not os.path.exists(path):
        if ignore_if_missing:
            return False
        else:
            raise SaltInvocationError('File not found: {0}'.format(path))

    if not __utils__['files.is_text'](path):
        raise SaltInvocationError(
            'Cannot perform string replacements on a binary file: {0}'
            .format(path)
        )

    if search_only and (append_if_not_found or prepend_if_not_found):
        raise SaltInvocationError(
            'search_only cannot be used with append/prepend_if_not_found'
        )

    if append_if_not_found and prepend_if_not_found:
        raise SaltInvocationError(
            'Only one of append and prepend_if_not_found is permitted'
        )

    flags_num = _get_flags(flags)
    cpattern = re.compile(salt.utils.stringutils.to_bytes(pattern), flags_num)
    filesize = os.path.getsize(path)
    if bufsize == 'file':
        bufsize = filesize

    # Search the file; track if any changes have been made for the return val
    has_changes = False
    orig_file = []  # used for show_changes and change detection
    new_file = []  # used for show_changes and change detection
    if not salt.utils.platform.is_windows():
        pre_user = get_user(path)
        pre_group = get_group(path)
        pre_mode = salt.utils.files.normalize_mode(get_mode(path))

    # Avoid TypeErrors by forcing repl to be bytearray related to mmap
    # Replacement text may contains integer: 123 for example
    repl = salt.utils.stringutils.to_bytes(six.text_type(repl))
    if not_found_content:
        not_found_content = salt.utils.stringutils.to_bytes(not_found_content)

    found = False
    temp_file = None
    content = salt.utils.stringutils.to_unicode(not_found_content) \
        if not_found_content and (prepend_if_not_found or append_if_not_found) \
        else salt.utils.stringutils.to_unicode(repl)

    try:
        # First check the whole file, determine whether to make the replacement
        # Searching first avoids modifying the time stamp if there are no changes
        r_data = None
        # Use a read-only handle to open the file
        with salt.utils.files.fopen(path,
                              mode='rb',
                              buffering=bufsize) as r_file:
            try:
                # mmap throws a ValueError if the file is empty.
                r_data = mmap.mmap(r_file.fileno(),
                                   0,
                                   access=mmap.ACCESS_READ)
            except (ValueError, mmap.error):
                # size of file in /proc is 0, but contains data
                r_data = salt.utils.stringutils.to_bytes("".join(r_file))
            if search_only:
                # Just search; bail as early as a match is found
                if re.search(cpattern, r_data):
                    return True  # `with` block handles file closure
            else:
                result, nrepl = re.subn(cpattern,
                                        repl.replace('\\', '\\\\') if backslash_literal else repl,
                                        r_data,
                                        count)

                # found anything? (even if no change)
                if nrepl > 0:
                    found = True
                    # Identity check the potential change
                    has_changes = True if pattern != repl else has_changes

                if prepend_if_not_found or append_if_not_found:
                    # Search for content, to avoid pre/appending the
                    # content if it was pre/appended in a previous run.
                    if re.search(salt.utils.stringutils.to_bytes('^{0}$'.format(re.escape(content))),
                                 r_data,
                                 flags=flags_num):
                        # Content was found, so set found.
                        found = True

                orig_file = r_data.read(filesize).splitlines(True) \
                    if isinstance(r_data, mmap.mmap) \
                    else r_data.splitlines(True)
                new_file = result.splitlines(True)

    except (OSError, IOError) as exc:
        raise CommandExecutionError(
            "Unable to open file '{0}'. "
            "Exception: {1}".format(path, exc)
            )
    finally:
        if r_data and isinstance(r_data, mmap.mmap):
            r_data.close()

    if has_changes and not dry_run:
        # Write the replacement text in this block.
        try:
            # Create a copy to read from and to use as a backup later
            temp_file = _mkstemp_copy(path=path,
                                      preserve_inode=preserve_inode)
        except (OSError, IOError) as exc:
            raise CommandExecutionError("Exception: {0}".format(exc))

        r_data = None
        try:
            # Open the file in write mode
            with salt.utils.files.fopen(path,
                        mode='w',
                        buffering=bufsize) as w_file:
                try:
                    # Open the temp file in read mode
                    with salt.utils.files.fopen(temp_file,
                                          mode='r',
                                          buffering=bufsize) as r_file:
                        r_data = mmap.mmap(r_file.fileno(),
                                           0,
                                           access=mmap.ACCESS_READ)
                        result, nrepl = re.subn(cpattern,
                                                repl.replace('\\', '\\\\') if backslash_literal else repl,
                                                r_data,
                                                count)
                        try:
                            w_file.write(salt.utils.stringutils.to_str(result))
                        except (OSError, IOError) as exc:
                            raise CommandExecutionError(
                                "Unable to write file '{0}'. Contents may "
                                "be truncated. Temporary file contains copy "
                                "at '{1}'. "
                                "Exception: {2}".format(path, temp_file, exc)
                                )
                except (OSError, IOError) as exc:
                    raise CommandExecutionError("Exception: {0}".format(exc))
                finally:
                    if r_data and isinstance(r_data, mmap.mmap):
                        r_data.close()
        except (OSError, IOError) as exc:
            raise CommandExecutionError("Exception: {0}".format(exc))

    if not found and (append_if_not_found or prepend_if_not_found):
        if not_found_content is None:
            not_found_content = repl
        if prepend_if_not_found:
            new_file.insert(0, not_found_content + salt.utils.stringutils.to_bytes(os.linesep))
        else:
            # append_if_not_found
            # Make sure we have a newline at the end of the file
            if 0 != len(new_file):
                if not new_file[-1].endswith(salt.utils.stringutils.to_bytes(os.linesep)):
                    new_file[-1] += salt.utils.stringutils.to_bytes(os.linesep)
            new_file.append(not_found_content + salt.utils.stringutils.to_bytes(os.linesep))
        has_changes = True
        if not dry_run:
            try:
                # Create a copy to read from and for later use as a backup
                temp_file = _mkstemp_copy(path=path,
                                          preserve_inode=preserve_inode)
            except (OSError, IOError) as exc:
                raise CommandExecutionError("Exception: {0}".format(exc))
            # write new content in the file while avoiding partial reads
            try:
                fh_ = salt.utils.atomicfile.atomic_open(path, 'wb')
                for line in new_file:
                    fh_.write(salt.utils.stringutils.to_bytes(line))
            finally:
                fh_.close()

    if backup and has_changes and not dry_run:
        # keep the backup only if it was requested
        # and only if there were any changes
        backup_name = '{0}{1}'.format(path, backup)
        try:
            shutil.move(temp_file, backup_name)
        except (OSError, IOError) as exc:
            raise CommandExecutionError(
                "Unable to move the temp file '{0}' to the "
                "backup file '{1}'. "
                "Exception: {2}".format(path, temp_file, exc)
                )
        if symlink:
            symlink_backup = '{0}{1}'.format(given_path, backup)
            target_backup = '{0}{1}'.format(target_path, backup)
            # Always clobber any existing symlink backup
            # to match the behaviour of the 'backup' option
            try:
                os.symlink(target_backup, symlink_backup)
            except OSError:
                os.remove(symlink_backup)
                os.symlink(target_backup, symlink_backup)
            except:
                raise CommandExecutionError(
                    "Unable create backup symlink '{0}'. "
                    "Target was '{1}'. "
                    "Exception: {2}".format(symlink_backup, target_backup,
                                            exc)
                    )
    elif temp_file:
        try:
            os.remove(temp_file)
        except (OSError, IOError) as exc:
            raise CommandExecutionError(
                "Unable to delete temp file '{0}'. "
                "Exception: {1}".format(temp_file, exc)
                )

    if not dry_run and not salt.utils.platform.is_windows():
        check_perms(path, None, pre_user, pre_group, pre_mode)

    def get_changes():
        orig_file_as_str = [salt.utils.stringutils.to_unicode(x) for x in orig_file]
        new_file_as_str = [salt.utils.stringutils.to_unicode(x) for x in new_file]
        return ''.join(difflib.unified_diff(orig_file_as_str, new_file_as_str))

    if show_changes:
        return get_changes()

    # We may have found a regex line match but don't need to change the line
    # (for situations where the pattern also matches the repl). Revert the
    # has_changes flag to False if the final result is unchanged.
    if not get_changes():
        has_changes = False

    return has_changes


def blockreplace(path,
        marker_start='#-- start managed zone --',
        marker_end='#-- end managed zone --',
        content='',
        append_if_not_found=False,
        prepend_if_not_found=False,
        backup='.bak',
        dry_run=False,
        show_changes=True,
        append_newline=False):
    '''
    .. versionadded:: 2014.1.0

    Replace content of a text block in a file, delimited by line markers

    A block of content delimited by comments can help you manage several lines
    entries without worrying about old entries removal.

    .. note::

        This function will store two copies of the file in-memory (the original
        version and the edited version) in order to detect changes and only
        edit the targeted file if necessary.

    path
        Filesystem path to the file to be edited

    marker_start
        The line content identifying a line as the start of the content block.
        Note that the whole line containing this marker will be considered, so
        whitespace or extra content before or after the marker is included in
        final output

    marker_end
        The line content identifying the end of the content block. As of
        versions 2017.7.5 and 2018.3.1, everything up to the text matching the
        marker will be replaced, so it's important to ensure that your marker
        includes the beginning of the text you wish to replace.

    content
        The content to be used between the two lines identified by marker_start
        and marker_stop.

    append_if_not_found : False
        If markers are not found and set to ``True`` then, the markers and
        content will be appended to the file.

    prepend_if_not_found : False
        If markers are not found and set to ``True`` then, the markers and
        content will be prepended to the file.


    backup
        The file extension to use for a backup of the file if any edit is made.
        Set to ``False`` to skip making a backup.

    dry_run : False
        If ``True``, do not make any edits to the file and simply return the
        changes that *would* be made.

    show_changes : True
        Controls how changes are presented. If ``True``, this function will
        return a unified diff of the changes made. If False, then it will
        return a boolean (``True`` if any changes were made, otherwise
        ``False``).

    append_newline : False
        Controls whether or not a newline is appended to the content block. If
        the value of this argument is ``True`` then a newline will be added to
        the content block. If it is ``False``, then a newline will *not* be
        added to the content block. If it is ``None`` then a newline will only
        be added to the content block if it does not already end in a newline.

        .. versionadded:: 2016.3.4
        .. versionchanged:: 2017.7.5,2018.3.1
            New behavior added when value is ``None``.
        .. versionchanged:: Fluorine
            The default value of this argument will change to ``None`` to match
            the behavior of the :py:func:`file.blockreplace state
            <salt.states.file.blockreplace>`

    CLI Example:

    .. code-block:: bash

        salt '*' file.blockreplace /etc/hosts '#-- start managed zone foobar : DO NOT EDIT --' \\
        '#-- end managed zone foobar --' $'10.0.1.1 foo.foobar\\n10.0.1.2 bar.foobar' True

    '''
    if append_if_not_found and prepend_if_not_found:
        raise SaltInvocationError(
            'Only one of append and prepend_if_not_found is permitted'
        )

    path = os.path.expanduser(path)

    if not os.path.exists(path):
        raise SaltInvocationError('File not found: {0}'.format(path))

    try:
        file_encoding = __utils__['files.get_encoding'](path)
    except CommandExecutionError:
        file_encoding = None

    if __utils__['files.is_binary'](path):
        if not file_encoding:
            raise SaltInvocationError(
                'Cannot perform string replacements on a binary file: {0}'
                .format(path)
        )

    if append_newline is None and not content.endswith((os.linesep, '\n')):
        append_newline = True

    # Split the content into a list of lines, removing newline characters. To
    # ensure that we handle both Windows and POSIX newlines, first split on
    # Windows newlines, and then split on POSIX newlines.
    split_content = []
    for win_line in content.split('\r\n'):
        for content_line in win_line.split('\n'):
            split_content.append(content_line)

    line_count = len(split_content)

    has_changes = False
    orig_file = []
    new_file = []
    in_block = False
    block_found = False
    linesep = None

    def _add_content(linesep, lines=None, include_marker_start=True,
                     end_line=None):
        if lines is None:
            lines = []
            include_marker_start = True

        if end_line is None:
            end_line = marker_end
        end_line = end_line.rstrip('\r\n') + linesep

        if include_marker_start:
            lines.append(marker_start + linesep)

        if split_content:
            for index, content_line in enumerate(split_content, 1):
                if index != line_count:
                    lines.append(content_line + linesep)
                else:
                    # We're on the last line of the content block
                    if append_newline:
                        lines.append(content_line + linesep)
                        lines.append(end_line)
                    else:
                        lines.append(content_line + end_line)
        else:
            lines.append(end_line)

        return lines

    # We do not use in-place editing to avoid file attrs modifications when
    # no changes are required and to avoid any file access on a partially
    # written file.
    try:
        fi_file = io.open(path, mode='r', encoding=file_encoding, newline='')
        for line in fi_file:
            write_line_to_new_file = True

            if linesep is None:
                # Auto-detect line separator
                if line.endswith('\r\n'):
                    linesep = '\r\n'
                elif line.endswith('\n'):
                    linesep = '\n'
                else:
                    # No newline(s) in file, fall back to system's linesep
                    linesep = os.linesep

            if marker_start in line:
                # We've entered the content block
                in_block = True
            else:
                if in_block:
                    # We're not going to write the lines from the old file to
                    # the new file until we have exited the block.
                    write_line_to_new_file = False

                    marker_end_pos = line.find(marker_end)
                    if marker_end_pos != -1:
                        # End of block detected
                        in_block = False
                        # We've found and exited the block
                        block_found = True

                        _add_content(linesep, lines=new_file,
                                     include_marker_start=False,
                                     end_line=line[marker_end_pos:])

            # Save the line from the original file
            orig_file.append(line)
            if write_line_to_new_file:
                new_file.append(line)

    except (IOError, OSError) as exc:
        raise CommandExecutionError(
            'Failed to read from {0}: {1}'.format(path, exc)
        )
    finally:
        if linesep is None:
            # If the file was empty, we will not have set linesep yet. Assume
            # the system's line separator. This is needed for when we
            # prepend/append later on.
            linesep = os.linesep
        try:
            fi_file.close()
        except Exception:
            pass

    if in_block:
        # unterminated block => bad, always fail
        raise CommandExecutionError(
            'Unterminated marked block. End of file reached before marker_end.'
        )

    if not block_found:
        if prepend_if_not_found:
            # add the markers and content at the beginning of file
            prepended_content = _add_content(linesep)
            prepended_content.extend(new_file)
            new_file = prepended_content
            block_found = True
        elif append_if_not_found:
            # Make sure we have a newline at the end of the file
            if 0 != len(new_file):
                if not new_file[-1].endswith(linesep):
                    new_file[-1] += linesep
            # add the markers and content at the end of file
            _add_content(linesep, lines=new_file)
            block_found = True
        else:
            raise CommandExecutionError(
                'Cannot edit marked block. Markers were not found in file.'
            )

    if block_found:
        diff = ''.join(difflib.unified_diff(orig_file, new_file))
        has_changes = diff is not ''
        if has_changes and not dry_run:
            # changes detected
            # backup file attrs
            perms = {}
            perms['user'] = get_user(path)
            perms['group'] = get_group(path)
            perms['mode'] = salt.utils.files.normalize_mode(get_mode(path))

            # backup old content
            if backup is not False:
                backup_path = '{0}{1}'.format(path, backup)
                shutil.copy2(path, backup_path)
                # copy2 does not preserve ownership
                check_perms(backup_path,
                        None,
                        perms['user'],
                        perms['group'],
                        perms['mode'])

            # write new content in the file while avoiding partial reads
            try:
                fh_ = salt.utils.atomicfile.atomic_open(path, 'wb')
                for line in new_file:
                    fh_.write(salt.utils.stringutils.to_bytes(line, encoding=file_encoding))
            finally:
                fh_.close()

            # this may have overwritten file attrs
            check_perms(path,
                    None,
                    perms['user'],
                    perms['group'],
                    perms['mode'])

        if show_changes:
            return diff

    return has_changes


def search(path,
        pattern,
        flags=8,
        bufsize=1,
        ignore_if_missing=False,
        multiline=False
        ):
    '''
    .. versionadded:: 0.17.0

    Search for occurrences of a pattern in a file

    Except for multiline, params are identical to
    :py:func:`~salt.modules.file.replace`.

    multiline
        If true, inserts 'MULTILINE' into ``flags`` and sets ``bufsize`` to
        'file'.

        .. versionadded:: 2015.8.0

    CLI Example:

    .. code-block:: bash

        salt '*' file.search /etc/crontab 'mymaintenance.sh'
    '''
    if multiline:
        flags = _add_flags(flags, 'MULTILINE')
        bufsize = 'file'

    # This function wraps file.replace on purpose in order to enforce
    # consistent usage, compatible regex's, expected behavior, *and* bugs. :)
    # Any enhancements or fixes to one should affect the other.
    return replace(path,
            pattern,
            '',
            flags=flags,
            bufsize=bufsize,
            dry_run=True,
            search_only=True,
            show_changes=False,
            ignore_if_missing=ignore_if_missing)


def patch(originalfile, patchfile, options='', dry_run=False):
    '''
    .. versionadded:: 0.10.4

    Apply a patch to a file or directory.

    Equivalent to:

    .. code-block:: bash

        patch <options> -i <patchfile> <originalfile>

    Or, when a directory is patched:

    .. code-block:: bash

        patch <options> -i <patchfile> -d <originalfile> -p0

    originalfile
        The full path to the file or directory to be patched
    patchfile
        A patch file to apply to ``originalfile``
    options
        Options to pass to patch.

    CLI Example:

    .. code-block:: bash

        salt '*' file.patch /opt/file.txt /tmp/file.txt.patch
    '''
    patchpath = salt.utils.path.which('patch')
    if not patchpath:
        raise CommandExecutionError(
            'patch executable not found. Is the distribution\'s patch '
            'package installed?'
        )

    cmd = [patchpath]
    cmd.extend(salt.utils.args.shlex_split(options))
    if dry_run:
        if __grains__['kernel'] in ('FreeBSD', 'OpenBSD'):
            cmd.append('-C')
        else:
            cmd.append('--dry-run')

    # this argument prevents interactive prompts when the patch fails to apply.
    # the exit code will still be greater than 0 if that is the case.
    if '-N' not in cmd and '--forward' not in cmd:
        cmd.append('--forward')

    has_rejectfile_option = False
    for option in cmd:
        if option == '-r' or option.startswith('-r ') \
                or option.startswith('--reject-file'):
            has_rejectfile_option = True
            break

    # by default, patch will write rejected patch files to <filename>.rej.
    # this option prevents that.
    if not has_rejectfile_option:
        cmd.append('--reject-file=-')

    cmd.extend(['-i', patchfile])

    if os.path.isdir(originalfile):
        cmd.extend(['-d', originalfile])

        has_strip_option = False
        for option in cmd:
            if option.startswith('-p') or option.startswith('--strip='):
                has_strip_option = True
                break

        if not has_strip_option:
            cmd.append('--strip=0')
    else:
        cmd.append(originalfile)

    return __salt__['cmd.run_all'](cmd, python_shell=False)


def contains(path, text):
    '''
    .. deprecated:: 0.17.0
       Use :func:`search` instead.

    Return ``True`` if the file at ``path`` contains ``text``

    CLI Example:

    .. code-block:: bash

        salt '*' file.contains /etc/crontab 'mymaintenance.sh'
    '''
    path = os.path.expanduser(path)

    if not os.path.exists(path):
        return False

    stripped_text = six.text_type(text).strip()
    try:
        with salt.utils.filebuffer.BufferedReader(path) as breader:
            for chunk in breader:
                if stripped_text in chunk:
                    return True
        return False
    except (IOError, OSError):
        return False


def contains_regex(path, regex, lchar=''):
    '''
    .. deprecated:: 0.17.0
       Use :func:`search` instead.

    Return True if the given regular expression matches on any line in the text
    of a given file.

    If the lchar argument (leading char) is specified, it
    will strip `lchar` from the left side of each line before trying to match

    CLI Example:

    .. code-block:: bash

        salt '*' file.contains_regex /etc/crontab
    '''
    path = os.path.expanduser(path)

    if not os.path.exists(path):
        return False

    try:
        with salt.utils.files.fopen(path, 'r') as target:
            for line in target:
                line = salt.utils.stringutils.to_unicode(line)
                if lchar:
                    line = line.lstrip(lchar)
                if re.search(regex, line):
                    return True
            return False
    except (IOError, OSError):
        return False


def contains_glob(path, glob_expr):
    '''
    .. deprecated:: 0.17.0
       Use :func:`search` instead.

    Return ``True`` if the given glob matches a string in the named file

    CLI Example:

    .. code-block:: bash

        salt '*' file.contains_glob /etc/foobar '*cheese*'
    '''
    path = os.path.expanduser(path)

    if not os.path.exists(path):
        return False

    try:
        with salt.utils.filebuffer.BufferedReader(path) as breader:
            for chunk in breader:
                if fnmatch.fnmatch(chunk, glob_expr):
                    return True
            return False
    except (IOError, OSError):
        return False


def append(path, *args, **kwargs):
    '''
    .. versionadded:: 0.9.5

    Append text to the end of a file

    path
        path to file

    `*args`
        strings to append to file

    CLI Example:

    .. code-block:: bash

        salt '*' file.append /etc/motd \\
                "With all thine offerings thou shalt offer salt." \\
                "Salt is what makes things taste bad when it isn't in them."

    .. admonition:: Attention

        If you need to pass a string to append and that string contains
        an equal sign, you **must** include the argument name, args.
        For example:

        .. code-block:: bash

            salt '*' file.append /etc/motd args='cheese=spam'

            salt '*' file.append /etc/motd args="['cheese=spam','spam=cheese']"

    '''
    path = os.path.expanduser(path)

    # Largely inspired by Fabric's contrib.files.append()

    if 'args' in kwargs:
        if isinstance(kwargs['args'], list):
            args = kwargs['args']
        else:
            args = [kwargs['args']]

    # Make sure we have a newline at the end of the file. Do this in binary
    # mode so SEEK_END with nonzero offset will work.
    with salt.utils.files.fopen(path, 'rb+') as ofile:
        linesep = salt.utils.stringutils.to_bytes(os.linesep)
        try:
            ofile.seek(-len(linesep), os.SEEK_END)
        except IOError as exc:
            if exc.errno in (errno.EINVAL, errno.ESPIPE):
                # Empty file, simply append lines at the beginning of the file
                pass
            else:
                raise
        else:
            if ofile.read(len(linesep)) != linesep:
                ofile.seek(0, os.SEEK_END)
                ofile.write(linesep)

    # Append lines in text mode
    with salt.utils.files.fopen(path, 'a') as ofile:
        for new_line in args:
            ofile.write(
                salt.utils.stringutils.to_str(
                    '{0}{1}'.format(new_line, os.linesep)
                )
            )

    return 'Wrote {0} lines to "{1}"'.format(len(args), path)


def prepend(path, *args, **kwargs):
    '''
    .. versionadded:: 2014.7.0

    Prepend text to the beginning of a file

    path
        path to file

    `*args`
        strings to prepend to the file

    CLI Example:

    .. code-block:: bash

        salt '*' file.prepend /etc/motd \\
                "With all thine offerings thou shalt offer salt." \\
                "Salt is what makes things taste bad when it isn't in them."

    .. admonition:: Attention

        If you need to pass a string to append and that string contains
        an equal sign, you **must** include the argument name, args.
        For example:

        .. code-block:: bash

            salt '*' file.prepend /etc/motd args='cheese=spam'

            salt '*' file.prepend /etc/motd args="['cheese=spam','spam=cheese']"

    '''
    path = os.path.expanduser(path)

    if 'args' in kwargs:
        if isinstance(kwargs['args'], list):
            args = kwargs['args']
        else:
            args = [kwargs['args']]

    try:
        with salt.utils.files.fopen(path) as fhr:
            contents = [salt.utils.stringutils.to_unicode(line)
                        for line in fhr.readlines()]
    except IOError:
        contents = []

    preface = []
    for line in args:
        preface.append('{0}\n'.format(line))

    with salt.utils.files.fopen(path, 'w') as ofile:
        contents = preface + contents
        ofile.write(salt.utils.stringutils.to_str(''.join(contents)))
    return 'Prepended {0} lines to "{1}"'.format(len(args), path)


def write(path, *args, **kwargs):
    '''
    .. versionadded:: 2014.7.0

    Write text to a file, overwriting any existing contents.

    path
        path to file

    `*args`
        strings to write to the file

    CLI Example:

    .. code-block:: bash

        salt '*' file.write /etc/motd \\
                "With all thine offerings thou shalt offer salt."

    .. admonition:: Attention

        If you need to pass a string to append and that string contains
        an equal sign, you **must** include the argument name, args.
        For example:

        .. code-block:: bash

            salt '*' file.write /etc/motd args='cheese=spam'

            salt '*' file.write /etc/motd args="['cheese=spam','spam=cheese']"

    '''
    path = os.path.expanduser(path)

    if 'args' in kwargs:
        if isinstance(kwargs['args'], list):
            args = kwargs['args']
        else:
            args = [kwargs['args']]

    contents = []
    for line in args:
        contents.append('{0}\n'.format(line))
    with salt.utils.files.fopen(path, "w") as ofile:
        ofile.write(salt.utils.stringutils.to_str(''.join(contents)))
    return 'Wrote {0} lines to "{1}"'.format(len(contents), path)


def touch(name, atime=None, mtime=None):
    '''
    .. versionadded:: 0.9.5

    Just like the ``touch`` command, create a file if it doesn't exist or
    simply update the atime and mtime if it already does.

    atime:
        Access time in Unix epoch time
    mtime:
        Last modification in Unix epoch time

    CLI Example:

    .. code-block:: bash

        salt '*' file.touch /var/log/emptyfile
    '''
    name = os.path.expanduser(name)

    if atime and atime.isdigit():
        atime = int(atime)
    if mtime and mtime.isdigit():
        mtime = int(mtime)
    try:
        if not os.path.exists(name):
            with salt.utils.files.fopen(name, 'a'):
                pass

        if not atime and not mtime:
            times = None
        elif not mtime and atime:
            times = (atime, time.time())
        elif not atime and mtime:
            times = (time.time(), mtime)
        else:
            times = (atime, mtime)
        os.utime(name, times)

    except TypeError:
        raise SaltInvocationError('atime and mtime must be integers')
    except (IOError, OSError) as exc:
        raise CommandExecutionError(exc.strerror)

    return os.path.exists(name)


def seek_read(path, size, offset):
    '''
    .. versionadded:: 2014.1.0

    Seek to a position on a file and read it

    path
        path to file

    seek
        amount to read at once

    offset
        offset to start into the file

    CLI Example:

    .. code-block:: bash

        salt '*' file.seek_read /path/to/file 4096 0
    '''
    path = os.path.expanduser(path)
    seek_fh = os.open(path, os.O_RDONLY)
    try:
        os.lseek(seek_fh, int(offset), 0)
        data = os.read(seek_fh, int(size))
    finally:
        os.close(seek_fh)
    return data


def seek_write(path, data, offset):
    '''
    .. versionadded:: 2014.1.0

    Seek to a position on a file and write to it

    path
        path to file

    data
        data to write to file

    offset
        position in file to start writing

    CLI Example:

    .. code-block:: bash

        salt '*' file.seek_write /path/to/file 'some data' 4096
    '''
    path = os.path.expanduser(path)
    seek_fh = os.open(path, os.O_WRONLY)
    try:
        os.lseek(seek_fh, int(offset), 0)
        ret = os.write(seek_fh, data)
        os.fsync(seek_fh)
    finally:
        os.close(seek_fh)
    return ret


def truncate(path, length):
    '''
    .. versionadded:: 2014.1.0

    Seek to a position on a file and delete everything after that point

    path
        path to file

    length
        offset into file to truncate

    CLI Example:

    .. code-block:: bash

        salt '*' file.truncate /path/to/file 512
    '''
    path = os.path.expanduser(path)
    with salt.utils.files.fopen(path, 'rb+') as seek_fh:
        seek_fh.truncate(int(length))


def link(src, path):
    '''
    .. versionadded:: 2014.1.0

    Create a hard link to a file

    CLI Example:

    .. code-block:: bash

        salt '*' file.link /path/to/file /path/to/link
    '''
    src = os.path.expanduser(src)

    if not os.path.isabs(src):
        raise SaltInvocationError('File path must be absolute.')

    try:
        os.link(src, path)
        return True
    except (OSError, IOError):
        raise CommandExecutionError('Could not create \'{0}\''.format(path))
    return False


def is_link(path):
    '''
    Check if the path is a symbolic link

    CLI Example:

    .. code-block:: bash

       salt '*' file.is_link /path/to/link
    '''
    # This function exists because os.path.islink does not support Windows,
    # therefore a custom function will need to be called. This function
    # therefore helps API consistency by providing a single function to call for
    # both operating systems.

    return os.path.islink(os.path.expanduser(path))


def symlink(src, path):
    '''
    Create a symbolic link (symlink, soft link) to a file

    CLI Example:

    .. code-block:: bash

        salt '*' file.symlink /path/to/file /path/to/link
    '''
    path = os.path.expanduser(path)

    try:
        if os.path.normpath(os.readlink(path)) == os.path.normpath(src):
            log.debug('link already in correct state: %s -> %s', path, src)
            return True
    except OSError:
        pass

    if not os.path.isabs(path):
        raise SaltInvocationError('File path must be absolute.')

    try:
        os.symlink(src, path)
        return True
    except (OSError, IOError):
        raise CommandExecutionError('Could not create \'{0}\''.format(path))
    return False


def rename(src, dst):
    '''
    Rename a file or directory

    CLI Example:

    .. code-block:: bash

        salt '*' file.rename /path/to/src /path/to/dst
    '''
    src = os.path.expanduser(src)
    dst = os.path.expanduser(dst)

    if not os.path.isabs(src):
        raise SaltInvocationError('File path must be absolute.')

    try:
        os.rename(src, dst)
        return True
    except OSError:
        raise CommandExecutionError(
            'Could not rename \'{0}\' to \'{1}\''.format(src, dst)
        )
    return False


def copy(src, dst, recurse=False, remove_existing=False):
    '''
    Copy a file or directory from source to dst

    In order to copy a directory, the recurse flag is required, and
    will by default overwrite files in the destination with the same path,
    and retain all other existing files. (similar to cp -r on unix)

    remove_existing will remove all files in the target directory,
    and then copy files from the source.

    .. note::
        The copy function accepts paths that are local to the Salt minion.
        This function does not support salt://, http://, or the other
        additional file paths that are supported by :mod:`states.file.managed
        <salt.states.file.managed>` and :mod:`states.file.recurse
        <salt.states.file.recurse>`.

    CLI Example:

    .. code-block:: bash

        salt '*' file.copy /path/to/src /path/to/dst
        salt '*' file.copy /path/to/src_dir /path/to/dst_dir recurse=True
        salt '*' file.copy /path/to/src_dir /path/to/dst_dir recurse=True remove_existing=True

    '''
    src = os.path.expanduser(src)
    dst = os.path.expanduser(dst)

    if not os.path.isabs(src):
        raise SaltInvocationError('File path must be absolute.')

    if not os.path.exists(src):
        raise CommandExecutionError('No such file or directory \'{0}\''.format(src))

    if not salt.utils.platform.is_windows():
        pre_user = get_user(src)
        pre_group = get_group(src)
        pre_mode = salt.utils.files.normalize_mode(get_mode(src))

    try:
        if (os.path.exists(dst) and os.path.isdir(dst)) or os.path.isdir(src):
            if not recurse:
                raise SaltInvocationError(
                    "Cannot copy overwriting a directory without recurse flag set to true!")
            if remove_existing:
                if os.path.exists(dst):
                    shutil.rmtree(dst)
                shutil.copytree(src, dst)
            else:
                salt.utils.files.recursive_copy(src, dst)
        else:
            shutil.copyfile(src, dst)
    except OSError:
        raise CommandExecutionError(
            'Could not copy \'{0}\' to \'{1}\''.format(src, dst)
        )

    if not salt.utils.platform.is_windows():
        check_perms(dst, None, pre_user, pre_group, pre_mode)
    return True


def lstat(path):
    '''
    .. versionadded:: 2014.1.0

    Returns the lstat attributes for the given file or dir. Does not support
    symbolic links.

    CLI Example:

    .. code-block:: bash

        salt '*' file.lstat /path/to/file
    '''
    path = os.path.expanduser(path)

    if not os.path.isabs(path):
        raise SaltInvocationError('Path to file must be absolute.')

    try:
        lst = os.lstat(path)
        return dict((key, getattr(lst, key)) for key in ('st_atime', 'st_ctime',
            'st_gid', 'st_mode', 'st_mtime', 'st_nlink', 'st_size', 'st_uid'))
    except Exception:
        return {}


def access(path, mode):
    '''
    .. versionadded:: 2014.1.0

    Test whether the Salt process has the specified access to the file. One of
    the following modes must be specified:

    .. code-block::text

        f: Test the existence of the path
        r: Test the readability of the path
        w: Test the writability of the path
        x: Test whether the path can be executed

    CLI Example:

    .. code-block:: bash

        salt '*' file.access /path/to/file f
        salt '*' file.access /path/to/file x
    '''
    path = os.path.expanduser(path)

    if not os.path.isabs(path):
        raise SaltInvocationError('Path to link must be absolute.')

    modes = {'f': os.F_OK,
             'r': os.R_OK,
             'w': os.W_OK,
             'x': os.X_OK}

    if mode in modes:
        return os.access(path, modes[mode])
    elif mode in six.itervalues(modes):
        return os.access(path, mode)
    else:
        raise SaltInvocationError('Invalid mode specified.')


def read(path, binary=False):
    '''
    .. versionadded:: 2017.7.0

    Return the content of the file.

    CLI Example:

    .. code-block:: bash

        salt '*' file.read /path/to/file
    '''
    access_mode = 'r'
    if binary is True:
        access_mode += 'b'
    with salt.utils.files.fopen(path, access_mode) as file_obj:
        return salt.utils.stringutils.to_unicode(file_obj.read())


def readlink(path, canonicalize=False):
    '''
    .. versionadded:: 2014.1.0

    Return the path that a symlink points to
    If canonicalize is set to True, then it return the final target

    CLI Example:

    .. code-block:: bash

        salt '*' file.readlink /path/to/link
    '''
    path = os.path.expanduser(path)

    if not os.path.isabs(path):
        raise SaltInvocationError('Path to link must be absolute.')

    if not os.path.islink(path):
        raise SaltInvocationError('A valid link was not specified.')

    if canonicalize:
        return os.path.realpath(path)
    else:
        return os.readlink(path)


def readdir(path):
    '''
    .. versionadded:: 2014.1.0

    Return a list containing the contents of a directory

    CLI Example:

    .. code-block:: bash

        salt '*' file.readdir /path/to/dir/
    '''
    path = os.path.expanduser(path)

    if not os.path.isabs(path):
        raise SaltInvocationError('Dir path must be absolute.')

    if not os.path.isdir(path):
        raise SaltInvocationError('A valid directory was not specified.')

    dirents = ['.', '..']
    dirents.extend(os.listdir(path))
    return dirents


def statvfs(path):
    '''
    .. versionadded:: 2014.1.0

    Perform a statvfs call against the filesystem that the file resides on

    CLI Example:

    .. code-block:: bash

        salt '*' file.statvfs /path/to/file
    '''
    path = os.path.expanduser(path)

    if not os.path.isabs(path):
        raise SaltInvocationError('File path must be absolute.')

    try:
        stv = os.statvfs(path)
        return dict((key, getattr(stv, key)) for key in ('f_bavail', 'f_bfree',
            'f_blocks', 'f_bsize', 'f_favail', 'f_ffree', 'f_files', 'f_flag',
            'f_frsize', 'f_namemax'))
    except (OSError, IOError):
        raise CommandExecutionError('Could not statvfs \'{0}\''.format(path))
    return False


def stats(path, hash_type=None, follow_symlinks=True):
    '''
    Return a dict containing the stats for a given file

    CLI Example:

    .. code-block:: bash

        salt '*' file.stats /etc/passwd
    '''
    path = os.path.expanduser(path)

    ret = {}
    if not os.path.exists(path):
        try:
            # Broken symlinks will return False for os.path.exists(), but still
            # have a uid and gid
            pstat = os.lstat(path)
        except OSError:
            # Not a broken symlink, just a nonexistent path
            # NOTE: The file.directory state checks the content of the error
            # message in this exception. Any changes made to the message for this
            # exception will reflect the file.directory state as well, and will
            # likely require changes there.
            raise CommandExecutionError('Path not found: {0}'.format(path))
    else:
        if follow_symlinks:
            pstat = os.stat(path)
        else:
            pstat = os.lstat(path)
    ret['inode'] = pstat.st_ino
    ret['uid'] = pstat.st_uid
    ret['gid'] = pstat.st_gid
    ret['group'] = gid_to_group(pstat.st_gid)
    ret['user'] = uid_to_user(pstat.st_uid)
    ret['atime'] = pstat.st_atime
    ret['mtime'] = pstat.st_mtime
    ret['ctime'] = pstat.st_ctime
    ret['size'] = pstat.st_size
    ret['mode'] = six.text_type(oct(stat.S_IMODE(pstat.st_mode)))
    if hash_type:
        ret['sum'] = get_hash(path, hash_type)
    ret['type'] = 'file'
    if stat.S_ISDIR(pstat.st_mode):
        ret['type'] = 'dir'
    if stat.S_ISCHR(pstat.st_mode):
        ret['type'] = 'char'
    if stat.S_ISBLK(pstat.st_mode):
        ret['type'] = 'block'
    if stat.S_ISREG(pstat.st_mode):
        ret['type'] = 'file'
    if stat.S_ISLNK(pstat.st_mode):
        ret['type'] = 'link'
    if stat.S_ISFIFO(pstat.st_mode):
        ret['type'] = 'pipe'
    if stat.S_ISSOCK(pstat.st_mode):
        ret['type'] = 'socket'
    ret['target'] = os.path.realpath(path)
    return ret


def rmdir(path):
    '''
    .. versionadded:: 2014.1.0

    Remove the specified directory. Fails if a directory is not empty.

    CLI Example:

    .. code-block:: bash

        salt '*' file.rmdir /tmp/foo/
    '''
    path = os.path.expanduser(path)

    if not os.path.isabs(path):
        raise SaltInvocationError('File path must be absolute.')

    if not os.path.isdir(path):
        raise SaltInvocationError('A valid directory was not specified.')

    try:
        os.rmdir(path)
        return True
    except OSError as exc:
        return exc.strerror


def remove(path):
    '''
    Remove the named file. If a directory is supplied, it will be recursively
    deleted.

    CLI Example:

    .. code-block:: bash

        salt '*' file.remove /tmp/foo
    '''
    path = os.path.expanduser(path)

    if not os.path.isabs(path):
        raise SaltInvocationError('File path must be absolute: {0}'.format(path))

    try:
        if os.path.isfile(path) or os.path.islink(path):
            os.remove(path)
            return True
        elif os.path.isdir(path):
            shutil.rmtree(path)
            return True
    except (OSError, IOError) as exc:
        raise CommandExecutionError(
            'Could not remove \'{0}\': {1}'.format(path, exc)
        )
    return False


def directory_exists(path):
    '''
    Tests to see if path is a valid directory.  Returns True/False.

    CLI Example:

    .. code-block:: bash

        salt '*' file.directory_exists /etc

    '''
    return os.path.isdir(os.path.expanduser(path))


def file_exists(path):
    '''
    Tests to see if path is a valid file.  Returns True/False.

    CLI Example:

    .. code-block:: bash

        salt '*' file.file_exists /etc/passwd

    '''
    return os.path.isfile(os.path.expanduser(path))


def path_exists_glob(path):
    '''
    Tests to see if path after expansion is a valid path (file or directory).
    Expansion allows usage of ? * and character ranges []. Tilde expansion
    is not supported. Returns True/False.

    .. versionadded:: 2014.7.0

    CLI Example:

    .. code-block:: bash

        salt '*' file.path_exists_glob /etc/pam*/pass*

    '''
    return True if glob.glob(os.path.expanduser(path)) else False


def restorecon(path, recursive=False):
    '''
    Reset the SELinux context on a given path

    CLI Example:

    .. code-block:: bash

         salt '*' file.restorecon /home/user/.ssh/authorized_keys
    '''
    if recursive:
        cmd = ['restorecon', '-FR', path]
    else:
        cmd = ['restorecon', '-F', path]
    return not __salt__['cmd.retcode'](cmd, python_shell=False)


def get_selinux_context(path):
    '''
    Get an SELinux context from a given path

    CLI Example:

    .. code-block:: bash

        salt '*' file.get_selinux_context /etc/hosts
    '''
    out = __salt__['cmd.run'](['ls', '-Z', path], python_shell=False)

    try:
        ret = re.search(r'\w+:\w+:\w+:\w+', out).group(0)
    except AttributeError:
        ret = (
            'No selinux context information is available for {0}'.format(path)
        )

    return ret


def set_selinux_context(path,
                        user=None,
                        role=None,
                        type=None,    # pylint: disable=W0622
                        range=None):  # pylint: disable=W0622
    '''
    Set a specific SELinux label on a given path

    CLI Example:

    .. code-block:: bash

        salt '*' file.set_selinux_context path <user> <role> <type> <range>
        salt '*' file.set_selinux_context /etc/yum.repos.d/epel.repo system_u object_r system_conf_t s0
    '''
    if not any((user, role, type, range)):
        return False

    cmd = ['chcon']
    if user:
        cmd.extend(['-u', user])
    if role:
        cmd.extend(['-r', role])
    if type:
        cmd.extend(['-t', type])
    if range:
        cmd.extend(['-l', range])
    cmd.append(path)

    ret = not __salt__['cmd.retcode'](cmd, python_shell=False)
    if ret:
        return get_selinux_context(path)
    else:
        return ret


def source_list(source, source_hash, saltenv):
    '''
    Check the source list and return the source to use

    CLI Example:

    .. code-block:: bash

        salt '*' file.source_list salt://http/httpd.conf '{hash_type: 'md5', 'hsum': <md5sum>}' base
    '''
    contextkey = '{0}_|-{1}_|-{2}'.format(source, source_hash, saltenv)
    if contextkey in __context__:
        return __context__[contextkey]

    # get the master file list
    if isinstance(source, list):
        mfiles = [(f, saltenv) for f in __salt__['cp.list_master'](saltenv)]
        mdirs = [(d, saltenv) for d in __salt__['cp.list_master_dirs'](saltenv)]
        for single in source:
            if isinstance(single, dict):
                single = next(iter(single))

            path, senv = salt.utils.url.parse(single)
            if senv:
                mfiles += [(f, senv) for f in __salt__['cp.list_master'](senv)]
                mdirs += [(d, senv) for d in __salt__['cp.list_master_dirs'](senv)]

        ret = None
        for single in source:
            if isinstance(single, dict):
                # check the proto, if it is http or ftp then download the file
                # to check, if it is salt then check the master list
                # if it is a local file, check if the file exists
                if len(single) != 1:
                    continue
                single_src = next(iter(single))
                single_hash = single[single_src] if single[single_src] else source_hash
                urlparsed_single_src = _urlparse(single_src)
                # Fix this for Windows
                if salt.utils.platform.is_windows():
                    # urlparse doesn't handle a local Windows path without the
                    # protocol indicator (file://). The scheme will be the
                    # drive letter instead of the protocol. So, we'll add the
                    # protocol and re-parse
                    if urlparsed_single_src.scheme.lower() in string.ascii_lowercase:
                        urlparsed_single_src = _urlparse('file://' + single_src)
                proto = urlparsed_single_src.scheme
                if proto == 'salt':
                    path, senv = salt.utils.url.parse(single_src)
                    if not senv:
                        senv = saltenv
                    if (path, saltenv) in mfiles or (path, saltenv) in mdirs:
                        ret = (single_src, single_hash)
                        break
                elif proto.startswith('http') or proto == 'ftp':
                    ret = (single_src, single_hash)
                    break
                elif proto == 'file' and (
                         os.path.exists(urlparsed_single_src.netloc) or
                         os.path.exists(urlparsed_single_src.path) or
                         os.path.exists(os.path.join(
                             urlparsed_single_src.netloc,
                             urlparsed_single_src.path))):
                    ret = (single_src, single_hash)
                    break
                elif single_src.startswith(os.sep) and os.path.exists(single_src):
                    ret = (single_src, single_hash)
                    break
            elif isinstance(single, six.string_types):
                path, senv = salt.utils.url.parse(single)
                if not senv:
                    senv = saltenv
                if (path, senv) in mfiles or (path, senv) in mdirs:
                    ret = (single, source_hash)
                    break
                urlparsed_src = _urlparse(single)
                if salt.utils.platform.is_windows():
                    # urlparse doesn't handle a local Windows path without the
                    # protocol indicator (file://). The scheme will be the
                    # drive letter instead of the protocol. So, we'll add the
                    # protocol and re-parse
                    if urlparsed_src.scheme.lower() in string.ascii_lowercase:
                        urlparsed_src = _urlparse('file://' + single)
                proto = urlparsed_src.scheme
                if proto == 'file' and (
                        os.path.exists(urlparsed_src.netloc) or
                        os.path.exists(urlparsed_src.path) or
                        os.path.exists(os.path.join(
                            urlparsed_src.netloc,
                            urlparsed_src.path))):
                    ret = (single, source_hash)
                    break
                elif proto.startswith('http') or proto == 'ftp':
                    ret = (single, source_hash)
                    break
                elif single.startswith(os.sep) and os.path.exists(single):
                    ret = (single, source_hash)
                    break
        if ret is None:
            # None of the list items matched
            raise CommandExecutionError(
                'none of the specified sources were found'
            )
    else:
        ret = (source, source_hash)

    __context__[contextkey] = ret
    return ret


def apply_template_on_contents(
        contents,
        template,
        context,
        defaults,
        saltenv):
    '''
    Return the contents after applying the templating engine

    contents
        template string

    template
        template format

    context
        Overrides default context variables passed to the template.

    defaults
        Default context passed to the template.

    CLI Example:

    .. code-block:: bash

        salt '*' file.apply_template_on_contents \\
            contents='This is a {{ template }} string.' \\
            template=jinja \\
            "context={}" "defaults={'template': 'cool'}" \\
            saltenv=base
    '''
    if template in salt.utils.templates.TEMPLATE_REGISTRY:
        context_dict = defaults if defaults else {}
        if context:
            context_dict.update(context)
        # Apply templating
        contents = salt.utils.templates.TEMPLATE_REGISTRY[template](
            contents,
            from_str=True,
            to_str=True,
            context=context_dict,
            saltenv=saltenv,
            grains=__opts__['grains'],
            pillar=__pillar__,
            salt=__salt__,
            opts=__opts__)['data']
        if six.PY2:
            contents = contents.encode('utf-8')
        elif six.PY3 and isinstance(contents, bytes):
            # bytes -> str
            contents = contents.decode('utf-8')
    else:
        ret = {}
        ret['result'] = False
        ret['comment'] = ('Specified template format {0} is not supported'
                          ).format(template)
        return ret
    return contents


def get_managed(
        name,
        template,
        source,
        source_hash,
        source_hash_name,
        user,
        group,
        mode,
        attrs,
        saltenv,
        context,
        defaults,
        skip_verify=False,
        **kwargs):
    '''
    Return the managed file data for file.managed

    name
        location where the file lives on the server

    template
        template format

    source
        managed source file

    source_hash
        hash of the source file

    source_hash_name
        When ``source_hash`` refers to a remote file, this specifies the
        filename to look for in that file.

        .. versionadded:: 2016.3.5

    user
        Owner of file

    group
        Group owner of file

    mode
        Permissions of file

    attrs
        Attributes of file

        .. versionadded:: 2018.3.0

    context
        Variables to add to the template context

    defaults
        Default values of for context_dict

    skip_verify
        If ``True``, hash verification of remote file sources (``http://``,
        ``https://``, ``ftp://``) will be skipped, and the ``source_hash``
        argument will be ignored.

        .. versionadded:: 2016.3.0

    CLI Example:

    .. code-block:: bash

        salt '*' file.get_managed /etc/httpd/conf.d/httpd.conf jinja salt://http/httpd.conf '{hash_type: 'md5', 'hsum': <md5sum>}' None root root '755' base None None
    '''
    # Copy the file to the minion and templatize it
    sfn = ''
    source_sum = {}

    def _get_local_file_source_sum(path):
        '''
        DRY helper for getting the source_sum value from a locally cached
        path.
        '''
        return {'hsum': get_hash(path, form='sha256'), 'hash_type': 'sha256'}

    # If we have a source defined, let's figure out what the hash is
    if source:
        urlparsed_source = _urlparse(source)
        parsed_scheme = urlparsed_source.scheme
        parsed_path = os.path.join(
                urlparsed_source.netloc, urlparsed_source.path).rstrip(os.sep)
        unix_local_source = parsed_scheme in ('file', '')

        if unix_local_source:
            sfn = parsed_path
            if not os.path.exists(sfn):
                msg = 'Local file source {0} does not exist'.format(sfn)
                return '', {}, msg

        if parsed_scheme and parsed_scheme.lower() in string.ascii_lowercase:
            parsed_path = ':'.join([parsed_scheme, parsed_path])
            parsed_scheme = 'file'

        if parsed_scheme == 'salt':
            source_sum = __salt__['cp.hash_file'](source, saltenv)
            if not source_sum:
                return '', {}, 'Source file {0} not found'.format(source)
        elif not source_hash and unix_local_source:
            source_sum = _get_local_file_source_sum(parsed_path)
        elif not source_hash and source.startswith(os.sep):
            # This should happen on Windows
            source_sum = _get_local_file_source_sum(source)
        else:
            if not skip_verify:
                if source_hash:
                    try:
                        source_sum = get_source_sum(name,
                                                    source,
                                                    source_hash,
                                                    source_hash_name,
                                                    saltenv)
                    except CommandExecutionError as exc:
                        return '', {}, exc.strerror
                else:
                    msg = (
                        'Unable to verify upstream hash of source file {0}, '
                        'please set source_hash or set skip_verify to True'
                        .format(source)
                    )
                    return '', {}, msg

    if source and (template or parsed_scheme in salt.utils.files.REMOTE_PROTOS):
        # Check if we have the template or remote file cached
        cache_refetch = False
        cached_dest = __salt__['cp.is_cached'](source, saltenv)
        if cached_dest and (source_hash or skip_verify):
            htype = source_sum.get('hash_type', 'sha256')
            cached_sum = get_hash(cached_dest, form=htype)
            if skip_verify:
                # prev: if skip_verify or cached_sum == source_sum['hsum']:
                # but `cached_sum == source_sum['hsum']` is elliptical as prev if
                sfn = cached_dest
                source_sum = {'hsum': cached_sum, 'hash_type': htype}
            elif cached_sum != source_sum.get('hsum', __opts__['hash_type']):
                cache_refetch = True
            else:
                sfn = cached_dest

        # If we didn't have the template or remote file, or the file has been
        # updated and the cache has to be refreshed, download the file.
        if not sfn or cache_refetch:
            try:
                sfn = __salt__['cp.cache_file'](
                    source,
                    saltenv,
                    source_hash=source_sum.get('hsum'))
            except Exception as exc:
                # A 404 or other error code may raise an exception, catch it
                # and return a comment that will fail the calling state.
                return '', {}, 'Failed to cache {0}: {1}'.format(source, exc)

        # If cache failed, sfn will be False, so do a truth check on sfn first
        # as invoking os.path.exists() on a bool raises a TypeError.
        if not sfn or not os.path.exists(sfn):
            return sfn, {}, 'Source file \'{0}\' not found'.format(source)
        if sfn == name:
            raise SaltInvocationError(
                'Source file cannot be the same as destination'
            )

        if template:
            if template in salt.utils.templates.TEMPLATE_REGISTRY:
                context_dict = defaults if defaults else {}
                if context:
                    context_dict.update(context)
                data = salt.utils.templates.TEMPLATE_REGISTRY[template](
                    sfn,
                    name=name,
                    source=source,
                    user=user,
                    group=group,
                    mode=mode,
                    attrs=attrs,
                    saltenv=saltenv,
                    context=context_dict,
                    salt=__salt__,
                    pillar=__pillar__,
                    grains=__opts__['grains'],
                    opts=__opts__,
                    **kwargs)
            else:
                return sfn, {}, ('Specified template format {0} is not supported'
                                 ).format(template)

            if data['result']:
                sfn = data['data']
                hsum = get_hash(sfn, form='sha256')
                source_sum = {'hash_type': 'sha256',
                              'hsum': hsum}
            else:
                __clean_tmp(sfn)
                return sfn, {}, data['data']

    return sfn, source_sum, ''


def extract_hash(hash_fn,
                 hash_type='sha256',
                 file_name='',
                 source='',
                 source_hash_name=None):
    '''
    .. versionchanged:: 2016.3.5
        Prior to this version, only the ``file_name`` argument was considered
        for filename matches in the hash file. This would be problematic for
        cases in which the user was relying on a remote checksum file that they
        do not control, and they wished to use a different name for that file
        on the minion from the filename on the remote server (and in the
        checksum file). For example, managing ``/tmp/myfile.tar.gz`` when the
        remote file was at ``https://mydomain.tld/different_name.tar.gz``. The
        :py:func:`file.managed <salt.states.file.managed>` state now also
        passes this function the source URI as well as the ``source_hash_name``
        (if specified). In cases where ``source_hash_name`` is specified, it
        takes precedence over both the ``file_name`` and ``source``. When it is
        not specified, ``file_name`` takes precedence over ``source``. This
        allows for better capability for matching hashes.
    .. versionchanged:: 2016.11.0
        File name and source URI matches are no longer disregarded when
        ``source_hash_name`` is specified. They will be used as fallback
        matches if there is no match to the ``source_hash_name`` value.

    This routine is called from the :mod:`file.managed
    <salt.states.file.managed>` state to pull a hash from a remote file.
    Regular expressions are used line by line on the ``source_hash`` file, to
    find a potential candidate of the indicated hash type. This avoids many
    problems of arbitrary file layout rules. It specifically permits pulling
    hash codes from debian ``*.dsc`` files.

    If no exact match of a hash and filename are found, then the first hash
    found (if any) will be returned. If no hashes at all are found, then
    ``None`` will be returned.

    For example:

    .. code-block:: yaml

        openerp_7.0-latest-1.tar.gz:
          file.managed:
            - name: /tmp/openerp_7.0-20121227-075624-1_all.deb
            - source: http://nightly.openerp.com/7.0/nightly/deb/openerp_7.0-20121227-075624-1.tar.gz
            - source_hash: http://nightly.openerp.com/7.0/nightly/deb/openerp_7.0-20121227-075624-1.dsc

    CLI Example:

    .. code-block:: bash

        salt '*' file.extract_hash /path/to/hash/file sha512 /etc/foo
    '''
    hash_len = HASHES.get(hash_type)
    if hash_len is None:
        if hash_type:
            log.warning(
                'file.extract_hash: Unsupported hash_type \'%s\', falling '
                'back to matching any supported hash_type', hash_type
            )
            hash_type = ''
        hash_len_expr = '{0},{1}'.format(min(HASHES_REVMAP), max(HASHES_REVMAP))
    else:
        hash_len_expr = six.text_type(hash_len)

    filename_separators = string.whitespace + r'\/'

    if source_hash_name:
        if not isinstance(source_hash_name, six.string_types):
            source_hash_name = six.text_type(source_hash_name)
        source_hash_name_idx = (len(source_hash_name) + 1) * -1
        log.debug(
            'file.extract_hash: Extracting %s hash for file matching '
            'source_hash_name \'%s\'',
            'any supported' if not hash_type else hash_type,
            source_hash_name
        )
    if file_name:
        if not isinstance(file_name, six.string_types):
            file_name = six.text_type(file_name)
        file_name_basename = os.path.basename(file_name)
        file_name_idx = (len(file_name_basename) + 1) * -1
    if source:
        if not isinstance(source, six.string_types):
            source = six.text_type(source)
        urlparsed_source = _urlparse(source)
        source_basename = os.path.basename(
            urlparsed_source.path or urlparsed_source.netloc
        )
        source_idx = (len(source_basename) + 1) * -1

    basename_searches = [x for x in (file_name, source) if x]
    if basename_searches:
        log.debug(
            'file.extract_hash: %s %s hash for file matching%s: %s',
            'If no source_hash_name match found, will extract'
                if source_hash_name
                else 'Extracting',
            'any supported' if not hash_type else hash_type,
            '' if len(basename_searches) == 1 else ' either of the following',
            ', '.join(basename_searches)
        )

    partial = None
    found = {}

    with salt.utils.files.fopen(hash_fn, 'r') as fp_:
        for line in fp_:
            line = salt.utils.stringutils.to_unicode(line.strip())
            hash_re = r'(?i)(?<![a-z0-9])([a-f0-9]{' + hash_len_expr + '})(?![a-z0-9])'
            hash_match = re.search(hash_re, line)
            matched = None
            if hash_match:
                matched_hsum = hash_match.group(1)
                if matched_hsum is not None:
                    matched_type = HASHES_REVMAP.get(len(matched_hsum))
                    if matched_type is None:
                        # There was a match, but it's not of the correct length
                        # to match one of the supported hash types.
                        matched = None
                    else:
                        matched = {'hsum': matched_hsum,
                                   'hash_type': matched_type}

            if matched is None:
                log.debug(
                    'file.extract_hash: In line \'%s\', no %shash found',
                    line,
                    '' if not hash_type else hash_type + ' '
                )
                continue

            if partial is None:
                partial = matched

            def _add_to_matches(found, line, match_type, value, matched):
                log.debug(
                    'file.extract_hash: Line \'%s\' matches %s \'%s\'',
                    line, match_type, value
                )
                found.setdefault(match_type, []).append(matched)

            hash_matched = False
            if source_hash_name:
                if line.endswith(source_hash_name):
                    # Checking the character before where the basename
                    # should start for either whitespace or a path
                    # separator. We can't just rsplit on spaces/whitespace,
                    # because the filename may contain spaces.
                    try:
                        if line[source_hash_name_idx] in string.whitespace:
                            _add_to_matches(found, line, 'source_hash_name',
                                            source_hash_name, matched)
                            hash_matched = True
                    except IndexError:
                        pass
                elif re.match(re.escape(source_hash_name) + r'\s+',
                              line):
                    _add_to_matches(found, line, 'source_hash_name',
                                    source_hash_name, matched)
                    hash_matched = True
            if file_name:
                if line.endswith(file_name_basename):
                    # Checking the character before where the basename
                    # should start for either whitespace or a path
                    # separator. We can't just rsplit on spaces/whitespace,
                    # because the filename may contain spaces.
                    try:
                        if line[file_name_idx] in filename_separators:
                            _add_to_matches(found, line, 'file_name',
                                            file_name, matched)
                            hash_matched = True
                    except IndexError:
                        pass
                elif re.match(re.escape(file_name) + r'\s+', line):
                    _add_to_matches(found, line, 'file_name',
                                    file_name, matched)
                    hash_matched = True
            if source:
                if line.endswith(source_basename):
                    # Same as above, we can't just do an rsplit here.
                    try:
                        if line[source_idx] in filename_separators:
                            _add_to_matches(found, line, 'source',
                                            source, matched)
                            hash_matched = True
                    except IndexError:
                        pass
                elif re.match(re.escape(source) + r'\s+', line):
                    _add_to_matches(found, line, 'source', source, matched)
                    hash_matched = True

            if not hash_matched:
                log.debug(
                    'file.extract_hash: Line \'%s\' contains %s hash '
                    '\'%s\', but line did not meet the search criteria',
                    line, matched['hash_type'], matched['hsum']
                )

    for found_type, found_str in (('source_hash_name', source_hash_name),
                                  ('file_name', file_name),
                                  ('source', source)):
        if found_type in found:
            if len(found[found_type]) > 1:
                log.debug(
                    'file.extract_hash: Multiple %s matches for %s: %s',
                    found_type,
                    found_str,
                    ', '.join(
                        ['{0} ({1})'.format(x['hsum'], x['hash_type'])
                         for x in found[found_type]]
                    )
                )
            ret = found[found_type][0]
            log.debug(
                'file.extract_hash: Returning %s hash \'%s\' as a match of %s',
                ret['hash_type'], ret['hsum'], found_str
            )
            return ret

    if partial:
        log.debug(
            'file.extract_hash: Returning the partially identified %s hash '
            '\'%s\'', partial['hash_type'], partial['hsum']
        )
        return partial

    log.debug('file.extract_hash: No matches, returning None')
    return None


def check_perms(name, ret, user, group, mode, attrs=None, follow_symlinks=False):
    '''
    Check the permissions on files, modify attributes and chown if needed. File
    attributes are only verified if lsattr(1) is installed.

    CLI Example:

    .. code-block:: bash

        salt '*' file.check_perms /etc/sudoers '{}' root root 400 ai

    .. versionchanged:: 2014.1.3
        ``follow_symlinks`` option added
    '''
    name = os.path.expanduser(name)

    if not ret:
        ret = {'name': name,
               'changes': {},
               'comment': [],
               'result': True}
        orig_comment = ''
    else:
        orig_comment = ret['comment']
        ret['comment'] = []

    # Check permissions
    perms = {}
    cur = stats(name, follow_symlinks=follow_symlinks)
    perms['luser'] = cur['user']
    perms['lgroup'] = cur['group']
    perms['lmode'] = salt.utils.files.normalize_mode(cur['mode'])

    is_dir = os.path.isdir(name)
    if not salt.utils.platform.is_windows() and not is_dir:
        lattrs = lsattr(name)
        if lattrs is not None:
            # List attributes on file
            perms['lattrs'] = ''.join(lattrs.get(name, ''))
            # Remove attributes on file so changes can be enforced.
            if perms['lattrs']:
                chattr(name, operator='remove', attributes=perms['lattrs'])

    # user/group changes if needed, then check if it worked
    if user:
        if isinstance(user, int):
            user = uid_to_user(user)
        if (salt.utils.platform.is_windows() and
                user_to_uid(user) != user_to_uid(perms['luser'])
            ) or (
            not salt.utils.platform.is_windows() and user != perms['luser']
        ):
            perms['cuser'] = user

    if group:
        if isinstance(group, int):
            group = gid_to_group(group)
        if (salt.utils.platform.is_windows() and
                group_to_gid(group) != group_to_gid(perms['lgroup'])
            ) or (
                not salt.utils.platform.is_windows() and group != perms['lgroup']
        ):
            perms['cgroup'] = group

    if 'cuser' in perms or 'cgroup' in perms:
        if not __opts__['test']:
            if os.path.islink(name) and not follow_symlinks:
                chown_func = lchown
            else:
                chown_func = chown
            if user is None:
                user = perms['luser']
            if group is None:
                group = perms['lgroup']
            try:
                chown_func(name, user, group)
            except OSError:
                ret['result'] = False

    if user:
        if isinstance(user, int):
            user = uid_to_user(user)
        if (salt.utils.platform.is_windows() and
                user_to_uid(user) != user_to_uid(
                    get_user(name, follow_symlinks=follow_symlinks)) and
                user != ''
            ) or (
            not salt.utils.platform.is_windows() and
                user != get_user(name, follow_symlinks=follow_symlinks) and
                user != ''
        ):
            if __opts__['test'] is True:
                ret['changes']['user'] = user
            else:
                ret['result'] = False
                ret['comment'].append('Failed to change user to {0}'
                                          .format(user))
        elif 'cuser' in perms and user != '':
            ret['changes']['user'] = user

    if group:
        if isinstance(group, int):
            group = gid_to_group(group)
        if (salt.utils.platform.is_windows() and
                group_to_gid(group) != group_to_gid(
                    get_group(name, follow_symlinks=follow_symlinks)) and
                user != '') or (
            not salt.utils.platform.is_windows() and
                group != get_group(name, follow_symlinks=follow_symlinks) and
                user != ''
        ):
            if __opts__['test'] is True:
                ret['changes']['group'] = group
            else:
                ret['result'] = False
                ret['comment'].append('Failed to change group to {0}'
                                      .format(group))
        elif 'cgroup' in perms and user != '':
            ret['changes']['group'] = group

    if not salt.utils.platform.is_windows() and not is_dir:
        # Replace attributes on file if it had been removed
        if perms.get('lattrs', ''):
            chattr(name, operator='add', attributes=perms['lattrs'])

    # Mode changes if needed
    if mode is not None:
        # File is a symlink, ignore the mode setting
        # if follow_symlinks is False
        if os.path.islink(name) and not follow_symlinks:
            pass
        else:
            mode = salt.utils.files.normalize_mode(mode)
            if mode != perms['lmode']:
                if __opts__['test'] is True:
                    ret['changes']['mode'] = mode
                else:
                    set_mode(name, mode)
                    if mode != salt.utils.files.normalize_mode(get_mode(name)):
                        ret['result'] = False
                        ret['comment'].append(
                            'Failed to change mode to {0}'.format(mode)
                        )
                    else:
                        ret['changes']['mode'] = mode

    # Modify attributes of file if needed
    if attrs is not None and not is_dir:
        # File is a symlink, ignore the mode setting
        # if follow_symlinks is False
        if os.path.islink(name) and not follow_symlinks:
            pass
        else:
            diff_attrs = _cmp_attrs(name, attrs)
            if diff_attrs is not None:
                if diff_attrs[0] is not None or diff_attrs[1] is not None:
                    if __opts__['test'] is True:
                        ret['changes']['attrs'] = attrs
                    else:
                        if diff_attrs[0] is not None:
                            chattr(name, operator="add", attributes=diff_attrs[0])
                        if diff_attrs[1] is not None:
                            chattr(name, operator="remove", attributes=diff_attrs[1])
                        cmp_attrs = _cmp_attrs(name, attrs)
                        if cmp_attrs[0] is not None or cmp_attrs[1] is not None:
                            ret['result'] = False
                            ret['comment'].append(
                                'Failed to change attributes to {0}'.format(attrs)
                            )
                        else:
                            ret['changes']['attrs'] = attrs

    # Only combine the comment list into a string
    # after all comments are added above
    if isinstance(orig_comment, six.string_types):
        if orig_comment:
            ret['comment'].insert(0, orig_comment)
        ret['comment'] = '; '.join(ret['comment'])

    # Set result to None at the very end of the function,
    # after all changes have been recorded above
    if __opts__['test'] is True and ret['changes']:
        ret['result'] = None

    return ret, perms


def check_managed(
        name,
        source,
        source_hash,
        source_hash_name,
        user,
        group,
        mode,
        attrs,
        template,
        context,
        defaults,
        saltenv,
        contents=None,
        skip_verify=False,
        **kwargs):
    '''
    Check to see what changes need to be made for a file

    CLI Example:

    .. code-block:: bash

        salt '*' file.check_managed /etc/httpd/conf.d/httpd.conf salt://http/httpd.conf '{hash_type: 'md5', 'hsum': <md5sum>}' root, root, '755' jinja True None None base
    '''
    # If the source is a list then find which file exists
    source, source_hash = source_list(source,           # pylint: disable=W0633
                                      source_hash,
                                      saltenv)

    sfn = ''
    source_sum = None

    if contents is None:
        # Gather the source file from the server
        sfn, source_sum, comments = get_managed(
            name,
            template,
            source,
            source_hash,
            source_hash_name,
            user,
            group,
            mode,
            attrs,
            saltenv,
            context,
            defaults,
            skip_verify,
            **kwargs)
        if comments:
            __clean_tmp(sfn)
            return False, comments
    changes = check_file_meta(name, sfn, source, source_sum, user,
                              group, mode, attrs, saltenv, contents)
    # Ignore permission for files written temporary directories
    # Files in any path will still be set correctly using get_managed()
    if name.startswith(tempfile.gettempdir()):
        for key in ['user', 'group', 'mode']:
            changes.pop(key, None)
    __clean_tmp(sfn)
    if changes:
        log.info(changes)
        comments = ['The following values are set to be changed:\n']
        comments.extend('{0}: {1}\n'.format(key, val)
                        for key, val in six.iteritems(changes))
        return None, ''.join(comments)
    return True, 'The file {0} is in the correct state'.format(name)


def check_managed_changes(
        name,
        source,
        source_hash,
        source_hash_name,
        user,
        group,
        mode,
        attrs,
        template,
        context,
        defaults,
        saltenv,
        contents=None,
        skip_verify=False,
        keep_mode=False,
        **kwargs):
    '''
    Return a dictionary of what changes need to be made for a file

    CLI Example:

    .. code-block:: bash

        salt '*' file.check_managed_changes /etc/httpd/conf.d/httpd.conf salt://http/httpd.conf '{hash_type: 'md5', 'hsum': <md5sum>}' root, root, '755' jinja True None None base
    '''
    # If the source is a list then find which file exists
    source, source_hash = source_list(source,           # pylint: disable=W0633
                                      source_hash,
                                      saltenv)

    sfn = ''
    source_sum = None

    if contents is None:
        # Gather the source file from the server
        sfn, source_sum, comments = get_managed(
            name,
            template,
            source,
            source_hash,
            source_hash_name,
            user,
            group,
            mode,
            attrs,
            saltenv,
            context,
            defaults,
            skip_verify,
            **kwargs)

        # Ensure that user-provided hash string is lowercase
        if source_sum and ('hsum' in source_sum):
            source_sum['hsum'] = source_sum['hsum'].lower()

        if comments:
            __clean_tmp(sfn)
            return False, comments
        if sfn and source and keep_mode:
            if _urlparse(source).scheme in ('salt', 'file') \
                    or source.startswith('/'):
                try:
                    mode = __salt__['cp.stat_file'](source, saltenv=saltenv, octal=True)
                except Exception as exc:
                    log.warning('Unable to stat %s: %s', sfn, exc)
    changes = check_file_meta(name, sfn, source, source_sum, user,
                              group, mode, attrs, saltenv, contents)
    __clean_tmp(sfn)
    return changes


def check_file_meta(
        name,
        sfn,
        source,
        source_sum,
        user,
        group,
        mode,
        attrs,
        saltenv,
        contents=None):
    '''
    Check for the changes in the file metadata.

    CLI Example:

    .. code-block:: bash

        salt '*' file.check_file_meta /etc/httpd/conf.d/httpd.conf salt://http/httpd.conf '{hash_type: 'md5', 'hsum': <md5sum>}' root, root, '755' base

    .. note::

        Supported hash types include sha512, sha384, sha256, sha224, sha1, and
        md5.

    name
        Path to file destination

    sfn
        Template-processed source file contents

    source
        URL to file source

    source_sum
        File checksum information as a dictionary

        .. code-block:: yaml

            {hash_type: md5, hsum: <md5sum>}

    user
        Destination file user owner

    group
        Destination file group owner

    mode
        Destination file permissions mode

    attrs
        Destination file attributes

        .. versionadded:: 2018.3.0

    saltenv
        Salt environment used to resolve source files

    contents
        File contents
    '''
    changes = {}
    if not source_sum:
        source_sum = dict()

    try:
        lstats = stats(name, hash_type=source_sum.get('hash_type', None),
                       follow_symlinks=False)
    except CommandExecutionError:
        lstats = {}

    if not lstats:
        changes['newfile'] = name
        return changes

    if 'hsum' in source_sum:
        if source_sum['hsum'] != lstats['sum']:
            if not sfn and source:
                sfn = __salt__['cp.cache_file'](
                    source,
                    saltenv,
                    source_hash=source_sum['hsum'])
            if sfn:
                try:
                    changes['diff'] = get_diff(
                        name, sfn, template=True, show_filenames=False)
                except CommandExecutionError as exc:
                    changes['diff'] = exc.strerror
            else:
                changes['sum'] = 'Checksum differs'

    if contents is not None:
        # Write a tempfile with the static contents
        tmp = salt.utils.files.mkstemp(prefix=salt.utils.files.TEMPFILE_PREFIX,
                                       text=True)
        if salt.utils.platform.is_windows():
            contents = os.linesep.join(
                _splitlines_preserving_trailing_newline(contents))
        with salt.utils.files.fopen(tmp, 'w') as tmp_:
            tmp_.write(salt.utils.stringutils.to_str(contents))
        # Compare the static contents with the named file
        try:
            differences = get_diff(name, tmp, show_filenames=False)
        except CommandExecutionError as exc:
            log.error('Failed to diff files: {0}'.format(exc))
            differences = exc.strerror
        __clean_tmp(tmp)
        if differences:
            if __salt__['config.option']('obfuscate_templates'):
                changes['diff'] = '<Obfuscated Template>'
            else:
                changes['diff'] = differences

    if not salt.utils.platform.is_windows():
        # Check owner
        if (user is not None
                and user != lstats['user']
                and user != lstats['uid']):
            changes['user'] = user

        # Check group
        if (group is not None
                and group != lstats['group']
                and group != lstats['gid']):
            changes['group'] = group

        # Normalize the file mode
        smode = salt.utils.files.normalize_mode(lstats['mode'])
        mode = salt.utils.files.normalize_mode(mode)
        if mode is not None and mode != smode:
            changes['mode'] = mode

        if attrs:
            diff_attrs = _cmp_attrs(name, attrs)
            if diff_attrs is not None:
                if attrs is not None \
                        and (diff_attrs[0] is not None
                             or diff_attrs[1] is not None):
                    changes['attrs'] = attrs

    return changes


def get_diff(file1,
             file2,
             saltenv='base',
             show_filenames=True,
             show_changes=True,
             template=False,
             source_hash_file1=None,
             source_hash_file2=None):
    '''
    Return unified diff of two files

    file1
        The first file to feed into the diff utility

        .. versionchanged:: 2018.3.0
            Can now be either a local or remote file. In earlier releases,
            thuis had to be a file local to the minion.

    file2
        The second file to feed into the diff utility

        .. versionchanged:: 2018.3.0
            Can now be either a local or remote file. In earlier releases, this
            had to be a file on the salt fileserver (i.e.
            ``salt://somefile.txt``)

    show_filenames : True
        Set to ``False`` to hide the filenames in the top two lines of the
        diff.

    show_changes : True
        If set to ``False``, and there are differences, then instead of a diff
        a simple message stating that show_changes is set to ``False`` will be
        returned.

    template : False
        Set to ``True`` if two templates are being compared. This is not useful
        except for within states, with the ``obfuscate_templates`` option set
        to ``True``.

        .. versionadded:: 2018.3.0

    source_hash_file1
        If ``file1`` is an http(s)/ftp URL and the file exists in the minion's
        file cache, this option can be passed to keep the minion from
        re-downloading the archive if the cached copy matches the specified
        hash.

        .. versionadded:: 2018.3.0

    source_hash_file2
        If ``file2`` is an http(s)/ftp URL and the file exists in the minion's
        file cache, this option can be passed to keep the minion from
        re-downloading the archive if the cached copy matches the specified
        hash.

        .. versionadded:: 2018.3.0

    CLI Examples:

    .. code-block:: bash

        salt '*' file.get_diff /home/fred/.vimrc salt://users/fred/.vimrc
        salt '*' file.get_diff /tmp/foo.txt /tmp/bar.txt
    '''
    files = (file1, file2)
    source_hashes = (source_hash_file1, source_hash_file2)
    paths = []
    errors = []

    for filename, source_hash in zip(files, source_hashes):
        try:
            # Local file paths will just return the same path back when passed
            # to cp.cache_file.
            cached_path = __salt__['cp.cache_file'](filename,
                                                    saltenv,
                                                    source_hash=source_hash)
            if cached_path is False:
                errors.append(
                    'File {0} not found'.format(
                        salt.utils.stringutils.to_unicode(filename)
                    )
                )
                continue
            paths.append(cached_path)
        except MinionError as exc:
            errors.append(salt.utils.stringutils.to_unicode(exc.__str__()))
            continue

    if errors:
        raise CommandExecutionError(
            'Failed to cache one or more files',
            info=errors
        )

    args = []
    for filename in files:
        try:
            with salt.utils.files.fopen(filename, 'rb') as fp_:
                args.append(fp_.readlines())
        except (IOError, OSError) as exc:
            raise CommandExecutionError(
                'Failed to read {0}: {1}'.format(
                    salt.utils.stringutils.to_unicode(filename),
                    exc.strerror
                )
            )

    if args[0] != args[1]:
        if template and __salt__['config.option']('obfuscate_templates'):
            ret = '<Obfuscated Template>'
        elif not show_changes:
            ret = '<show_changes=False>'
        else:
            bdiff = _binary_replace(*files)
            if bdiff:
                ret = bdiff
            else:
                if show_filenames:
                    args.extend(files)
                ret = ''.join(
                    difflib.unified_diff(
                        *salt.utils.data.decode(args)
                    )
                )
        return ret
    return ''


def manage_file(name,
                sfn,
                ret,
                source,
                source_sum,
                user,
                group,
                mode,
                attrs,
                saltenv,
                backup,
                makedirs=False,
                template=None,   # pylint: disable=W0613
                show_changes=True,
                contents=None,
                dir_mode=None,
                follow_symlinks=True,
                skip_verify=False,
                keep_mode=False,
                encoding=None,
                encoding_errors='strict',
                **kwargs):
    '''
    Checks the destination against what was retrieved with get_managed and
    makes the appropriate modifications (if necessary).

    name
        location to place the file

    sfn
        location of cached file on the minion

        This is the path to the file stored on the minion. This file is placed
        on the minion using cp.cache_file.  If the hash sum of that file
        matches the source_sum, we do not transfer the file to the minion
        again.

        This file is then grabbed and if it has template set, it renders the
        file to be placed into the correct place on the system using
        salt.files.utils.copyfile()

    ret
        The initial state return data structure. Pass in ``None`` to use the
        default structure.

    source
        file reference on the master

    source_sum
        sum hash for source

    user
        user owner

    group
        group owner

    backup
        backup_mode

    attrs
        attributes to be set on file: '' means remove all of them

        .. versionadded:: 2018.3.0

    makedirs
        make directories if they do not exist

    template
        format of templating

    show_changes
        Include diff in state return

    contents:
        contents to be placed in the file

    dir_mode
        mode for directories created with makedirs

    skip_verify : False
        If ``True``, hash verification of remote file sources (``http://``,
        ``https://``, ``ftp://``) will be skipped, and the ``source_hash``
        argument will be ignored.

        .. versionadded:: 2016.3.0

    keep_mode : False
        If ``True``, and the ``source`` is a file from the Salt fileserver (or
        a local file on the minion), the mode of the destination file will be
        set to the mode of the source file.

        .. note:: keep_mode does not work with salt-ssh.

            As a consequence of how the files are transferred to the minion, and
            the inability to connect back to the master with salt-ssh, salt is
            unable to stat the file as it exists on the fileserver and thus
            cannot mirror the mode on the salt-ssh minion

    encoding
        If specified, then the specified encoding will be used. Otherwise, the
        file will be encoded using the system locale (usually UTF-8). See
        https://docs.python.org/3/library/codecs.html#standard-encodings for
        the list of available encodings.

        .. versionadded:: 2017.7.0

    encoding_errors : 'strict'
        Default is ```'strict'```.
        See https://docs.python.org/2/library/codecs.html#codec-base-classes
        for the error handling schemes.

        .. versionadded:: 2017.7.0

    CLI Example:

    .. code-block:: bash

        salt '*' file.manage_file /etc/httpd/conf.d/httpd.conf '' '{}' salt://http/httpd.conf '{hash_type: 'md5', 'hsum': <md5sum>}' root root '755' base ''

    .. versionchanged:: 2014.7.0
        ``follow_symlinks`` option added

    '''
    name = os.path.expanduser(name)

    if not ret:
        ret = {'name': name,
               'changes': {},
               'comment': '',
               'result': True}
    # Ensure that user-provided hash string is lowercase
    if source_sum and ('hsum' in source_sum):
        source_sum['hsum'] = source_sum['hsum'].lower()

    if source:
        if not sfn:
            # File is not present, cache it
            sfn = __salt__['cp.cache_file'](source, saltenv)
            if not sfn:
                return _error(
                    ret, 'Source file \'{0}\' not found'.format(source))
            htype = source_sum.get('hash_type', __opts__['hash_type'])
            # Recalculate source sum now that file has been cached
            source_sum = {
                'hash_type': htype,
                'hsum': get_hash(sfn, form=htype)
            }

        if keep_mode:
            if _urlparse(source).scheme in ('salt', 'file', ''):
                try:
                    mode = __salt__['cp.stat_file'](source, saltenv=saltenv, octal=True)
                except Exception as exc:
                    log.warning('Unable to stat %s: %s', sfn, exc)

    # Check changes if the target file exists
    if os.path.isfile(name) or os.path.islink(name):
        if os.path.islink(name) and follow_symlinks:
            real_name = os.path.realpath(name)
        else:
            real_name = name

        # Only test the checksums on files with managed contents
        if source and not (not follow_symlinks and os.path.islink(real_name)):
            name_sum = get_hash(real_name, source_sum.get('hash_type', __opts__['hash_type']))
        else:
            name_sum = None

        # Check if file needs to be replaced
        if source and (name_sum is None or source_sum.get('hsum', __opts__['hash_type']) != name_sum):
            if not sfn:
                sfn = __salt__['cp.cache_file'](source, saltenv)
            if not sfn:
                return _error(
                    ret, 'Source file \'{0}\' not found'.format(source))
            # If the downloaded file came from a non salt server or local
            # source, and we are not skipping checksum verification, then
            # verify that it matches the specified checksum.
            if not skip_verify \
                    and _urlparse(source).scheme != 'salt':
                dl_sum = get_hash(sfn, source_sum['hash_type'])
                if dl_sum != source_sum['hsum']:
                    ret['comment'] = (
                        'Specified {0} checksum for {1} ({2}) does not match '
                        'actual checksum ({3}). If the \'source_hash\' value '
                        'refers to a remote file with multiple possible '
                        'matches, then it may be necessary to set '
                        '\'source_hash_name\'.'.format(
                            source_sum['hash_type'],
                            source,
                            source_sum['hsum'],
                            dl_sum
                        )
                    )
                    ret['result'] = False
                    return ret

            # Print a diff equivalent to diff -u old new
            if __salt__['config.option']('obfuscate_templates'):
                ret['changes']['diff'] = '<Obfuscated Template>'
            elif not show_changes:
                ret['changes']['diff'] = '<show_changes=False>'
            else:
                try:
                    ret['changes']['diff'] = get_diff(
                        real_name, sfn, show_filenames=False)
                except CommandExecutionError as exc:
                    ret['changes']['diff'] = exc.strerror

            # Pre requisites are met, and the file needs to be replaced, do it
            try:
                salt.utils.files.copyfile(sfn,
                                    real_name,
                                    __salt__['config.backup_mode'](backup),
                                    __opts__['cachedir'])
            except IOError as io_error:
                __clean_tmp(sfn)
                return _error(
                    ret, 'Failed to commit change: {0}'.format(io_error))

        if contents is not None:
            # Write the static contents to a temporary file
            tmp = salt.utils.files.mkstemp(prefix=salt.utils.files.TEMPFILE_PREFIX,
                                           text=True)
            if salt.utils.platform.is_windows():
                contents = os.linesep.join(
                    _splitlines_preserving_trailing_newline(contents))
            with salt.utils.files.fopen(tmp, 'wb') as tmp_:
                if encoding:
                    log.debug('File will be encoded with %s', encoding)
                    tmp_.write(contents.encode(encoding=encoding, errors=encoding_errors))
                else:
                    tmp_.write(salt.utils.stringutils.to_bytes(contents))

            try:
                differences = get_diff(
                    real_name, tmp, show_filenames=False,
                    show_changes=show_changes, template=True)

            except CommandExecutionError as exc:
                ret.setdefault('warnings', []).append(
                    'Failed to detect changes to file: {0}'.format(exc.strerror)
                )
                differences = ''

            if differences:
                ret['changes']['diff'] = differences

                # Pre requisites are met, the file needs to be replaced, do it
                try:
                    salt.utils.files.copyfile(tmp,
                                        real_name,
                                        __salt__['config.backup_mode'](backup),
                                        __opts__['cachedir'])
                except IOError as io_error:
                    __clean_tmp(tmp)
                    return _error(
                        ret, 'Failed to commit change: {0}'.format(io_error))
            __clean_tmp(tmp)

        # Check for changing symlink to regular file here
        if os.path.islink(name) and not follow_symlinks:
            if not sfn:
                sfn = __salt__['cp.cache_file'](source, saltenv)
            if not sfn:
                return _error(
                    ret, 'Source file \'{0}\' not found'.format(source))
            # If the downloaded file came from a non salt server source verify
            # that it matches the intended sum value
            if not skip_verify and _urlparse(source).scheme != 'salt':
                dl_sum = get_hash(sfn, source_sum['hash_type'])
                if dl_sum != source_sum['hsum']:
                    ret['comment'] = (
                        'Specified {0} checksum for {1} ({2}) does not match '
                        'actual checksum ({3})'.format(
                            source_sum['hash_type'],
                            name,
                            source_sum['hsum'],
                            dl_sum
                        )
                    )
                    ret['result'] = False
                    return ret

            try:
                salt.utils.files.copyfile(sfn,
                                    name,
                                    __salt__['config.backup_mode'](backup),
                                    __opts__['cachedir'])
            except IOError as io_error:
                __clean_tmp(sfn)
                return _error(
                    ret, 'Failed to commit change: {0}'.format(io_error))

            ret['changes']['diff'] = \
                'Replace symbolic link with regular file'

        if salt.utils.platform.is_windows():
            # This function resides in win_file.py and will be available
            # on Windows. The local function will be overridden
            # pylint: disable=E1120,E1121,E1123
            ret = check_perms(
               path=name,
                ret=ret,
                owner=kwargs.get('win_owner'),
                grant_perms=kwargs.get('win_perms'),
                deny_perms=kwargs.get('win_deny_perms'),
                inheritance=kwargs.get('win_inheritance', True),
                reset=kwargs.get('win_perms_reset', False))
            # pylint: enable=E1120,E1121,E1123
        else:
            ret, _ = check_perms(name, ret, user, group, mode, attrs, follow_symlinks)

        if ret['changes']:
            ret['comment'] = 'File {0} updated'.format(
                salt.utils.locales.sdecode(name)
            )

        elif not ret['changes'] and ret['result']:
            ret['comment'] = 'File {0} is in the correct state'.format(
                salt.utils.locales.sdecode(name)
            )
        if sfn:
            __clean_tmp(sfn)
        return ret
    else:  # target file does not exist
        contain_dir = os.path.dirname(name)

        def _set_mode_and_make_dirs(name, dir_mode, mode, user, group):
            # check for existence of windows drive letter
            if salt.utils.platform.is_windows():
                drive, _ = os.path.splitdrive(name)
                if drive and not os.path.exists(drive):
                    __clean_tmp(sfn)
                    return _error(ret,
                                  '{0} drive not present'.format(drive))
            if dir_mode is None and mode is not None:
                # Add execute bit to each nonzero digit in the mode, if
                # dir_mode was not specified. Otherwise, any
                # directories created with makedirs_() below can't be
                # listed via a shell.
                mode_list = [x for x in six.text_type(mode)][-3:]
                for idx in range(len(mode_list)):
                    if mode_list[idx] != '0':
                        mode_list[idx] = six.text_type(int(mode_list[idx]) | 1)
                dir_mode = ''.join(mode_list)

            if salt.utils.platform.is_windows():
                # This function resides in win_file.py and will be available
                # on Windows. The local function will be overridden
                # pylint: disable=E1120,E1121,E1123
                makedirs_(
                    path=name,
                    owner=kwargs.get('win_owner'),
                    grant_perms=kwargs.get('win_perms'),
                    deny_perms=kwargs.get('win_deny_perms'),
                    inheritance=kwargs.get('win_inheritance', True),
                    reset=kwargs.get('win_perms_reset', False))
                # pylint: enable=E1120,E1121,E1123
            else:
                makedirs_(name, user=user, group=group, mode=dir_mode)

        if source:
            # Apply the new file
            if not sfn:
                sfn = __salt__['cp.cache_file'](source, saltenv)
            if not sfn:
                return _error(
                    ret, 'Source file \'{0}\' not found'.format(source))
            # If the downloaded file came from a non salt server source verify
            # that it matches the intended sum value
            if not skip_verify \
                    and _urlparse(source).scheme != 'salt':
                dl_sum = get_hash(sfn, source_sum['hash_type'])
                if dl_sum != source_sum['hsum']:
                    ret['comment'] = (
                        'Specified {0} checksum for {1} ({2}) does not match '
                        'actual checksum ({3})'.format(
                            source_sum['hash_type'],
                            name,
                            source_sum['hsum'],
                            dl_sum
                        )
                    )
                    ret['result'] = False
                    return ret
            # It is a new file, set the diff accordingly
            ret['changes']['diff'] = 'New file'
            if not os.path.isdir(contain_dir):
                if makedirs:
                    _set_mode_and_make_dirs(name, dir_mode, mode, user, group)
                else:
                    __clean_tmp(sfn)
                    # No changes actually made
                    ret['changes'].pop('diff', None)
                    return _error(ret, 'Parent directory not present')
        else:  # source != True
            if not os.path.isdir(contain_dir):
                if makedirs:
                    _set_mode_and_make_dirs(name, dir_mode, mode, user, group)
                else:
                    __clean_tmp(sfn)
                    # No changes actually made
                    ret['changes'].pop('diff', None)
                    return _error(ret, 'Parent directory not present')

            # Create the file, user rw-only if mode will be set to prevent
            # a small security race problem before the permissions are set
            with salt.utils.files.set_umask(0o077 if mode else None):
                # Create a new file when test is False and source is None
                if contents is None:
                    if not __opts__['test']:
                        if touch(name):
                            ret['changes']['new'] = 'file {0} created'.format(name)
                            ret['comment'] = 'Empty file'
                        else:
                            return _error(
                                ret, 'Empty file {0} not created'.format(name)
                            )
                else:
                    if not __opts__['test']:
                        if touch(name):
                            ret['changes']['diff'] = 'New file'
                        else:
                            return _error(
                                ret, 'File {0} not created'.format(name)
                            )

        if contents is not None:
            # Write the static contents to a temporary file
            tmp = salt.utils.files.mkstemp(prefix=salt.utils.files.TEMPFILE_PREFIX,
                                           text=True)
            if salt.utils.platform.is_windows():
                contents = os.linesep.join(
                    _splitlines_preserving_trailing_newline(contents))
            with salt.utils.files.fopen(tmp, 'wb') as tmp_:
                if encoding:
                    log.debug('File will be encoded with %s', encoding)
                    tmp_.write(contents.encode(encoding=encoding, errors=encoding_errors))
                else:
                    tmp_.write(salt.utils.stringutils.to_bytes(contents))

            # Copy into place
            salt.utils.files.copyfile(tmp,
                                name,
                                __salt__['config.backup_mode'](backup),
                                __opts__['cachedir'])
            __clean_tmp(tmp)
        # Now copy the file contents if there is a source file
        elif sfn:
            salt.utils.files.copyfile(sfn,
                                name,
                                __salt__['config.backup_mode'](backup),
                                __opts__['cachedir'])
            __clean_tmp(sfn)

        # This is a new file, if no mode specified, use the umask to figure
        # out what mode to use for the new file.
        if mode is None and not salt.utils.platform.is_windows():
            # Get current umask
            mask = salt.utils.files.get_umask()
            # Calculate the mode value that results from the umask
            mode = oct((0o777 ^ mask) & 0o666)

        if salt.utils.platform.is_windows():
            # This function resides in win_file.py and will be available
            # on Windows. The local function will be overridden
            # pylint: disable=E1120,E1121,E1123
            ret = check_perms(
               path=name,
                ret=ret,
                owner=kwargs.get('win_owner'),
                grant_perms=kwargs.get('win_perms'),
                deny_perms=kwargs.get('win_deny_perms'),
                inheritance=kwargs.get('win_inheritance', True),
                reset=kwargs.get('win_perms_reset', False))
            # pylint: enable=E1120,E1121,E1123
        else:
            ret, _ = check_perms(name, ret, user, group, mode, attrs)

        if not ret['comment']:
            ret['comment'] = 'File ' + name + ' updated'

        if __opts__['test']:
            ret['comment'] = 'File ' + name + ' not updated'
        elif not ret['changes'] and ret['result']:
            ret['comment'] = 'File ' + name + ' is in the correct state'
        if sfn:
            __clean_tmp(sfn)

        return ret


def mkdir(dir_path,
          user=None,
          group=None,
          mode=None):
    '''
    Ensure that a directory is available.

    CLI Example:

    .. code-block:: bash

        salt '*' file.mkdir /opt/jetty/context
    '''
    dir_path = os.path.expanduser(dir_path)

    directory = os.path.normpath(dir_path)

    if not os.path.isdir(directory):
        # If a caller such as managed() is invoked  with makedirs=True, make
        # sure that any created dirs are created with the same user and group
        # to follow the principal of least surprise method.
        makedirs_perms(directory, user, group, mode)

    return True


def makedirs_(path,
              user=None,
              group=None,
              mode=None):
    '''
    Ensure that the directory containing this path is available.

    .. note::

        The path must end with a trailing slash otherwise the directory/directories
        will be created up to the parent directory. For example if path is
        ``/opt/code``, then it would be treated as ``/opt/`` but if the path
        ends with a trailing slash like ``/opt/code/``, then it would be
        treated as ``/opt/code/``.

    CLI Example:

    .. code-block:: bash

        salt '*' file.makedirs /opt/code/
    '''
    path = os.path.expanduser(path)

    if mode:
        mode = salt.utils.files.normalize_mode(mode)

    # walk up the directory structure until we find the first existing
    # directory
    dirname = os.path.normpath(os.path.dirname(path))

    if os.path.isdir(dirname):
        # There's nothing for us to do
        msg = 'Directory \'{0}\' already exists'.format(dirname)
        log.debug(msg)
        return msg

    if os.path.exists(dirname):
        msg = 'The path \'{0}\' already exists and is not a directory'.format(
            dirname
        )
        log.debug(msg)
        return msg

    directories_to_create = []
    while True:
        if os.path.isdir(dirname):
            break

        directories_to_create.append(dirname)
        current_dirname = dirname
        dirname = os.path.dirname(dirname)

        if current_dirname == dirname:
            raise SaltInvocationError(
                'Recursive creation for path \'{0}\' would result in an '
                'infinite loop. Please use an absolute path.'.format(dirname)
            )

    # create parent directories from the topmost to the most deeply nested one
    directories_to_create.reverse()
    for directory_to_create in directories_to_create:
        # all directories have the user, group and mode set!!
        log.debug('Creating directory: %s', directory_to_create)
        mkdir(directory_to_create, user=user, group=group, mode=mode)


def makedirs_perms(name,
                   user=None,
                   group=None,
                   mode='0755'):
    '''
    Taken and modified from os.makedirs to set user, group and mode for each
    directory created.

    CLI Example:

    .. code-block:: bash

        salt '*' file.makedirs_perms /opt/code
    '''
    name = os.path.expanduser(name)

    path = os.path
    head, tail = path.split(name)
    if not tail:
        head, tail = path.split(head)
    if head and tail and not path.exists(head):
        try:
            makedirs_perms(head, user, group, mode)
        except OSError as exc:
            # be happy if someone already created the path
            if exc.errno != errno.EEXIST:
                raise
        if tail == os.curdir:  # xxx/newdir/. exists if xxx/newdir exists
            return
    os.mkdir(name)
    check_perms(name,
                None,
                user,
                group,
                int('{0}'.format(mode)) if mode else None)


def get_devmm(name):
    '''
    Get major/minor info from a device

    CLI Example:

    .. code-block:: bash

       salt '*' file.get_devmm /dev/chr
    '''
    name = os.path.expanduser(name)

    if is_chrdev(name) or is_blkdev(name):
        stat_structure = os.stat(name)
        return (
                os.major(stat_structure.st_rdev),
                os.minor(stat_structure.st_rdev))
    else:
        return (0, 0)


def is_chrdev(name):
    '''
    Check if a file exists and is a character device.

    CLI Example:

    .. code-block:: bash

       salt '*' file.is_chrdev /dev/chr
    '''
    name = os.path.expanduser(name)

    stat_structure = None
    try:
        stat_structure = os.stat(name)
    except OSError as exc:
        if exc.errno == errno.ENOENT:
            # If the character device does not exist in the first place
            return False
        else:
            raise
    return stat.S_ISCHR(stat_structure.st_mode)


def mknod_chrdev(name,
                 major,
                 minor,
                 user=None,
                 group=None,
                 mode='0660'):
    '''
    .. versionadded:: 0.17.0

    Create a character device.

    CLI Example:

    .. code-block:: bash

       salt '*' file.mknod_chrdev /dev/chr 180 31
    '''
    name = os.path.expanduser(name)

    ret = {'name': name,
           'changes': {},
           'comment': '',
           'result': False}
    log.debug('Creating character device name:{0} major:{1} minor:{2} mode:{3}'
              .format(name, major, minor, mode))
    try:
        if __opts__['test']:
            ret['changes'] = {'new': 'Character device {0} created.'.format(name)}
            ret['result'] = None
        else:
            if os.mknod(name,
                        int(six.text_type(mode).lstrip('0Oo'), 8) | stat.S_IFCHR,
                        os.makedev(major, minor)) is None:
                ret['changes'] = {'new': 'Character device {0} created.'.format(name)}
                ret['result'] = True
    except OSError as exc:
        # be happy it is already there....however, if you are trying to change the
        # major/minor, you will need to unlink it first as os.mknod will not overwrite
        if exc.errno != errno.EEXIST:
            raise
        else:
            ret['comment'] = 'File {0} exists and cannot be overwritten'.format(name)
    # quick pass at verifying the permissions of the newly created character device
    check_perms(name,
                None,
                user,
                group,
                int('{0}'.format(mode)) if mode else None)
    return ret


def is_blkdev(name):
    '''
    Check if a file exists and is a block device.

    CLI Example:

    .. code-block:: bash

       salt '*' file.is_blkdev /dev/blk
    '''
    name = os.path.expanduser(name)

    stat_structure = None
    try:
        stat_structure = os.stat(name)
    except OSError as exc:
        if exc.errno == errno.ENOENT:
            # If the block device does not exist in the first place
            return False
        else:
            raise
    return stat.S_ISBLK(stat_structure.st_mode)


def mknod_blkdev(name,
                 major,
                 minor,
                 user=None,
                 group=None,
                 mode='0660'):
    '''
    .. versionadded:: 0.17.0

    Create a block device.

    CLI Example:

    .. code-block:: bash

       salt '*' file.mknod_blkdev /dev/blk 8 999
    '''
    name = os.path.expanduser(name)

    ret = {'name': name,
           'changes': {},
           'comment': '',
           'result': False}
    log.debug('Creating block device name:{0} major:{1} minor:{2} mode:{3}'
              .format(name, major, minor, mode))
    try:
        if __opts__['test']:
            ret['changes'] = {'new': 'Block device {0} created.'.format(name)}
            ret['result'] = None
        else:
            if os.mknod(name,
                        int(six.text_type(mode).lstrip('0Oo'), 8) | stat.S_IFBLK,
                        os.makedev(major, minor)) is None:
                ret['changes'] = {'new': 'Block device {0} created.'.format(name)}
                ret['result'] = True
    except OSError as exc:
        # be happy it is already there....however, if you are trying to change the
        # major/minor, you will need to unlink it first as os.mknod will not overwrite
        if exc.errno != errno.EEXIST:
            raise
        else:
            ret['comment'] = 'File {0} exists and cannot be overwritten'.format(name)
    # quick pass at verifying the permissions of the newly created block device
    check_perms(name,
                None,
                user,
                group,
                int('{0}'.format(mode)) if mode else None)
    return ret


def is_fifo(name):
    '''
    Check if a file exists and is a FIFO.

    CLI Example:

    .. code-block:: bash

       salt '*' file.is_fifo /dev/fifo
    '''
    name = os.path.expanduser(name)

    stat_structure = None
    try:
        stat_structure = os.stat(name)
    except OSError as exc:
        if exc.errno == errno.ENOENT:
            # If the fifo does not exist in the first place
            return False
        else:
            raise
    return stat.S_ISFIFO(stat_structure.st_mode)


def mknod_fifo(name,
               user=None,
               group=None,
               mode='0660'):
    '''
    .. versionadded:: 0.17.0

    Create a FIFO pipe.

    CLI Example:

    .. code-block:: bash

       salt '*' file.mknod_fifo /dev/fifo
    '''
    name = os.path.expanduser(name)

    ret = {'name': name,
           'changes': {},
           'comment': '',
           'result': False}
    log.debug('Creating FIFO name: {0}'.format(name))
    try:
        if __opts__['test']:
            ret['changes'] = {'new': 'Fifo pipe {0} created.'.format(name)}
            ret['result'] = None
        else:
            if os.mkfifo(name, int(six.text_type(mode).lstrip('0Oo'), 8)) is None:
                ret['changes'] = {'new': 'Fifo pipe {0} created.'.format(name)}
                ret['result'] = True
    except OSError as exc:
        # be happy it is already there
        if exc.errno != errno.EEXIST:
            raise
        else:
            ret['comment'] = 'File {0} exists and cannot be overwritten'.format(name)
    # quick pass at verifying the permissions of the newly created fifo
    check_perms(name,
                None,
                user,
                group,
                int('{0}'.format(mode)) if mode else None)
    return ret


def mknod(name,
          ntype,
          major=0,
          minor=0,
          user=None,
          group=None,
          mode='0600'):
    '''
    .. versionadded:: 0.17.0

    Create a block device, character device, or fifo pipe.
    Identical to the gnu mknod.

    CLI Examples:

    .. code-block:: bash

        salt '*' file.mknod /dev/chr c 180 31
        salt '*' file.mknod /dev/blk b 8 999
        salt '*' file.nknod /dev/fifo p
    '''
    ret = False
    makedirs_(name, user, group)
    if ntype == 'c':
        ret = mknod_chrdev(name, major, minor, user, group, mode)
    elif ntype == 'b':
        ret = mknod_blkdev(name, major, minor, user, group, mode)
    elif ntype == 'p':
        ret = mknod_fifo(name, user, group, mode)
    else:
        raise SaltInvocationError(
            'Node type unavailable: \'{0}\'. Available node types are '
            'character (\'c\'), block (\'b\'), and pipe (\'p\').'.format(ntype)
        )
    return ret


def list_backups(path, limit=None):
    '''
    .. versionadded:: 0.17.0

    Lists the previous versions of a file backed up using Salt's :ref:`file
    state backup <file-state-backups>` system.

    path
        The path on the minion to check for backups
    limit
        Limit the number of results to the most recent N backups

    CLI Example:

    .. code-block:: bash

        salt '*' file.list_backups /foo/bar/baz.txt
    '''
    path = os.path.expanduser(path)

    try:
        limit = int(limit)
    except TypeError:
        pass
    except ValueError:
        log.error('file.list_backups: \'limit\' value must be numeric')
        limit = None

    bkroot = _get_bkroot()
    parent_dir, basename = os.path.split(path)
    if salt.utils.platform.is_windows():
        # ':' is an illegal filesystem path character on Windows
        src_dir = parent_dir.replace(':', '_')
    else:
        src_dir = parent_dir[1:]
    # Figure out full path of location of backup file in minion cache
    bkdir = os.path.join(bkroot, src_dir)

    if not os.path.isdir(bkdir):
        return {}

    files = {}
    for fname in [x for x in os.listdir(bkdir)
                  if os.path.isfile(os.path.join(bkdir, x))]:
        if salt.utils.platform.is_windows():
            # ':' is an illegal filesystem path character on Windows
            strpfmt = '{0}_%a_%b_%d_%H-%M-%S_%f_%Y'.format(basename)
        else:
            strpfmt = '{0}_%a_%b_%d_%H:%M:%S_%f_%Y'.format(basename)
        try:
            timestamp = datetime.datetime.strptime(fname, strpfmt)
        except ValueError:
            # File didn't match the strp format string, so it's not a backup
            # for this file. Move on to the next one.
            continue
        if salt.utils.platform.is_windows():
            str_format = '%a %b %d %Y %H-%M-%S.%f'
        else:
            str_format = '%a %b %d %Y %H:%M:%S.%f'
        files.setdefault(timestamp, {})['Backup Time'] = \
            timestamp.strftime(str_format)
        location = os.path.join(bkdir, fname)
        files[timestamp]['Size'] = os.stat(location).st_size
        files[timestamp]['Location'] = location

    return dict(list(zip(
        list(range(len(files))),
        [files[x] for x in sorted(files, reverse=True)[:limit]]
    )))

list_backup = salt.utils.functools.alias_function(list_backups, 'list_backup')


def list_backups_dir(path, limit=None):
    '''
    Lists the previous versions of a directory backed up using Salt's :ref:`file
    state backup <file-state-backups>` system.

    path
        The directory on the minion to check for backups
    limit
        Limit the number of results to the most recent N backups

    CLI Example:

    .. code-block:: bash

        salt '*' file.list_backups_dir /foo/bar/baz/
    '''
    path = os.path.expanduser(path)

    try:
        limit = int(limit)
    except TypeError:
        pass
    except ValueError:
        log.error('file.list_backups_dir: \'limit\' value must be numeric')
        limit = None

    bkroot = _get_bkroot()
    parent_dir, basename = os.path.split(path)
    # Figure out full path of location of backup folder in minion cache
    bkdir = os.path.join(bkroot, parent_dir[1:])

    if not os.path.isdir(bkdir):
        return {}

    files = {}
    f = dict([(i, len(list(n))) for i, n in itertools.groupby([x.split("_")[0] for x in sorted(os.listdir(bkdir))])])
    ff = os.listdir(bkdir)
    for i, n in six.iteritems(f):
        ssfile = {}
        for x in sorted(ff):
            basename = x.split('_')[0]
            if i == basename:
                strpfmt = '{0}_%a_%b_%d_%H:%M:%S_%f_%Y'.format(basename)
                try:
                    timestamp = datetime.datetime.strptime(x, strpfmt)
                except ValueError:
                    # Folder didn't match the strp format string, so it's not a backup
                    # for this folder. Move on to the next one.
                    continue
                ssfile.setdefault(timestamp, {})['Backup Time'] = \
                    timestamp.strftime('%a %b %d %Y %H:%M:%S.%f')
                location = os.path.join(bkdir, x)
                ssfile[timestamp]['Size'] = os.stat(location).st_size
                ssfile[timestamp]['Location'] = location

        sfiles = dict(list(zip(list(range(n)), [ssfile[x] for x in sorted(ssfile, reverse=True)[:limit]])))
        sefiles = {i: sfiles}
        files.update(sefiles)
    return files


def restore_backup(path, backup_id):
    '''
    .. versionadded:: 0.17.0

    Restore a previous version of a file that was backed up using Salt's
    :ref:`file state backup <file-state-backups>` system.

    path
        The path on the minion to check for backups
    backup_id
        The numeric id for the backup you wish to restore, as found using
        :mod:`file.list_backups <salt.modules.file.list_backups>`

    CLI Example:

    .. code-block:: bash

        salt '*' file.restore_backup /foo/bar/baz.txt 0
    '''
    path = os.path.expanduser(path)

    # Note: This only supports minion backups, so this function will need to be
    # modified if/when master backups are implemented.
    ret = {'result': False,
           'comment': 'Invalid backup_id \'{0}\''.format(backup_id)}
    try:
        if len(six.text_type(backup_id)) == len(six.text_type(int(backup_id))):
            backup = list_backups(path)[int(backup_id)]
        else:
            return ret
    except ValueError:
        return ret
    except KeyError:
        ret['comment'] = 'backup_id \'{0}\' does not exist for ' \
                         '{1}'.format(backup_id, path)
        return ret

    salt.utils.files.backup_minion(path, _get_bkroot())
    try:
        shutil.copyfile(backup['Location'], path)
    except IOError as exc:
        ret['comment'] = \
            'Unable to restore {0} to {1}: ' \
            '{2}'.format(backup['Location'], path, exc)
        return ret
    else:
        ret['result'] = True
        ret['comment'] = 'Successfully restored {0} to ' \
                         '{1}'.format(backup['Location'], path)

    # Try to set proper ownership
    if not salt.utils.platform.is_windows():
        try:
            fstat = os.stat(path)
        except (OSError, IOError):
            ret['comment'] += ', but was unable to set ownership'
        else:
            os.chown(path, fstat.st_uid, fstat.st_gid)

    return ret


def delete_backup(path, backup_id):
    '''
    .. versionadded:: 0.17.0

    Delete a previous version of a file that was backed up using Salt's
    :ref:`file state backup <file-state-backups>` system.

    path
        The path on the minion to check for backups
    backup_id
        The numeric id for the backup you wish to delete, as found using
        :mod:`file.list_backups <salt.modules.file.list_backups>`

    CLI Example:

    .. code-block:: bash

        salt '*' file.delete_backup /var/cache/salt/minion/file_backup/home/foo/bar/baz.txt 0
    '''
    path = os.path.expanduser(path)

    ret = {'result': False,
           'comment': 'Invalid backup_id \'{0}\''.format(backup_id)}
    try:
        if len(six.text_type(backup_id)) == len(six.text_type(int(backup_id))):
            backup = list_backups(path)[int(backup_id)]
        else:
            return ret
    except ValueError:
        return ret
    except KeyError:
        ret['comment'] = 'backup_id \'{0}\' does not exist for ' \
                         '{1}'.format(backup_id, path)
        return ret

    try:
        os.remove(backup['Location'])
    except IOError as exc:
        ret['comment'] = 'Unable to remove {0}: {1}'.format(backup['Location'],
                                                            exc)
    else:
        ret['result'] = True
        ret['comment'] = 'Successfully removed {0}'.format(backup['Location'])

    return ret

remove_backup = salt.utils.functools.alias_function(delete_backup, 'remove_backup')


def grep(path,
         pattern,
         *opts):
    '''
    Grep for a string in the specified file

    .. note::
        This function's return value is slated for refinement in future
        versions of Salt

    path
        Path to the file to be searched

        .. note::
            Globbing is supported (i.e. ``/var/log/foo/*.log``, but if globbing
            is being used then the path should be quoted to keep the shell from
            attempting to expand the glob expression.

    pattern
        Pattern to match. For example: ``test``, or ``a[0-5]``

    opts
        Additional command-line flags to pass to the grep command. For example:
        ``-v``, or ``-i -B2``

        .. note::
            The options should come after a double-dash (as shown in the
            examples below) to keep Salt's own argument parser from
            interpreting them.

    CLI Example:

    .. code-block:: bash

        salt '*' file.grep /etc/passwd nobody
        salt '*' file.grep /etc/sysconfig/network-scripts/ifcfg-eth0 ipaddr -- -i
        salt '*' file.grep /etc/sysconfig/network-scripts/ifcfg-eth0 ipaddr -- -i -B2
        salt '*' file.grep "/etc/sysconfig/network-scripts/*" ipaddr -- -i -l
    '''
    path = os.path.expanduser(path)

    split_opts = []
    for opt in opts:
        try:
            split = salt.utils.args.shlex_split(opt)
        except AttributeError:
            split = salt.utils.args.shlex_split(six.text_type(opt))
        if len(split) > 1:
            raise SaltInvocationError(
                'Passing multiple command line arguments in a single string '
                'is not supported, please pass the following arguments '
                'separately: {0}'.format(opt)
            )
        split_opts.extend(split)

    cmd = ['grep'] + split_opts + [pattern, path]
    try:
        ret = __salt__['cmd.run_all'](cmd, python_shell=False)
    except (IOError, OSError) as exc:
        raise CommandExecutionError(exc.strerror)

    return ret


def open_files(by_pid=False):
    '''
    Return a list of all physical open files on the system.

    CLI Examples:

    .. code-block:: bash

        salt '*' file.open_files
        salt '*' file.open_files by_pid=True
    '''
    # First we collect valid PIDs
    pids = {}
    procfs = os.listdir('/proc/')
    for pfile in procfs:
        try:
            pids[int(pfile)] = []
        except ValueError:
            # Not a valid PID, move on
            pass

    # Then we look at the open files for each PID
    files = {}
    for pid in pids:
        ppath = '/proc/{0}'.format(pid)
        try:
            tids = os.listdir('{0}/task'.format(ppath))
        except OSError:
            continue

        # Collect the names of all of the file descriptors
        fd_ = []

        #try:
        #    fd_.append(os.path.realpath('{0}/task/{1}exe'.format(ppath, tid)))
        #except:
        #    pass

        for fpath in os.listdir('{0}/fd'.format(ppath)):
            fd_.append('{0}/fd/{1}'.format(ppath, fpath))

        for tid in tids:
            try:
                fd_.append(
                    os.path.realpath('{0}/task/{1}/exe'.format(ppath, tid))
                )
            except OSError:
                continue

            for tpath in os.listdir('{0}/task/{1}/fd'.format(ppath, tid)):
                fd_.append('{0}/task/{1}/fd/{2}'.format(ppath, tid, tpath))

        fd_ = sorted(set(fd_))

        # Loop through file descriptors and return useful data for each file
        for fdpath in fd_:
            # Sometimes PIDs and TIDs disappear before we can query them
            try:
                name = os.path.realpath(fdpath)
                # Running stat on the file cuts out all of the sockets and
                # deleted files from the list
                os.stat(name)
            except OSError:
                continue

            if name not in files:
                files[name] = [pid]
            else:
                # We still want to know which PIDs are using each file
                files[name].append(pid)
                files[name] = sorted(set(files[name]))

            pids[pid].append(name)
            pids[pid] = sorted(set(pids[pid]))

    if by_pid:
        return pids
    return files


def pardir():
    '''
    Return the relative parent directory path symbol for underlying OS

    .. versionadded:: 2014.7.0

    This can be useful when constructing Salt Formulas.

    .. code-block:: jinja

        {% set pardir = salt['file.pardir']() %}
        {% set final_path = salt['file.join']('subdir', pardir, 'confdir') %}

    CLI Example:

    .. code-block:: bash

        salt '*' file.pardir
    '''
    return os.path.pardir


def normpath(path):
    '''
    Returns Normalize path, eliminating double slashes, etc.

    .. versionadded:: 2015.5.0

    This can be useful at the CLI but is frequently useful when scripting.

    .. code-block:: jinja

        {%- from salt['file.normpath'](tpldir + '/../vars.jinja') import parent_vars %}

    CLI Example:

    .. code-block:: bash

        salt '*' file.normpath 'a/b/c/..'
    '''
    return os.path.normpath(path)


def basename(path):
    '''
    Returns the final component of a pathname

    .. versionadded:: 2015.5.0

    This can be useful at the CLI but is frequently useful when scripting.

    .. code-block:: jinja

        {%- set filename = salt['file.basename'](source_file) %}

    CLI Example:

    .. code-block:: bash

        salt '*' file.basename 'test/test.config'
    '''
    return os.path.basename(path)


def dirname(path):
    '''
    Returns the directory component of a pathname

    .. versionadded:: 2015.5.0

    This can be useful at the CLI but is frequently useful when scripting.

    .. code-block:: jinja

        {%- from salt['file.dirname'](tpldir) + '/vars.jinja' import parent_vars %}

    CLI Example:

    .. code-block:: bash

        salt '*' file.dirname 'test/path/filename.config'
    '''
    return os.path.dirname(path)


def join(*args):
    '''
    Return a normalized file system path for the underlying OS

    .. versionadded:: 2014.7.0

    This can be useful at the CLI but is frequently useful when scripting
    combining path variables:

    .. code-block:: jinja

        {% set www_root = '/var' %}
        {% set app_dir = 'myapp' %}

        myapp_config:
          file:
            - managed
            - name: {{ salt['file.join'](www_root, app_dir, 'config.yaml') }}

    CLI Example:

    .. code-block:: bash

        salt '*' file.join '/' 'usr' 'local' 'bin'
    '''
    return os.path.join(*args)


def move(src, dst):
    '''
    Move a file or directory

    CLI Example:

    .. code-block:: bash

        salt '*' file.move /path/to/src /path/to/dst
    '''
    src = os.path.expanduser(src)
    dst = os.path.expanduser(dst)

    if not os.path.isabs(src):
        raise SaltInvocationError('Source path must be absolute.')

    if not os.path.isabs(dst):
        raise SaltInvocationError('Destination path must be absolute.')

    ret = {
        'result': True,
        'comment': "'{0}' moved to '{1}'".format(src, dst),
    }

    try:
        shutil.move(src, dst)
    except (OSError, IOError) as exc:
        raise CommandExecutionError(
            "Unable to move '{0}' to '{1}': {2}".format(src, dst, exc)
        )

    return ret


def diskusage(path):
    '''
    Recursively calculate disk usage of path and return it
    in bytes

    CLI Example:

    .. code-block:: bash

        salt '*' file.diskusage /path/to/check
    '''

    total_size = 0
    seen = set()
    if os.path.isfile(path):
        stat_structure = os.stat(path)
        ret = stat_structure.st_size
        return ret

    for dirpath, dirnames, filenames in salt.utils.path.os_walk(path):
        for f in filenames:
            fp = os.path.join(dirpath, f)

            try:
                stat_structure = os.stat(fp)
            except OSError:
                continue

            if stat_structure.st_ino in seen:
                continue

            seen.add(stat_structure.st_ino)

            total_size += stat_structure.st_size

    ret = total_size
    return ret