[win_get_url] feature: Add support checksum to module win_get_url

changelogs/fragments/win_get_url-checksum.yaml

@@ -0,0 +1,3 @@
+minor_changes:
+- win_get_url - Add the ``checksum`` option to verify the integrity of a downloaded file.
+- win_get_url - Add idempotency check if the remote file has the same contents as the dest file.

lib/ansible/modules/windows/win_get_url.py

@@ -34,7 +34,7 @@
     required: yes
   force:
     description:
-    - If C(yes), will always download the file. If C(no), will only
+    - If C(yes), will download the file every time and replace the file if the contents change. If C(no), will only
       download the file if it does not exist or the remote file has been
       modified more recently than the local file.
     - This works by sending an http HEAD request to retrieve last modified
@@ -73,6 +73,36 @@
     type: bool
     default: yes
     version_added: '2.4'
+  checksum:
+    description:
+      - If a I(checksum) is passed to this parameter, the digest of the
+        destination file will be calculated after it is downloaded to ensure
+        its integrity and verify that the transfer completed successfully.
+      - This option cannot be set with I(checksum_url).
+    type: str
+    version_added: "2.8"
+  checksum_algorithm:
+    description:
+      - Specifies the hashing algorithm used when calculating the checksum of
+        the remote and destination file.
+    type: str
+    choices:
+      - md5
+      - sha1
+      - sha256
+      - sha385
+      - sha512
+    default: sha1
+    version_added: "2.8"
+  checksum_url:
+    description:
+      - Specifies a URL that contains the checksum values for the resource at
+        I(url).
+      - Like C(checksum), this is used to verify the integrity of the remote
+        transfer.
+      - This option cannot be set with I(checksum).
+    type: str
+    version_added: "2.8"
   proxy_url:
     description:
     - The full URL of the proxy server to download through.
@@ -105,6 +135,9 @@
 - If your URL includes an escaped slash character (%2F) this module will convert it to a real slash.
   This is a result of the behaviour of the System.Uri class as described in
   L(the documentation,https://docs.microsoft.com/en-us/dotnet/framework/configure-apps/file-schema/network/schemesettings-element-uri-settings#remarks).
+- Since Ansible 2.8, the module will skip reporting a change if the remote
+  checksum is the same as the local local even when C(force=yes). This is to
+  better align with M(get_url).
 seealso:
 - module: get_url
 - module: uri
@@ -140,6 +173,22 @@
     dest: '%TEMP%\ftp-file.txt'
     url_username: ftp-user
     url_password: ftp-password
+
+- name: Download src with sha256 checksum url
+  win_get_url:
+    url: http://www.example.com/earthrise.jpg
+    dest: C:\temp\earthrise.jpg
+    checksum_url: http://www.example.com/sha256sum.txt
+    checksum_algorithm: sha256
+    force: True
+
+- name: Download src with sha256 checksum url
+  win_get_url:
+    url: http://www.example.com/earthrise.jpg
+    dest: C:\temp\earthrise.jpg
+    checksum: a97e6837f60cec6da4491bab387296bbcd72bdba
+    checksum_algorithm: sha1
+    force: True
 '''
 
 RETURN = r'''
@@ -148,11 +197,26 @@
     returned: always
     type: str
     sample: C:\Users\RandomUser\earthrise.jpg
+checksum_dest:
+    description: <algorithm> checksum of the file after the download
+    returned: success and dest has been downloaded
+    type: str
+    sample: 6e642bb8dd5c2e027bf21dd923337cbb4214f827
+checksum_src:
+    description: <algorithm> checksum of the remote resource
+    returned: force=yes or dest did not exist
+    type: str
+    sample: 6e642bb8dd5c2e027bf21dd923337cbb4214f827
 elapsed:
     description: The elapsed seconds between the start of poll and the end of the module.
     returned: always
     type: float
     sample: 2.1406487
+size:
+    description: size of the dest file
+    returned: success
+    type: int
+    sample: 1220
 url:
     description: requested url
     returned: always

test/integration/targets/win_get_url/defaults/main.yml

@@ -2,3 +2,6 @@
 test_win_get_url_path: '{{win_output_dir}}\win_get_url'
 test_win_get_url_host: www.redhat.com
 test_win_get_url_env_var: WIN_GET_URL
+
+remote_tmp_path: '{{win_output_dir}}\win_get_url_dest'
+remote_http_path: '{{test_win_get_url_path}}\http'

test/integration/targets/win_get_url/tasks/main.yml

@@ -1,8 +1,12 @@
 ---
 - name: ensure testing folder is present
   win_file:
-    path: '{{test_win_get_url_path}}'
+    path: '{{ item }}'
     state: directory
+  loop:
+    - '{{ test_win_get_url_path }}'
+    - '{{ remote_http_path }}'
+    - '{{ remote_tmp_path }}'
 
 - name: copy across testing files
   win_copy:
@@ -41,6 +45,9 @@
   - name: run FTP tests
     include_tasks: tests_ftp.yml
 
+  - name: run checksum tests
+    include_tasks: tests_checksum.yml
+
   always:
   - name: remove SlimFTPd service
     win_service:
@@ -53,7 +60,10 @@
       level: machine
       state: absent
 
-  - name: remove testing folder
+  - name: remove testing folders
     win_file:
-      path: '{{test_win_get_url_path}}'
+      path: '{{ item }}'
       state: absent
+    loop:
+      - '{{ test_win_get_url_path }}'
+      - '{{ remote_tmp_path }}'

test/integration/targets/win_get_url/tasks/tests_checksum.yml

@@ -0,0 +1,202 @@
+---
+- name: create src file
+  win_copy:
+    dest: '{{ remote_http_path }}\27617.txt'
+    content: 'ptux'
+
+- name: create sha1 checksum file of src
+  win_copy:
+    dest: '{{ remote_http_path }}\sha1sum.txt'
+    content: 'a97e6837f60cec6da4491bab387296bbcd72bdba  27617.txt'
+
+- name: add sha1 checksum not going to be downloaded
+  win_lineinfile:
+    dest: '{{ remote_http_path }}\sha1sum.txt'
+    line: '{{ item }}'
+  loop:
+    - '3911340502960ca33aece01129234460bfeb2791  not_target1.txt'
+    - '1b4b6adf30992cedb0f6edefd6478ff0a593b2e4  not_target2.txt'
+
+- name: create sha256 checksum file of src
+  win_copy:
+    dest: '{{ remote_http_path }}\sha256sum.txt'
+    content: 'b1b6ce5073c8fac263a8fc5edfffdbd5dec1980c784e09c5bc69f8fb6056f006.  27617.txt'
+
+- name: add sha256 checksum not going to be downloaded
+  win_lineinfile:
+    dest: '{{ remote_http_path }}\sha256sum.txt'
+    line: '{{ item }}'
+  loop:
+    - '30949cc401e30ac494d695ab8764a9f76aae17c5d73c67f65e9b558f47eff892  not_target1.txt'
+    - 'd0dbfc1945bc83bf6606b770e442035f2c4e15c886ee0c22fb3901ba19900b5b  not_target2.txt'
+
+- name: create sha256 checksum file of src with a dot leading path
+  win_copy:
+    dest: '{{ remote_http_path }}\sha256sum_with_dot.txt'
+    content: 'b1b6ce5073c8fac263a8fc5edfffdbd5dec1980c784e09c5bc69f8fb6056f006.  ./27617.txt'
+
+- name: add sha256 checksums with dot leading path not going to be downloaded
+  win_lineinfile:
+    dest: '{{ remote_http_path }}\sha256sum_with_dot.txt'
+    line: '{{ item }}'
+  loop:
+    - '30949cc401e30ac494d695ab8764a9f76aae17c5d73c67f65e9b558f47eff892  ./not_target1.txt'
+    - 'd0dbfc1945bc83bf6606b770e442035f2c4e15c886ee0c22fb3901ba19900b5b  ./not_target2.txt'
+
+- name: copy files to ftp
+  win_copy:
+    src: '{{ remote_http_path }}\'
+    dest: '{{ test_win_get_url_path }}\ftp\{{ item }}\'
+    remote_src: yes
+  loop: '{{ dest_folders }}'
+  vars:
+    dest_folders:
+      - anon
+      - user
+      - user-pass
+
+- name: download src with sha1 checksum url
+  win_get_url:
+    url: 'ftp://localhost/anon/27617.txt'
+    dest: '{{ remote_tmp_path }}'
+    checksum_url: 'ftp://localhost/anon/sha1sum.txt'
+    force: True
+  register: result_sha1
+
+- name: download src with sha1 checksum value
+  win_get_url:
+    url: 'ftp://localhost/anon/27617.txt'
+    dest: '{{ remote_tmp_path }}'
+    checksum: 'a97e6837f60cec6da4491bab387296bbcd72bdba'
+    force: True
+  register: result_sha1_alt
+
+- win_stat:
+    path: '{{ remote_tmp_path }}\27617.txt'
+  register: stat_result_sha1
+
+- name: download src with sha256 checksum url
+  win_get_url:
+    url: 'ftp://localhost/anon/27617.txt'
+    dest: '{{ remote_tmp_path }}\27617sha256.txt'
+    checksum_url: 'ftp://localhost/anon/sha256sum.txt'
+    checksum_algorithm: sha256
+    force: True
+  register: result_sha256
+
+- win_stat:
+    path: '{{ remote_tmp_path }}\27617.txt'
+  register: stat_result_sha256
+
+- name: download src with sha256 checksum url with dot leading paths
+  win_get_url:
+    url: 'ftp://localhost/anon/27617.txt'
+    dest: '{{ remote_tmp_path }}\27617sha256_with_dot.txt'
+    checksum_url: 'ftp://localhost/anon/sha256sum_with_dot.txt'
+    checksum_algorithm: sha256
+    force: True
+  register: result_sha256_with_dot
+
+- win_stat:
+    path: '{{ remote_tmp_path }}\27617sha256_with_dot.txt'
+  register: stat_result_sha256_with_dot
+
+- name: Assert that the file was downloaded
+  assert:
+    that:
+      - result_sha1 is changed
+      - result_sha1.checksum_dest == 'a97e6837f60cec6da4491bab387296bbcd72bdba'
+      - result_sha1_alt is succeeded
+      - not result_sha1_alt is changed
+      - result_sha1_alt.checksum_dest == 'a97e6837f60cec6da4491bab387296bbcd72bdba'
+      - result_sha256 is changed
+      - result_sha256_with_dot is changed
+      - stat_result_sha1.stat.exists | bool
+      - stat_result_sha256.stat.exists | bool
+      - stat_result_sha256_with_dot.stat.exists | bool
+
+# Check download with force: False
+
+- name: download src with sha1 checksum url
+  win_get_url:
+    url: 'ftp://localhost/anon/27617.txt'
+    dest: '{{ remote_tmp_path }}'
+    checksum_url: 'ftp://localhost/anon/sha1sum.txt'
+    checksum_algorithm: sha1
+    force: False
+  register: result_sha1_no_force
+
+- name: download src with sha256 checksum url
+  win_get_url:
+    url: 'ftp://localhost/anon/27617.txt'
+    dest: '{{ remote_tmp_path }}/27617sha256.txt'
+    checksum_url: 'ftp://localhost/anon/sha256sum.txt'
+    checksum_algorithm: sha256
+    force: False
+  register: result_sha256_no_force
+
+- name: assert download single file with force no
+  assert:
+    that:
+      - result_sha1_no_force is not changed
+      - result_sha256_no_force is not changed
+
+# Check download with check_mode: True
+
+- name: download src with sha1 checksum url | checkmode
+  win_get_url:
+    url: 'ftp://localhost/anon/27617.txt'
+    dest: '{{ remote_tmp_path }}'
+    checksum_url: 'ftp://localhost/anon/sha1sum.txt'
+    checksum_algorithm: sha1
+  check_mode: True
+  register: result_sha1_check_mode
+
+- name: download src with sha256 checksum url | checkmode
+  win_get_url:
+    url: 'ftp://localhost/anon/27617.txt'
+    dest: '{{ remote_tmp_path }}\27617sha256.txt'
+    checksum_url: 'ftp://localhost/anon/sha256sum.txt'
+    checksum_algorithm: sha256
+  check_mode: True
+  register: result_sha256_check_mode
+
+- name: assert download single file with force no
+  assert:
+    that:
+      - result_sha1_check_mode.checksum_dest == 'a97e6837f60cec6da4491bab387296bbcd72bdba'
+      - result_sha256_check_mode.checksum_dest == 'b1b6ce5073c8fac263a8fc5edfffdbd5dec1980c784e09c5bc69f8fb6056f006'
+      - result_sha1_check_mode.checksum_src | length
+      - result_sha256_check_mode.checksum_src | length
+
+# Check download with wrong checksum
+
+- name: download src with sha1 checksum value | fail
+  win_get_url:
+    url: 'ftp://localhost/anon/27617.txt'
+    dest: '{{ remote_tmp_path }}'
+    checksum: 'redacted'
+    checksum_algorithm: sha1
+  failed_when:
+    - '"did not match" not in result_sha1_failed.msg'
+    - '"The checksum" not in result_sha1_failed.msg'
+    - '"Unknown error downloading" not in result_sha1_failed.msg'
+  register: result_sha1_failed
+
+- name: download src with sha256 checksum value | fail
+  win_get_url:
+    url: 'ftp://localhost/anon/27617.txt'
+    dest: '{{ remote_tmp_path }}\27617sha256.txt'
+    checksum: 'redacted'
+    checksum_algorithm: sha256
+  failed_when:
+    - '"did not match" not in result_sha256_failed.msg'
+    - '"The checksum" not in result_sha256_failed.msg'
+    - '"Unknown error downloading" not in result_sha256_failed.msg'
+  register: result_sha256_failed
+
+- name: assert failed downloads
+  assert:
+    that:
+      - result_sha1_failed is succeeded
+      - result_sha256_failed is succeeded

test/integration/targets/win_get_url/tasks/tests_url.yml

@@ -39,7 +39,16 @@
     - http_download.dest
     - http_download_result.stat.exists
 
-# TODO: add check for idempotent run once it is added with force: yes
+- name: download single file (idempotent)
+  win_get_url:
+    url: https://{{test_win_get_url_host}}
+    dest: '{{test_win_get_url_path}}\web.html'
+  register: http_download_again
+
+- name: assert download single file (idempotent)
+  assert:
+    that:
+    - not http_download_again is changed
 
 - name: download single file with force no
   win_get_url:
@@ -53,8 +62,10 @@
     that:
     - http_download_no_force is not changed
 
-- name: manually change last modified time on FTP source to older datetime
-  win_shell: (Get-Item -Path '{{test_win_get_url_path}}\web.html').LastWriteTime = (Get-Date -Date "01/01/1970")
+- name: manually change the content and last modified time on FTP source to older datetime
+  win_shell: |
+    Set-Content -Path '{{test_win_get_url_path}}\web.html' -Value 'abc'
+    (Get-Item -Path '{{test_win_get_url_path}}\web.html').LastWriteTime = (Get-Date -Date "01/01/1970")
 
 - name: download newer file with force no
   win_get_url:

test/sanity/pslint/ignore.txt

@@ -38,7 +38,7 @@ lib/ansible/modules/windows/win_find.ps1 PSAvoidUsingEmptyCatchBlock
 lib/ansible/modules/windows/win_find.ps1 PSAvoidUsingWMICmdlet
 lib/ansible/modules/windows/win_firewall_rule.ps1 PSAvoidUsingCmdletAliases
 lib/ansible/modules/windows/win_firewall_rule.ps1 PSUseApprovedVerbs
-lib/ansible/modules/windows/win_get_url.ps1 PSUseApprovedVerbs
+lib/ansible/modules/windows/win_get_url.ps1 PSUsePSCredentialType  # Credential param can take a base64 encoded string as well as a PSCredential
 lib/ansible/modules/windows/win_hotfix.ps1 PSUseApprovedVerbs
 lib/ansible/modules/windows/win_iis_webbinding.ps1 PSUseApprovedVerbs
 lib/ansible/modules/windows/win_iis_website.ps1 PSAvoidUsingCmdletAliases