start of work to add spack audit packages-https checker

Signed-off-by: vsoch <vsoch@users.noreply.github.com>

lib/spack/spack/audit.py

@@ -37,12 +37,16 @@ def _search_duplicate_compilers(error_cls):
 """
 import collections
 import itertools
+import re
+
+from six.moves.urllib.request import urlopen
 
 try:
     from collections.abc import Sequence  # novm
 except ImportError:
     from collections import Sequence
 
+
 #: Map an audit tag to a list of callables implementing checks
 CALLBACKS = {}
 
@@ -261,6 +265,45 @@ def _search_duplicate_specs_in_externals(error_cls):
     kwargs=('pkgs',)
 )
 
+#: Sanity checks on linting
+# This can take some time, so it's run separately from packages
+package_https_directives = AuditClass(
+    group='packages-https',
+    tag='PKG-HTTPS-DIRECTIVES',
+    description='Sanity checks on https checks of package urls, etc.',
+    kwargs=('pkgs',)
+)
+
+
+@package_https_directives
+def _linting_package_file(pkgs, error_cls):
+    """Check for correctness of links
+    """
+    import llnl.util.lang
+
+    import spack.repo
+    import spack.spec
+
+    errors = []
+    for pkg_name in pkgs:
+        pkg = spack.repo.get(pkg_name)
+
+        # Does the homepage have http, and if so, does https work?
+        if pkg.homepage.startswith('http://'):
+            https = re.sub("http", "https", pkg.homepage, 1)
+            try:
+                response = urlopen(https)
+            except Exception as e:
+                msg = 'Error with attempting https for "{0}": '
+                errors.append(error_cls(msg.format(pkg.name), [str(e)]))
+                continue
+
+            if response.getcode() == 200:
+                msg = 'Package "{0}" uses http but has a valid https endpoint.'
+                errors.append(msg.format(pkg.name))
+
+    return llnl.util.lang.dedupe(errors)
+
 
 @package_directives
 def _unknown_variants_in_directives(pkgs, error_cls):

lib/spack/spack/cmd/audit.py

@@ -19,12 +19,17 @@ def setup_parser(subparser):
     # Audit configuration files
     sp.add_parser('configs', help='audit configuration files')
 
+    # Https and other linting
+    https_parser = sp.add_parser('packages-https', help='check https in packages')
+
     # Audit package recipes
     pkg_parser = sp.add_parser('packages', help='audit package recipes')
-    pkg_parser.add_argument(
-        'name', metavar='PKG', nargs='*',
-        help='package to be analyzed (if none all packages will be processed)',
-    )
+
+    for group in [pkg_parser, https_parser]:
+        group.add_argument(
+            'name', metavar='PKG', nargs='*',
+            help='package to be analyzed (if none all packages will be processed)',
+        )
 
     # List all checks
     sp.add_parser('list', help='list available checks and exits')
@@ -41,6 +46,12 @@ def packages(parser, args):
     _process_reports(reports)
 
 
+def packages_https(parser, args):
+    pkgs = args.name or spack.repo.path.all_package_names()
+    reports = spack.audit.run_group(args.subcommand, pkgs=pkgs)
+    _process_reports(reports)
+
+
 def list(parser, args):
     for subcommand, check_tags in spack.audit.GROUPS.items():
         print(cl.colorize('@*b{' + subcommand + '}:'))
@@ -58,6 +69,7 @@ def audit(parser, args):
     subcommands = {
         'configs': configs,
         'packages': packages,
+        'packages-https': packages_https,
         'list': list
     }
     subcommands[args.subcommand](parser, args)

share/spack/spack-completion.bash

@@ -386,14 +386,23 @@ _spack_audit() {
     then
         SPACK_COMPREPLY="-h --help"
     else
-        SPACK_COMPREPLY="configs packages list"
+        SPACK_COMPREPLY="configs packages-https packages list"
     fi
 }
 
 _spack_audit_configs() {
     SPACK_COMPREPLY="-h --help"
 }
 
+_spack_audit_packages_https() {
+    if $list_options
+    then
+        SPACK_COMPREPLY="-h --help"
+    else
+        SPACK_COMPREPLY=""
+    fi
+}
+
 _spack_audit_packages() {
     if $list_options
     then

var/spack/repos/builtin/packages/abyss/package.py

@@ -22,7 +22,7 @@ class Abyss(AutotoolsPackage):
        that is designed for short reads. The single-processor version
        is useful for assembling genomes up to 100 Mbases in size."""
 
-    homepage = "http://www.bcgsc.ca/platform/bioinfo/software/abyss"
+    homepage = "https://www.bcgsc.ca/platform/bioinfo/software/abyss"
     url      = "https://github.com/bcgsc/abyss/releases/download/2.3.1/abyss-2.3.1.tar.gz"
 
     version('2.3.1', sha256='664045e7903e9732411effc38edb9ebb1a0c1b7636c64b3a14a681f465f43677')

var/spack/repos/builtin/packages/acct/package.py

@@ -9,7 +9,7 @@
 class Acct(AutotoolsPackage):
     """Utilities for monitoring process activities."""
 
-    homepage = "http://www.gnu.org/software/acct"
+    homepage = "https://www.gnu.org/software/acct"
     url      = "https://ftp.gnu.org/gnu/acct/acct-6.6.4.tar.gz"
 
     version('6.6.4', sha256='4c15bf2b58b16378bcc83f70e77d4d40ab0b194acf2ebeefdb507f151faa663f')

var/spack/repos/builtin/packages/acts/package.py

@@ -29,7 +29,7 @@ class Acts(CMakePackage, CudaPackage):
     propagation and fitting, basic seed finding algorithms.
     """
 
-    homepage = "http://acts.web.cern.ch/ACTS/"
+    homepage = "https://acts.web.cern.ch/ACTS/"
     git      = "https://github.com/acts-project/acts.git"
     maintainers = ['HadrienG2']
 

var/spack/repos/builtin/packages/adios/package.py

@@ -13,7 +13,7 @@ class Adios(AutotoolsPackage):
     read, or processed outside of the running simulation.
     """
 
-    homepage = "http://www.olcf.ornl.gov/center-projects/adios/"
+    homepage = "https://www.olcf.ornl.gov/center-projects/adios/"
     url      = "https://github.com/ornladios/ADIOS/archive/v1.12.0.tar.gz"
     git      = "https://github.com/ornladios/ADIOS.git"
 

var/spack/repos/builtin/packages/advancecomp/package.py

@@ -10,7 +10,7 @@ class Advancecomp(AutotoolsPackage):
     """AdvanceCOMP contains recompression utilities for your .zip archives,
     .png images, .mng video clips and .gz files."""
 
-    homepage = "http://www.advancemame.it"
+    homepage = "https://www.advancemame.it"
     url      = "https://github.com/amadvance/advancecomp/archive/v2.1.tar.gz"
 
     version('2.1',  sha256='6113c2b6272334af710ba486e8312faa3cee5bd6dc8ca422d00437725e2b602a')

var/spack/repos/builtin/packages/aegean/package.py

@@ -12,7 +12,7 @@ class Aegean(MakefilePackage):
        as well as a C library whose API provides access to AEGeAn's core
        functions and data structures."""
 
-    homepage = "http://brendelgroup.github.io/AEGeAn/"
+    homepage = "https://brendelgroup.github.io/AEGeAn/"
     url      = "https://github.com/BrendelGroup/AEGeAn/archive/v0.15.2.tar.gz"
 
     version('0.15.2', sha256='734c9dd23ab3415c3966083bfde5fb72c81e6ace84e08ee3fe0d4c338331d975')

var/spack/repos/builtin/packages/agrep/package.py

@@ -13,7 +13,7 @@ class Agrep(MakefilePackage):
     Developed 1989-1991 by Udi Manber, Sun Wu et al. at the University
     of Arizona."""
 
-    homepage = "http://www.tgries.de/agrep"
+    homepage = "https://www.tgries.de/agrep"
     url      = "https://www.tgries.de/agrep/agrep-3.41.tgz"
 
     version('3.41', sha256='0508eafaf9725fc67cc955eb6d32ba4f50138443a4fea4275508d2c3f67a234e')

var/spack/repos/builtin/packages/aida/package.py

@@ -9,7 +9,7 @@
 class Aida(Package):
     """Abstract Interfaces for Data Analysis"""
 
-    homepage = "http://aida.freehep.org/"
+    homepage = "https://aida.freehep.org/"
     url      = "ftp://ftp.slac.stanford.edu/software/freehep/AIDA/v3.2.1/aida-3.2.1.tar.gz"
 
     tags = ['hep']

var/spack/repos/builtin/packages/allpaths-lg/package.py

@@ -10,7 +10,7 @@ class AllpathsLg(AutotoolsPackage):
     """ALLPATHS-LG is our original short read assembler and it works on both
        small and large (mammalian size) genomes."""
 
-    homepage = "http://www.broadinstitute.org/software/allpaths-lg/blog/"
+    homepage = "https://www.broadinstitute.org/software/allpaths-lg/blog/"
     url      = "ftp://ftp.broadinstitute.org/pub/crd/ALLPATHS/Release-LG/latest_source_code/allpathslg-52488.tar.gz"
 
     version('52488', sha256='035b49cb21b871a6b111976757d7aee9c2513dd51af04678f33375e620998542')

var/spack/repos/builtin/packages/amber/package.py

@@ -22,7 +22,7 @@ class Amber(Package, CudaPackage):
        Only the latter version of ambertools for each amber version is supported.
        """
 
-    homepage = "http://ambermd.org/"
+    homepage = "https://ambermd.org/"
     url = "file://{0}/Amber18.tar.bz2".format(os.getcwd())
     manual_download = True
 

var/spack/repos/builtin/packages/amdlibflame/package.py

@@ -30,7 +30,7 @@ class Amdlibflame(LibflameBase):
     """
 
     _name = 'amdlibflame'
-    homepage = "http://developer.amd.com/amd-cpu-libraries/blas-library/#libflame"
+    homepage = "https://developer.amd.com/amd-cpu-libraries/blas-library/#libflame"
     url = "https://github.com/amd/libflame/archive/3.0.tar.gz"
     git = "https://github.com/amd/libflame.git"
 

var/spack/repos/builtin/packages/ant/package.py

@@ -12,7 +12,7 @@ class Ant(Package):
        dependent upon each other
     """
 
-    homepage = "http://ant.apache.org/"
+    homepage = "https://ant.apache.org/"
     url = "https://archive.apache.org/dist/ant/source/apache-ant-1.9.7-src.tar.gz"
 
     version('1.10.7', sha256='2f9c4ef094581663b41a7412324f65b854f17622e5b2da9fcb9541ca8737bd52')

var/spack/repos/builtin/packages/ape/package.py

@@ -10,7 +10,7 @@ class Ape(Package):
     """A tool for generating atomic pseudopotentials within a Density-Functional
     Theory framework"""
 
-    homepage = "http://www.tddft.org/programs/APE/"
+    homepage = "https://www.tddft.org/programs/APE/"
     url      = "http://www.tddft.org/programs/APE/sites/default/files/ape-2.2.1.tar.gz"
 
     version('2.2.1', sha256='1bdb7f987fde81f8a5f335da6b59fa884e6d185d4a0995c90fde7c04376ce9e3')

var/spack/repos/builtin/packages/appres/package.py

@@ -12,7 +12,7 @@ class Appres(AutotoolsPackage, XorgPackage):
     names.  It can be used to determine which resources a particular
     program will load."""
 
-    homepage = "http://cgit.freedesktop.org/xorg/app/appres"
+    homepage = "https://cgit.freedesktop.org/xorg/app/appres"
     xorg_mirror_path = "app/appres-1.0.4.tar.gz"
 
     version('1.0.4', sha256='22cb6f639c891ffdbb5371bc50a88278185789eae6907d05e9e0bd1086a80803')

var/spack/repos/builtin/packages/argobots/package.py

@@ -15,7 +15,7 @@ class Argobots(AutotoolsPackage):
     mapping, and data placement strategies. It consists of an
     execution model and a memory model."""
 
-    homepage = "http://www.argobots.org/"
+    homepage = "https://www.argobots.org/"
     url      = "https://github.com/pmodels/argobots/releases/download/v1.0b1/argobots-1.0b1.tar.gz"
     git      = "https://github.com/pmodels/argobots.git"
     maintainers = ['shintaro-iwasaki']

var/spack/repos/builtin/packages/arrow/package.py

@@ -11,7 +11,7 @@ class Arrow(CMakePackage, CudaPackage):
     This package contains the C++ bindings.
     """
 
-    homepage = "http://arrow.apache.org"
+    homepage = "https://arrow.apache.org"
     url      = "https://github.com/apache/arrow/archive/apache-arrow-0.9.0.tar.gz"
 
     version('4.0.1',  sha256='79d3e807df4a179cfab1e7a1ab5f79d95f7b72ac2c33aba030febd125d77eb3b')

var/spack/repos/builtin/packages/asio/package.py

@@ -11,7 +11,7 @@
 class Asio(AutotoolsPackage):
     """C++ library for network and low-level I/O programming."""
 
-    homepage = "http://think-async.com/Asio/"
+    homepage = "https://think-async.com/Asio/"
     url = "https://github.com/chriskohlhoff/asio/archive/1.18.2.tar.gz"
     git = "https://github.com/chriskohlhoff/asio.git"
     maintainers = ["msimberg"]

var/spack/repos/builtin/packages/astra/package.py

@@ -9,7 +9,7 @@
 class Astra(Package):
     """A Space Charge Tracking Algorithm."""
 
-    homepage = "http://www.desy.de/~mpyflo/"
+    homepage = "https://www.desy.de/~mpyflo/"
 
     version('2020-02-03',
             sha256='ca9ee7d3d369f9040fbd595f57f3153f712d789b66385fd2d2de88a69a774b83',

var/spack/repos/builtin/packages/atompaw/package.py

@@ -15,7 +15,7 @@ class Atompaw(Package):
 
     User's guide: ~/doc/atompaw-usersguide.pdf
     """
-    homepage = "http://users.wfu.edu/natalie/papers/pwpaw/man.html"
+    homepage = "https://users.wfu.edu/natalie/papers/pwpaw/man.html"
     url = "http://users.wfu.edu/natalie/papers/pwpaw/atompaw-4.0.0.13.tar.gz"
 
     version('4.1.1.0', sha256='b1ee2b53720066655d98523ef337e54850cb1e68b3a2da04ff5a1576d3893891')

var/spack/repos/builtin/packages/augustus/package.py

@@ -12,7 +12,7 @@ class Augustus(MakefilePackage):
     """AUGUSTUS is a program that predicts genes in eukaryotic
        genomic sequences"""
 
-    homepage = "http://bioinf.uni-greifswald.de/augustus/"
+    homepage = "https://bioinf.uni-greifswald.de/augustus/"
     url      = "https://github.com/Gaius-Augustus/Augustus/archive/v3.3.4.tar.gz"
 
     # Releases have moved to github

var/spack/repos/builtin/packages/autofact/package.py

@@ -11,7 +11,7 @@
 class Autofact(Package):
     """An Automatic Functional Annotation and Classification Tool"""
 
-    homepage = "http://megasun.bch.umontreal.ca/Software/AutoFACT.htm"
+    homepage = "https://megasun.bch.umontreal.ca/Software/AutoFACT.htm"
     url      = "http://megasun.bch.umontreal.ca/Software/AutoFACT_v3_4.tar"
 
     version('3_4', sha256='1465d263b19adb42f01f6e636ac40ef1c2e3dbd63461f977b89da9493fe9c6f4')

var/spack/repos/builtin/packages/automake/package.py

@@ -8,7 +8,7 @@
 class Automake(AutotoolsPackage, GNUMirrorPackage):
     """Automake -- make file builder part of autotools"""
 
-    homepage = 'http://www.gnu.org/software/automake/'
+    homepage = 'https://www.gnu.org/software/automake/'
     gnu_mirror_path = 'automake/automake-1.15.tar.gz'
 
     version('1.16.3', sha256='ce010788b51f64511a1e9bb2a1ec626037c6d0e7ede32c1c103611b9d3cba65f')

var/spack/repos/builtin/packages/babeltrace/package.py

@@ -11,7 +11,7 @@ class Babeltrace(AutotoolsPackage):
     Common Trace Format (CTF). Its main use is to pretty-print CTF traces
     into a human-readable text output ordered by time."""
 
-    homepage = "http://www.efficios.com/babeltrace"
+    homepage = "https://www.efficios.com/babeltrace"
     url      = "https://www.efficios.com/files/babeltrace/babeltrace-1.2.4.tar.bz2"
 
     version('1.2.4', sha256='666e3a1ad2dc7d5703059963056e7800f0eab59c8eeb6be2efe4f3acc5209eb1')

var/spack/repos/builtin/packages/bamutil/package.py

@@ -12,7 +12,7 @@ class Bamutil(MakefilePackage):
        are built into a single executable, bam.
     """
 
-    homepage = "http://genome.sph.umich.edu/wiki/BamUtil"
+    homepage = "https://genome.sph.umich.edu/wiki/BamUtil"
     url      = "http://genome.sph.umich.edu/w/images/7/70/BamUtilLibStatGen.1.0.13.tgz"
 
     version('1.0.13', sha256='16c1d01c37d1f98b98c144f3dd0fda6068c1902f06bd0989f36ce425eb0c592b')

var/spack/repos/builtin/packages/barvinok/package.py

@@ -10,7 +10,7 @@ class Barvinok(AutotoolsPackage):
     """barvinok is a library for counting the number of integer points in parametric and
     non-parametric polytopes."""
 
-    homepage = "http://barvinok.gforge.inria.fr"
+    homepage = "https://barvinok.gforge.inria.fr"
     url      = "http://barvinok.gforge.inria.fr/barvinok-0.41.5.tar.bz2"
 
     version('0.41.5', sha256='e70493318fe76c0c202f98d7861bdf5dda8c4d79c21024af2e04b009ffa79734')

var/spack/repos/builtin/packages/bcftools/package.py

@@ -10,7 +10,7 @@ class Bcftools(AutotoolsPackage):
        commands work transparently with both VCFs and BCFs, both
        uncompressed and BGZF-compressed."""
 
-    homepage = "http://samtools.github.io/bcftools/"
+    homepage = "https://samtools.github.io/bcftools/"
     url      = "https://github.com/samtools/bcftools/releases/download/1.3.1/bcftools-1.3.1.tar.bz2"
 
     version('1.12', sha256='7a0e6532b1495b9254e38c6698d955e5176c1ee08b760dfea2235ee161a024f5')

var/spack/repos/builtin/packages/bdftopcf/package.py

@@ -14,7 +14,7 @@ class Bdftopcf(AutotoolsPackage, XorgPackage):
     appropriate machine, but the files are still portable (but read more
     slowly) on other machines."""
 
-    homepage = "http://cgit.freedesktop.org/xorg/app/bdftopcf"
+    homepage = "https://cgit.freedesktop.org/xorg/app/bdftopcf"
     xorg_mirror_path = "app/bdftopcf-1.0.5.tar.gz"
 
     version('1.0.5', sha256='78a5ec945de1d33e6812167b1383554fda36e38576849e74a9039dc7364ff2c3')

var/spack/repos/builtin/packages/beagle/package.py

@@ -12,7 +12,7 @@ class Beagle(Package):
     """Beagle is a software package for phasing genotypes and for imputing
        ungenotyped markers."""
 
-    homepage = "http://faculty.washington.edu/browning/beagle/beagle.html"
+    homepage = "https://faculty.washington.edu/browning/beagle/beagle.html"
 
     version('5.1', sha256='994f926a4ec0eac665631f37c4a961d3f75c966c71841079275364013c90996c',
             expand=False, url='http://faculty.washington.edu/browning/beagle/beagle.25Nov19.28d.jar')

var/spack/repos/builtin/packages/beast-tracer/package.py

@@ -12,7 +12,7 @@ class BeastTracer(Package):
     """Tracer is a graphical tool for visualization and diagnostics of MCMC
        output."""
 
-    homepage = "http://beast.community/tracer"
+    homepage = "https://beast.community/tracer"
     url      = "https://github.com/beast-dev/tracer/archive/v1.7.1.tar.gz"
 
     version('1.7.1', sha256='947d51c5afa52354099b9b182ba6036e352356bd62df94031f33cdcb7e8effd3')