Implement repository readme validation as part of shed_lint.

Check restructured text if a README.rst is found. Warn the user if they have accidently used a README.md file. Otherwise just present INFO about the readme detected.

Closes #91
Closes #89

planemo/shed_lint.py

@@ -2,6 +2,7 @@
 import re
 import yaml
 from galaxy.tools.lint import LintContext
+from galaxy.tools.linters.help import rst_invalid
 from planemo.lint import lint_xsd
 from planemo.shed import (
     path_to_repo_name,
@@ -55,6 +56,11 @@ def lint_repository(ctx, path, **kwds):
         lint_shed_yaml,
         path,
     )
+    lint_ctx.lint(
+        "readme",
+        lint_readme,
+        path,
+    )
     if kwds["tools"]:
         for (tool_path, tool_xml) in yield_tool_xmls(ctx, path):
             info("+Linting tool %s" % tool_path)
@@ -69,6 +75,41 @@ def lint_repository(ctx, path, **kwds):
     return 1 if failed else 0
 
 
+def lint_readme(path, lint_ctx):
+    readme_rst = os.path.join(path, "README.rst")
+    readme = os.path.join(path, "README")
+    readme_txt = os.path.join(path, "README.txt")
+
+    readme_found = False
+    for readme in [readme_rst, readme, readme_txt]:
+        if os.path.exists(readme):
+            readme_found = readme
+
+    if not readme_found and os.path.exists("README.md"):
+        lint_ctx.warn("Tool Shed doesn't render markdown, "
+                      "README.md is invalid readme.")
+        return
+
+    if not readme_found:
+        # TODO: filter on TYPE and make this a warning if
+        # unrestricted repository - need to update iuc standards
+        # first though.
+        lint_ctx.info("No README found skipping.")
+        return
+
+    if readme_found.endswith(".rst"):
+        readme_text = open(readme_found, "r").read()
+        invalid_rst = rst_invalid(readme_text)
+        if invalid_rst:
+            template = "Invalid restructured text found in README [%s]."
+            msg = template % invalid_rst
+            lint_ctx.warn(msg)
+            return
+        lint_ctx.info("README found containing valid reStructuredText.")
+    else:
+        lint_ctx.info("README found containing plain text.")
+
+
 def lint_tool_dependencies(path, lint_ctx):
     tool_dependencies = os.path.join(path, "tool_dependencies.xml")
     if not os.path.exists(tool_dependencies):

planemo_ext/galaxy/tools/linters/help.py

@@ -18,11 +18,7 @@ def lint_help(tool_xml, lint_ctx):
         return
 
     lint_ctx.valid("Tool contains help section.")
-    invalid_rst = False
-    try:
-        rst_to_html(help)
-    except Exception as e:
-        invalid_rst = str(e)
+    invalid_rst = rst_invalid(help)
 
     if "TODO" in help:
         lint_ctx.warn("Help contains TODO text.")
@@ -31,3 +27,12 @@ def lint_help(tool_xml, lint_ctx):
         lint_ctx.warn("Invalid reStructuredText found in help - [%s]." % invalid_rst)
     else:
         lint_ctx.valid("Help contains valid reStructuredText.")
+
+
+def rst_invalid(text):
+    invalid_rst = False
+    try:
+        rst_to_html(text)
+    except Exception as e:
+        invalid_rst = str(e)
+    return invalid_rst

tests/data/repos/bad_readme_md/.shed.yml

@@ -0,0 +1,8 @@
+name: "single_tool"
+owner: "iuc"
+description: "a simple tool"
+type: "unrestricted"
+remote_repository_url: "https://github.com/galaxyproject/planemo/tree/master/tests/data/repos/single_tool"
+homepage_url: "http://planemo.readthedocs.org/en/latest/"
+categories:
+  - "Text Manipulation"

tests/data/repos/bad_readme_md/README.md

@@ -0,0 +1,2 @@
+This is fine markdown but tool shed doesn't implement markdown so this won't show up and should result in a shed_lint warning.
+

tests/data/repos/bad_readme_md/cat.xml

@@ -0,0 +1,23 @@
+<tool id="cat" name="Concatenate datasets (for test workflows)" version="1.0">
+    <description>tail-to-head</description>
+    <command>
+        cat $input1 #for $q in $queries# ${q.input2} #end for# > $out_file1
+    </command>
+    <inputs>
+        <param name="input1" type="data" label="Concatenate Dataset"/>
+        <repeat name="queries" title="Dataset">
+            <param name="input2" type="data" label="Select" />
+        </repeat>
+    </inputs>
+    <outputs>
+        <data name="out_file1" format="input" metadata_source="input1"/>
+    </outputs>
+    <tests>
+      <test>
+        <param name="input1" value="1.bed"/>
+        <output name="out_file1" file="1.bed"/>
+      </test>
+    </tests>
+    <help>
+    </help>
+</tool>

tests/data/repos/bad_readme_md/related_file

@@ -0,0 +1,2 @@
+A related non-tool file.
+

tests/data/repos/bad_readme_md/test-data/1.bed

@@ -0,0 +1,65 @@
+chr1	147962192	147962580	CCDS989.1_cds_0_0_chr1_147962193_r	0	-
+chr1	147984545	147984630	CCDS990.1_cds_0_0_chr1_147984546_f	0	+
+chr1	148078400	148078582	CCDS993.1_cds_0_0_chr1_148078401_r	0	-
+chr1	148185136	148185276	CCDS996.1_cds_0_0_chr1_148185137_f	0	+
+chr10	55251623	55253124	CCDS7248.1_cds_0_0_chr10_55251624_r	0	-
+chr11	116124407	116124501	CCDS8374.1_cds_0_0_chr11_116124408_r	0	-
+chr11	116206508	116206563	CCDS8377.1_cds_0_0_chr11_116206509_f	0	+
+chr11	116211733	116212337	CCDS8378.1_cds_0_0_chr11_116211734_r	0	-
+chr11	1812377	1812407	CCDS7726.1_cds_0_0_chr11_1812378_f	0	+
+chr12	38440094	38440321	CCDS8736.1_cds_0_0_chr12_38440095_r	0	-
+chr13	112381694	112381953	CCDS9526.1_cds_0_0_chr13_112381695_f	0	+
+chr14	98710240	98712285	CCDS9949.1_cds_0_0_chr14_98710241_r	0	-
+chr15	41486872	41487060	CCDS10096.1_cds_0_0_chr15_41486873_r	0	-
+chr15	41673708	41673857	CCDS10097.1_cds_0_0_chr15_41673709_f	0	+
+chr15	41679161	41679250	CCDS10098.1_cds_0_0_chr15_41679162_r	0	-
+chr15	41826029	41826196	CCDS10101.1_cds_0_0_chr15_41826030_f	0	+
+chr16	142908	143003	CCDS10397.1_cds_0_0_chr16_142909_f	0	+
+chr16	179963	180135	CCDS10401.1_cds_0_0_chr16_179964_r	0	-
+chr16	244413	244681	CCDS10402.1_cds_0_0_chr16_244414_f	0	+
+chr16	259268	259383	CCDS10403.1_cds_0_0_chr16_259269_r	0	-
+chr18	23786114	23786321	CCDS11891.1_cds_0_0_chr18_23786115_r	0	-
+chr18	59406881	59407046	CCDS11985.1_cds_0_0_chr18_59406882_f	0	+
+chr18	59455932	59456337	CCDS11986.1_cds_0_0_chr18_59455933_r	0	-
+chr18	59600586	59600754	CCDS11988.1_cds_0_0_chr18_59600587_f	0	+
+chr19	59068595	59069564	CCDS12866.1_cds_0_0_chr19_59068596_f	0	+
+chr19	59236026	59236146	CCDS12872.1_cds_0_0_chr19_59236027_r	0	-
+chr19	59297998	59298008	CCDS12877.1_cds_0_0_chr19_59297999_f	0	+
+chr19	59302168	59302288	CCDS12878.1_cds_0_0_chr19_59302169_r	0	-
+chr2	118288583	118288668	CCDS2120.1_cds_0_0_chr2_118288584_f	0	+
+chr2	118394148	118394202	CCDS2121.1_cds_0_0_chr2_118394149_r	0	-
+chr2	220190202	220190242	CCDS2441.1_cds_0_0_chr2_220190203_f	0	+
+chr2	220229609	220230869	CCDS2443.1_cds_0_0_chr2_220229610_r	0	-
+chr20	33330413	33330423	CCDS13249.1_cds_0_0_chr20_33330414_r	0	-
+chr20	33513606	33513792	CCDS13255.1_cds_0_0_chr20_33513607_f	0	+
+chr20	33579500	33579527	CCDS13256.1_cds_0_0_chr20_33579501_r	0	-
+chr20	33593260	33593348	CCDS13257.1_cds_0_0_chr20_33593261_f	0	+
+chr21	32707032	32707192	CCDS13614.1_cds_0_0_chr21_32707033_f	0	+
+chr21	32869641	32870022	CCDS13615.1_cds_0_0_chr21_32869642_r	0	-
+chr21	33321040	33322012	CCDS13620.1_cds_0_0_chr21_33321041_f	0	+
+chr21	33744994	33745040	CCDS13625.1_cds_0_0_chr21_33744995_r	0	-
+chr22	30120223	30120265	CCDS13897.1_cds_0_0_chr22_30120224_f	0	+
+chr22	30160419	30160661	CCDS13898.1_cds_0_0_chr22_30160420_r	0	-
+chr22	30665273	30665360	CCDS13901.1_cds_0_0_chr22_30665274_f	0	+
+chr22	30939054	30939266	CCDS13903.1_cds_0_0_chr22_30939055_r	0	-
+chr5	131424298	131424460	CCDS4149.1_cds_0_0_chr5_131424299_f	0	+
+chr5	131556601	131556672	CCDS4151.1_cds_0_0_chr5_131556602_r	0	-
+chr5	131621326	131621419	CCDS4152.1_cds_0_0_chr5_131621327_f	0	+
+chr5	131847541	131847666	CCDS4155.1_cds_0_0_chr5_131847542_r	0	-
+chr6	108299600	108299744	CCDS5061.1_cds_0_0_chr6_108299601_r	0	-
+chr6	108594662	108594687	CCDS5063.1_cds_0_0_chr6_108594663_f	0	+
+chr6	108640045	108640151	CCDS5064.1_cds_0_0_chr6_108640046_r	0	-
+chr6	108722976	108723115	CCDS5067.1_cds_0_0_chr6_108722977_f	0	+
+chr7	113660517	113660685	CCDS5760.1_cds_0_0_chr7_113660518_f	0	+
+chr7	116512159	116512389	CCDS5771.1_cds_0_0_chr7_116512160_r	0	-
+chr7	116714099	116714152	CCDS5773.1_cds_0_0_chr7_116714100_f	0	+
+chr7	116945541	116945787	CCDS5774.1_cds_0_0_chr7_116945542_r	0	-
+chr8	118881131	118881317	CCDS6324.1_cds_0_0_chr8_118881132_r	0	-
+chr9	128764156	128764189	CCDS6914.1_cds_0_0_chr9_128764157_f	0	+
+chr9	128787519	128789136	CCDS6915.1_cds_0_0_chr9_128787520_r	0	-
+chr9	128882427	128882523	CCDS6917.1_cds_0_0_chr9_128882428_f	0	+
+chr9	128937229	128937445	CCDS6919.1_cds_0_0_chr9_128937230_r	0	-
+chrX	122745047	122745924	CCDS14606.1_cds_0_0_chrX_122745048_f	0	+
+chrX	152648964	152649196	CCDS14733.1_cds_0_0_chrX_152648965_r	0	-
+chrX	152691446	152691471	CCDS14735.1_cds_0_0_chrX_152691447_f	0	+
+chrX	152694029	152694263	CCDS14736.1_cds_0_0_chrX_152694030_r	0	-

tests/data/repos/bad_readme_md/test-data/1_bed_random_lines_1_seed_asdf_out.bed

@@ -0,0 +1 @@
+chr5	131424298	131424460	CCDS4149.1_cds_0_0_chr5_131424299_f	0	+

tests/data/repos/bad_readme_rst/.shed.yml

@@ -0,0 +1,8 @@
+name: "single_tool"
+owner: "iuc"
+description: "a simple tool"
+type: "unrestricted"
+remote_repository_url: "https://github.com/galaxyproject/planemo/tree/master/tests/data/repos/single_tool"
+homepage_url: "http://planemo.readthedocs.org/en/latest/"
+categories:
+  - "Text Manipulation"

tests/data/repos/bad_readme_rst/README.rst

@@ -0,0 +1,6 @@
+
+.. header invalid since top and bottom don't match
+
+==========
+Mooo
+=========

tests/data/repos/bad_readme_rst/cat.xml

@@ -0,0 +1,23 @@
+<tool id="cat" name="Concatenate datasets (for test workflows)" version="1.0">
+    <description>tail-to-head</description>
+    <command>
+        cat $input1 #for $q in $queries# ${q.input2} #end for# > $out_file1
+    </command>
+    <inputs>
+        <param name="input1" type="data" label="Concatenate Dataset"/>
+        <repeat name="queries" title="Dataset">
+            <param name="input2" type="data" label="Select" />
+        </repeat>
+    </inputs>
+    <outputs>
+        <data name="out_file1" format="input" metadata_source="input1"/>
+    </outputs>
+    <tests>
+      <test>
+        <param name="input1" value="1.bed"/>
+        <output name="out_file1" file="1.bed"/>
+      </test>
+    </tests>
+    <help>
+    </help>
+</tool>

tests/data/repos/bad_readme_rst/related_file

@@ -0,0 +1,2 @@
+A related non-tool file.
+

tests/data/repos/bad_readme_rst/test-data/1.bed

@@ -0,0 +1,65 @@
+chr1	147962192	147962580	CCDS989.1_cds_0_0_chr1_147962193_r	0	-
+chr1	147984545	147984630	CCDS990.1_cds_0_0_chr1_147984546_f	0	+
+chr1	148078400	148078582	CCDS993.1_cds_0_0_chr1_148078401_r	0	-
+chr1	148185136	148185276	CCDS996.1_cds_0_0_chr1_148185137_f	0	+
+chr10	55251623	55253124	CCDS7248.1_cds_0_0_chr10_55251624_r	0	-
+chr11	116124407	116124501	CCDS8374.1_cds_0_0_chr11_116124408_r	0	-
+chr11	116206508	116206563	CCDS8377.1_cds_0_0_chr11_116206509_f	0	+
+chr11	116211733	116212337	CCDS8378.1_cds_0_0_chr11_116211734_r	0	-
+chr11	1812377	1812407	CCDS7726.1_cds_0_0_chr11_1812378_f	0	+
+chr12	38440094	38440321	CCDS8736.1_cds_0_0_chr12_38440095_r	0	-
+chr13	112381694	112381953	CCDS9526.1_cds_0_0_chr13_112381695_f	0	+
+chr14	98710240	98712285	CCDS9949.1_cds_0_0_chr14_98710241_r	0	-
+chr15	41486872	41487060	CCDS10096.1_cds_0_0_chr15_41486873_r	0	-
+chr15	41673708	41673857	CCDS10097.1_cds_0_0_chr15_41673709_f	0	+
+chr15	41679161	41679250	CCDS10098.1_cds_0_0_chr15_41679162_r	0	-
+chr15	41826029	41826196	CCDS10101.1_cds_0_0_chr15_41826030_f	0	+
+chr16	142908	143003	CCDS10397.1_cds_0_0_chr16_142909_f	0	+
+chr16	179963	180135	CCDS10401.1_cds_0_0_chr16_179964_r	0	-
+chr16	244413	244681	CCDS10402.1_cds_0_0_chr16_244414_f	0	+
+chr16	259268	259383	CCDS10403.1_cds_0_0_chr16_259269_r	0	-
+chr18	23786114	23786321	CCDS11891.1_cds_0_0_chr18_23786115_r	0	-
+chr18	59406881	59407046	CCDS11985.1_cds_0_0_chr18_59406882_f	0	+
+chr18	59455932	59456337	CCDS11986.1_cds_0_0_chr18_59455933_r	0	-
+chr18	59600586	59600754	CCDS11988.1_cds_0_0_chr18_59600587_f	0	+
+chr19	59068595	59069564	CCDS12866.1_cds_0_0_chr19_59068596_f	0	+
+chr19	59236026	59236146	CCDS12872.1_cds_0_0_chr19_59236027_r	0	-
+chr19	59297998	59298008	CCDS12877.1_cds_0_0_chr19_59297999_f	0	+
+chr19	59302168	59302288	CCDS12878.1_cds_0_0_chr19_59302169_r	0	-
+chr2	118288583	118288668	CCDS2120.1_cds_0_0_chr2_118288584_f	0	+
+chr2	118394148	118394202	CCDS2121.1_cds_0_0_chr2_118394149_r	0	-
+chr2	220190202	220190242	CCDS2441.1_cds_0_0_chr2_220190203_f	0	+
+chr2	220229609	220230869	CCDS2443.1_cds_0_0_chr2_220229610_r	0	-
+chr20	33330413	33330423	CCDS13249.1_cds_0_0_chr20_33330414_r	0	-
+chr20	33513606	33513792	CCDS13255.1_cds_0_0_chr20_33513607_f	0	+
+chr20	33579500	33579527	CCDS13256.1_cds_0_0_chr20_33579501_r	0	-
+chr20	33593260	33593348	CCDS13257.1_cds_0_0_chr20_33593261_f	0	+
+chr21	32707032	32707192	CCDS13614.1_cds_0_0_chr21_32707033_f	0	+
+chr21	32869641	32870022	CCDS13615.1_cds_0_0_chr21_32869642_r	0	-
+chr21	33321040	33322012	CCDS13620.1_cds_0_0_chr21_33321041_f	0	+
+chr21	33744994	33745040	CCDS13625.1_cds_0_0_chr21_33744995_r	0	-
+chr22	30120223	30120265	CCDS13897.1_cds_0_0_chr22_30120224_f	0	+
+chr22	30160419	30160661	CCDS13898.1_cds_0_0_chr22_30160420_r	0	-
+chr22	30665273	30665360	CCDS13901.1_cds_0_0_chr22_30665274_f	0	+
+chr22	30939054	30939266	CCDS13903.1_cds_0_0_chr22_30939055_r	0	-
+chr5	131424298	131424460	CCDS4149.1_cds_0_0_chr5_131424299_f	0	+
+chr5	131556601	131556672	CCDS4151.1_cds_0_0_chr5_131556602_r	0	-
+chr5	131621326	131621419	CCDS4152.1_cds_0_0_chr5_131621327_f	0	+
+chr5	131847541	131847666	CCDS4155.1_cds_0_0_chr5_131847542_r	0	-
+chr6	108299600	108299744	CCDS5061.1_cds_0_0_chr6_108299601_r	0	-
+chr6	108594662	108594687	CCDS5063.1_cds_0_0_chr6_108594663_f	0	+
+chr6	108640045	108640151	CCDS5064.1_cds_0_0_chr6_108640046_r	0	-
+chr6	108722976	108723115	CCDS5067.1_cds_0_0_chr6_108722977_f	0	+
+chr7	113660517	113660685	CCDS5760.1_cds_0_0_chr7_113660518_f	0	+
+chr7	116512159	116512389	CCDS5771.1_cds_0_0_chr7_116512160_r	0	-
+chr7	116714099	116714152	CCDS5773.1_cds_0_0_chr7_116714100_f	0	+
+chr7	116945541	116945787	CCDS5774.1_cds_0_0_chr7_116945542_r	0	-
+chr8	118881131	118881317	CCDS6324.1_cds_0_0_chr8_118881132_r	0	-
+chr9	128764156	128764189	CCDS6914.1_cds_0_0_chr9_128764157_f	0	+
+chr9	128787519	128789136	CCDS6915.1_cds_0_0_chr9_128787520_r	0	-
+chr9	128882427	128882523	CCDS6917.1_cds_0_0_chr9_128882428_f	0	+
+chr9	128937229	128937445	CCDS6919.1_cds_0_0_chr9_128937230_r	0	-
+chrX	122745047	122745924	CCDS14606.1_cds_0_0_chrX_122745048_f	0	+
+chrX	152648964	152649196	CCDS14733.1_cds_0_0_chrX_152648965_r	0	-
+chrX	152691446	152691471	CCDS14735.1_cds_0_0_chrX_152691447_f	0	+
+chrX	152694029	152694263	CCDS14736.1_cds_0_0_chrX_152694030_r	0	-

tests/data/repos/bad_readme_rst/test-data/1_bed_random_lines_1_seed_asdf_out.bed

@@ -0,0 +1 @@
+chr5	131424298	131424460	CCDS4149.1_cds_0_0_chr5_131424299_f	0	+

tests/test_shed_lint.py

@@ -4,7 +4,7 @@
 
 class ShedLineTestCase(CliTestCase):
 
-    def test_valid_repo(self):
+    def test_valid_repos(self):
         with self._isolate_repo("single_tool"):
             self._check_exit_code(["shed_lint"])
         with self._isolate_repo("multi_repos_nested"):
@@ -15,3 +15,10 @@ def test_valid_repo(self):
             self._check_exit_code(["shed_lint"])
         with self._isolate_repo("workflow_1"):
             self._check_exit_code(["shed_lint"])
+
+    def test_invalid_repos(self):
+        # And now
+        with self._isolate_repo("bad_readme_rst"):
+            self._check_exit_code(["shed_lint"], exit_code=1)
+        with self._isolate_repo("bad_readme_md"):
+            self._check_exit_code(["shed_lint"], exit_code=1)