From 20f14f7a2cc1a7c0ed7df4cbfb4c154b43d5345b Mon Sep 17 00:00:00 2001 From: Matthew Brett Date: Wed, 28 Nov 2018 12:57:03 +0000 Subject: [PATCH 1/4] Rebase case-sensitive FS changes, refine Added extra flag to signify local build; only check case-sensitivity of pages in that case. --- scripts/generate_book.py | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/scripts/generate_book.py b/scripts/generate_book.py index 5bcd1f26a..8de74429c 100644 --- a/scripts/generate_book.py +++ b/scripts/generate_book.py @@ -8,6 +8,7 @@ from tqdm import tqdm import numpy as np from glob import glob +from uuid import uuid4 import argparse DESCRIPTION = ("Convert a collection of Jupyter Notebooks into Jekyll " @@ -26,6 +27,8 @@ parser.add_argument("--path-toc", default=None, help="Path to the Table of Contents YAML file") parser.add_argument("--overwrite", action='store_true', help="Overwrite md files if they already exist.") parser.add_argument("--execute", action='store_true', help="Execute notebooks before converting to MD.") +parser.add_argument("--local-build", action='store_true', + help="Specify you are building site locally for later upload.") parser.set_defaults(overwrite=False, execute=False) # Defaults @@ -75,6 +78,24 @@ def _copy_non_content_files(): sh.copy2(ifile, new_path) +def _case_sensitive_fs(path): + """True when filesystem at `path` is case sensitive, False otherwise. + + Checks this by attempting to write two files, one w/ upper case, one + with lower. If after this only one file exists, the system is case-insensitive. + """ + root = op.join(path, uuid4().hex) + fnames = [root + suffix for suffix in 'aA'] + try: + for fname in fnames: + with open(fname, 'wt') as fobj: + fobj.write('text') + written = glob(root + '*') + finally: + for fname in written: + os.unlink(fname) + return len(written) == 2 + if __name__ == '__main__': ############################################################################### @@ -118,6 +139,7 @@ def _copy_non_content_files(): n_skipped_files = 0 n_built_files = 0 + cased_fs = _case_sensitive_fs(BUILD_FOLDER) print("Convert and copy notebook/md files...") for ix_file, page in enumerate(tqdm(list(toc))): url_page = page.get('url', None) @@ -233,8 +255,17 @@ def _copy_non_content_files(): # Front-matter YAML yaml_fm = [] yaml_fm += ['---'] + # In case pre-existing links are sanitized + sanitized = url_page.lower().replace('_', '-') + if not args.local_build and cased_fs and url_page.lower() == sanitized: + raise RuntimeError('Redirect {} clashes with page {} ' + 'for local build on case-insensitive FS\n' + .format(sanitized, url_page) + + 'Consider renaming source page to lower case ' + 'or building on a case sensitive FS, for example ' + 'a case-sensitive disk image on Mac') yaml_fm += ['redirect_from:'] - yaml_fm += [' - "{}"'.format(_prepare_url(url_page).replace('_', '-').lower())] + yaml_fm += [' - "{}"'.format(sanitized)] if ix_file == 0: yaml_fm += [' - "/"'] if path_url_page.endswith('.ipynb'): From 95e3225d95e164cfe9e1ea41135c4625ec64d170 Mon Sep 17 00:00:00 2001 From: Chris Holdgraf Date: Wed, 28 Nov 2018 18:03:37 -0800 Subject: [PATCH 2/4] updating tests so they're a bit more robust and making filesystem case checking pass --- scripts/generate_book.py | 2 ++ scripts/tests/test_build.py | 3 ++- scripts/tests/test_license.py | 3 +++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts/generate_book.py b/scripts/generate_book.py index 8de74429c..36401f38a 100644 --- a/scripts/generate_book.py +++ b/scripts/generate_book.py @@ -91,6 +91,8 @@ def _case_sensitive_fs(path): with open(fname, 'wt') as fobj: fobj.write('text') written = glob(root + '*') + except Exception: + written = [] finally: for fname in written: os.unlink(fname) diff --git a/scripts/tests/test_build.py b/scripts/tests/test_build.py index b86d504a0..706b87039 100644 --- a/scripts/tests/test_build.py +++ b/scripts/tests/test_build.py @@ -31,9 +31,10 @@ def is_not_in(lines, check): if op.isdir(op.join(curdir, 'site', '_build')): sh.rmtree(op.join(curdir, 'site', '_build')) +print("Building site for test suite...") cmd = ["python", op.join(curdir, "..", "generate_book.py"), "--site-root", op.join(curdir, "site"), "--path-template", op.join(curdir, "..", "templates", "jekyllmd.tpl")] -out = subprocess.call(cmd) +out = subprocess.check_call(cmd) #################################################### # Check outputs diff --git a/scripts/tests/test_license.py b/scripts/tests/test_license.py index cdf3bbbc0..113f0bc9f 100644 --- a/scripts/tests/test_license.py +++ b/scripts/tests/test_license.py @@ -10,6 +10,9 @@ def test_license(): # Not yes/no answers should error + if op.exists(op.join(curdir, "site", "content", "LICENSE.md")): + os.remove(op.join(curdir, "site", "content", "LICENSE.md")) + cmd_error = ["python", op.join(curdir, "..", "license.py"), "--path", op.join(curdir, "site", "content"), "--use-license", "blah"] From cc36c2634ad733e85cdb8c449767e1b7f0caa820 Mon Sep 17 00:00:00 2001 From: Matthew Brett Date: Fri, 30 Nov 2018 20:46:53 +0000 Subject: [PATCH 3/4] Create directory for case sensitive check Allow errors to propagate if files cannot be made. --- scripts/generate_book.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/generate_book.py b/scripts/generate_book.py index 36401f38a..17f7c6520 100644 --- a/scripts/generate_book.py +++ b/scripts/generate_book.py @@ -83,7 +83,11 @@ def _case_sensitive_fs(path): Checks this by attempting to write two files, one w/ upper case, one with lower. If after this only one file exists, the system is case-insensitive. + + Makes directory `path` if it does not exist. """ + if not op.exists(path): + os.makedirs(path) root = op.join(path, uuid4().hex) fnames = [root + suffix for suffix in 'aA'] try: @@ -91,8 +95,6 @@ def _case_sensitive_fs(path): with open(fname, 'wt') as fobj: fobj.write('text') written = glob(root + '*') - except Exception: - written = [] finally: for fname in written: os.unlink(fname) From 80fbf3c7cc861334652440d7efb411bfc4c6af46 Mon Sep 17 00:00:00 2001 From: Matthew Brett Date: Sat, 1 Dec 2018 12:38:50 +0000 Subject: [PATCH 4/4] Fix up sanitized checks (maybe) --- scripts/generate_book.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/scripts/generate_book.py b/scripts/generate_book.py index 17f7c6520..50e232359 100644 --- a/scripts/generate_book.py +++ b/scripts/generate_book.py @@ -143,7 +143,7 @@ def _case_sensitive_fs(path): n_skipped_files = 0 n_built_files = 0 - cased_fs = _case_sensitive_fs(BUILD_FOLDER) + case_check = _case_sensitive_fs(BUILD_FOLDER) and args.local_build print("Convert and copy notebook/md files...") for ix_file, page in enumerate(tqdm(list(toc))): url_page = page.get('url', None) @@ -261,17 +261,17 @@ def _case_sensitive_fs(path): yaml_fm += ['---'] # In case pre-existing links are sanitized sanitized = url_page.lower().replace('_', '-') - if not args.local_build and cased_fs and url_page.lower() == sanitized: - raise RuntimeError('Redirect {} clashes with page {} ' - 'for local build on case-insensitive FS\n' - .format(sanitized, url_page) + - 'Consider renaming source page to lower case ' - 'or building on a case sensitive FS, for example ' - 'a case-sensitive disk image on Mac') - yaml_fm += ['redirect_from:'] - yaml_fm += [' - "{}"'.format(sanitized)] - if ix_file == 0: - yaml_fm += [' - "/"'] + if sanitized != url_page: + if case_check and url_page.lower() == sanitized: + raise RuntimeError( + 'Redirect {} clashes with page {} for local build on ' + 'case-insensitive FS\n'.format(sanitized, url_page) + + 'Rename source page to lower case or build on a case ' + 'sensitive FS, e.g. case-sensitive disk image on Mac') + yaml_fm += ['redirect_from:'] + yaml_fm += [' - "{}"'.format(sanitized)] + if ix_file == 0: + yaml_fm += [' - "/"'] if path_url_page.endswith('.ipynb'): interact_path = 'content/' + path_url_page.split('content/')[-1] yaml_fm += ['interact_link: {}'.format(interact_path)]