From 3ff1332a43e1280cec49b7abfebe371053b85908 Mon Sep 17 00:00:00 2001 From: Andrey Kurdyumov Date: Wed, 4 Feb 2015 16:59:22 +0600 Subject: [PATCH] Add script which fix license issues across all languages in the our docs. --- bin/fixyaml | 65 +++++++++++++++++++ bin/fixyaml.bat | 26 ++++++++ lib/docs_validator.js | 142 ++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 229 insertions(+), 4 deletions(-) create mode 100644 bin/fixyaml create mode 100644 bin/fixyaml.bat diff --git a/bin/fixyaml b/bin/fixyaml new file mode 100644 index 00000000000..c1a88767457 --- /dev/null +++ b/bin/fixyaml @@ -0,0 +1,65 @@ +#!/usr/bin/env node + +/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/ +/*jslint node:true, nomen: true */ + +var yargs = require('yargs') + .count("verbose") + .alias('v', 'verbose') + .describe('verbose', 'Increase verbosity level of produced output') + .describe('source', 'Path to the documentation sources. Default: public') + .usage('Usage: $0 [-v] [lang] [version] [--edge]\n' + + + 'Fix the output which is produced by CrowdIn for MD files\n' + + '\n' + + ' : Language for which fix YAML header. If not specified then fixed for all languages.\n' + + ' : Version for which fix YAML header. If not specified then fixed all versions.\n') + .usage('Usage: $0\n' + + 'Fix the output which is produced by CrowdIn for MD files'); +var argv = yargs.argv; + +if (argv.help) { + yargs.showHelp(); + process.exit(1); +} + +var language = null; +var version = null; +if (argv.edge) { + language = null; + version = "edge"; +} else { + var argumentsCount = argv._.length; + if (argumentsCount !== 0) { + if (argumentsCount === 2) { + language = argv._[0]; + version = argv._[1]; + } else { + var data = argv._[0]; + if (data.length === 2) { + language = data; + } else { + version = data; + } + } + } +} + +var validator = require('../lib/docs_validator'); +new validator().fixYamlHeader(argv.source, language, version, argv.verbose); \ No newline at end of file diff --git a/bin/fixyaml.bat b/bin/fixyaml.bat new file mode 100644 index 00000000000..2c682542ba4 --- /dev/null +++ b/bin/fixyaml.bat @@ -0,0 +1,26 @@ +:: Licensed to the Apache Software Foundation (ASF) under one +:: or more contributor license agreements. See the NOTICE file +:: distributed with this work for additional information +:: regarding copyright ownership. The ASF licenses this file +:: to you under the Apache License, Version 2.0 (the +:: "License"); you may not use this file except in compliance +:: with the License. You may obtain a copy of the License at +:: +:: http://www.apache.org/licenses/LICENSE-2.0 +:: +:: Unless required by applicable law or agreed to in writing, +:: software distributed under the License is distributed on an +:: "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +:: KIND, either express or implied. See the License for the +:: specific language governing permissions and limitations +:: under the License. + +@ECHO OFF +SET script_path="%~dp0fixyaml" +IF EXIST %script_path% ( + node "%script_path%" %* +) ELSE ( + ECHO. + ECHO ERROR: Could not find 'fixyaml' script in 'bin' folder, aborting...>&2 + EXIT /B 1 +) \ No newline at end of file diff --git a/lib/docs_validator.js b/lib/docs_validator.js index c680b48db39..0df395c52e4 100644 --- a/lib/docs_validator.js +++ b/lib/docs_validator.js @@ -150,7 +150,6 @@ var DocsValidator = (function () { }); }); }; - /** * Validates the specific version of documentation @@ -321,9 +320,7 @@ var DocsValidator = (function () { // green for additions, red for deletions // grey for common parts var color = part.added ? 'green' : (part.removed ? 'red' : 'grey'); - //if (part.added || part.removed) { - process.stderr.write(part.value[color]); - //} + process.stderr.write(part.value[color]); }); console.log(); @@ -341,11 +338,148 @@ var DocsValidator = (function () { if (!match) { console.log("File " + sourceFile + " miss the YAML license header"); + return 1; } else { if (match[1].indexOf("license:") === -1) { console.log("File " + sourceFile + " has invalid YAML license header"); + return 2; + } + } + + return 0; + }; + + /** + * Validates the specific version of documentation + * @param outputDirectory Directory where documentation is stored. + * @param language Language which has to be validated. + * @param version Version which files has to be validated. + * @param verbose_mode Verbosity level. + */ + DocsValidator.prototype.fixYamlHeader = function (docsDirectory, language, version, verbose_mode) { + var self = this, + outputDirectory = path.resolve(docsDirectory || FileHelpers.getDefaultInputDirectory()), + ignore_list = ['.', '..', '.DS_Store', 'test']; + + verbose_mode = verbose_mode || 0; + if (verbose_mode > 0) { + console.log("Fixing YAML headers for lang " + language + " and version " + version); + console.log("Clearing output directory"); + } + + fs.readdirSync(outputDirectory).forEach(function (language_dir) { + if (ignore_list.indexOf(language_dir) !== -1) { + return; + } + + if (language && language_dir !== language) { + return; + } + + var language_path = path.join(outputDirectory, language_dir); + + fs.readdirSync(language_path).forEach(function (version_dir) { + if (ignore_list.indexOf(version_dir) !== -1) { + return; + } + + if (version && version_dir !== version) { + return; + } + + var input_path = path.join(outputDirectory, language_dir, version_dir), + options = { + lang: language_dir, + version: version_dir, + verbose: verbose_mode + }; + + console.log(" => Fix YAML header for version " + version_dir + " on language " + language_dir + "..."); + self.doFixYamlHeader(input_path, options); + }); + }); + }; + DocsValidator.prototype.doFixYamlHeader = function (lang_directory, options) { + var self = this, + compareFiles, + completed; + console.log("Fixing " + lang_directory); + completed = false; + dir.readFiles(lang_directory, + { match: /\.md/ }, + function (err, content, filename, next) { + if (err) { + throw err; + } + + var relativePath = path.relative(lang_directory, filename); + fs.readFile(filename, 'utf8', function (err, data) { + if (err) { + throw err; + } + + var target = data, + validationResult = self.validateYaml(filename, content, options), + yamlReplaceRegex1, + yamlReplaceRegex2, + eol = require('os').type() === 'win32' ? "\r\n" : "\n", + prefix = " ", + correctLicense = '---' + eol + + "license: Licensed to the Apache Software Foundation (ASF) under one" + eol + + prefix + "or more contributor license agreements. See the NOTICE file" + eol + + prefix + "distributed with this work for additional information" + eol + + prefix + "regarding copyright ownership. The ASF licenses this file" + eol + + prefix + "to you under the Apache License, Version 2.0 (the" + eol + + prefix + "\"License\"); you may not use this file except in compliance" + eol + + prefix + "with the License. You may obtain a copy of the License at" + eol + + eol + + prefix + " http://www.apache.org/licenses/LICENSE-2.0" + eol + + eol + + prefix + "Unless required by applicable law or agreed to in writing," + eol + + prefix + "software distributed under the License is distributed on an" + eol + + prefix + "\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY" + eol + + prefix + "KIND, either express or implied. See the License for the" + eol + + prefix + "specific language governing permissions and limitations" + eol + + prefix + "under the License." + eol + + '---' + eol + eol; + + if (validationResult !== 0) { + yamlReplaceRegex1 = /^(\* \* \*\s*\n[\s\S]*?\n?)^(\#\# (under the License\.|unter der Lizenz\.|aux termes de la licence\.|con la licenza\.|ライセンス。|라이센스\.|根據許可證。)\s*$\n?)/m; + if (yamlReplaceRegex1.exec(content)) { + content = correctLicense + content.replace(yamlReplaceRegex1, ''); + } else { + yamlReplaceRegex2 = /^(\* \* \*\s*\n[\s\S]*?\n?)^(\* \* \*\s*\s*$\n?)/m; + if (yamlReplaceRegex2.exec(content)) { + content = correctLicense + content.replace(yamlReplaceRegex2, ''); + } + } + + fs.writeFile(filename, content, 'utf8', function (err, data) { + if (err) { + throw err; + } + + next(); + }); + } else { + next(); + } + }); + }, + function (err, files) { + if (err) { + throw err; + } + + completed = true; + }); + function waitCompletition() { + if (!completed) { + setImmediate(waitCompletition); } } + + setImmediate(waitCompletition); }; return DocsValidator;