Major refactoring of filesystemadaptor

The code here had got a bit broken by some PRs that I should have checked more carefully. I’ve done a major refactoring which will hopefully make it easier to understand, and fixes a number of problems: * Problem with eg .md tiddlers not being deleted correctly * Problem with Windows path separators not being usable within $:/config/FileSystemPaths on Windows * Problem with filename clashes not being detected correctly when saving to a different directory via $:/config/FileSystemPaths * Enables slashes within tiddler titles to be mapped into folders * Enables plain text files like .md and .css to be saved with .meta files instead of as .tid files (see #2558) * No longer replaces spaces with underscores As this is such a major update, I’d be grateful if Node.js users could give it a careful run through — in particular, you’ll need to try creating new tiddlers of various types and ensure that the expected files are created.
Jermolene · Feb 11, 2017 · 3708f6c · 3708f6c · pmario · Feb 11, 2017
1 parent 1961db6
commit 3708f6c
Show file tree

Hide file tree

Showing 3 changed files with 149 additions and 82 deletions.
diff --git a/core/modules/utils/utils.js b/core/modules/utils/utils.js
@@ -697,7 +697,6 @@ exports.tagToCssSelector = function(tagName) {
 	});
 };
 
-
 /*
 IE does not have sign function
 */
@@ -725,4 +724,82 @@ exports.strEndsWith = function(str,ending,position) {
 	}
 };
 
+/*
+Transliterate string from eg. Cyrillic Russian to Latin
+*/
+var transliterationPairs = {
+	"Ё":"YO",
+	"Й":"I",
+	"Ц":"TS",
+	"У":"U",
+	"К":"K",
+	"Е":"E",
+	"Н":"N",
+	"Г":"G",
+	"Ш":"SH",
+	"Щ":"SCH",
+	"З":"Z",
+	"Х":"H",
+	"Ъ":"'",
+	"ё":"yo",
+	"й":"i",
+	"ц":"ts",
+	"у":"u",
+	"к":"k",
+	"е":"e",
+	"н":"n",
+	"г":"g",
+	"ш":"sh",
+	"щ":"sch",
+	"з":"z",
+	"х":"h",
+	"ъ":"'",
+	"Ф":"F",
+	"Ы":"I",
+	"В":"V",
+	"А":"a",
+	"П":"P",
+	"Р":"R",
+	"О":"O",
+	"Л":"L",
+	"Д":"D",
+	"Ж":"ZH",
+	"Э":"E",
+	"ф":"f",
+	"ы":"i",
+	"в":"v",
+	"а":"a",
+	"п":"p",
+	"р":"r",
+	"о":"o",
+	"л":"l",
+	"д":"d",
+	"ж":"zh",
+	"э":"e",
+	"Я":"Ya",
+	"Ч":"CH",
+	"С":"S",
+	"М":"M",
+	"И":"I",
+	"Т":"T",
+	"Ь":"'",
+	"Б":"B",
+	"Ю":"YU",
+	"я":"ya",
+	"ч":"ch",
+	"с":"s",
+	"м":"m",
+	"и":"i",
+	"т":"t",
+	"ь":"'",
+	"б":"b",
+	"ю":"yu"
+};
+
+exports.transliterate = function(str) {
+	return str.split("").map(function(char) {
+		return transliterationPairs[char] || char;
+	}).join("");
+};
+
 })();
diff --git a/editions/tw5.com/tiddlers/nodejs/Customising Tiddler File Naming.tid b/editions/tw5.com/tiddlers/nodejs/Customising Tiddler File Naming.tid
@@ -6,9 +6,9 @@ type: text/vnd.tiddlywiki
 
 By default, a [[TiddlyWiki on Node.js]] instance using a [[wiki folder|TiddlyWikiFolders]] will create new tiddler files by using the sanitised and disambiguated title as filename.
 
-This can be customised by creating a tiddler [[$:/config/FileSystemPaths]] containing one or more [[filter expressions|Filter Syntax]], each on a line of its own. Newly created tiddlers are matched to each filter in turn, and the first output of the first filter to produce any output is taken as a logical path to be used for the tiddler file. Logical paths don't include the `.tid` extension, and they always use `/` as directory separator (when generating the physical path, this is replaced by the correct separator for the platform ~TiddlyWiki is running on). If none of the filters matches, the logical path is simply the title with all occurences of `/` replaced by `_` (for backwards compatibility).
+This can be customised by creating a tiddler [[$:/config/FileSystemPaths]] containing one or more [[filter expressions|Filter Syntax]], each on a line of its own. Newly created tiddlers are matched to each filter in turn, and the first output of the first filter to produce any output is taken as a logical path to be used for the tiddler file. Logical paths don't include the `.tid` extension, and they can use `/` or `\` as directory separator (when generating the physical path, this is replaced by the correct separator for the platform ~TiddlyWiki is running on). If none of the filters matches, the logical path is simply the title with all occurences of `/` replaced by `_` (for backwards compatibility).
 
-In both cases, the characters `<>:"\|?*^ ` are replaced by `_` in order to guarantee that the resulting path is legal on all supported platforms.
+In both cases, the characters `<>:"\|?*^` are replaced by `_` in order to guarantee that the resulting path is legal on all supported platforms.
 
 !! Example
 
@@ -21,3 +21,5 @@ In both cases, the characters `<>:"\|?*^ ` are replaced by `_` in order to guara
 This will store newly created system tiddlers in `tiddlers/_system` (after stripping the `$:/` prefix), tiddlers tagged [[task]] in a subdirectory `tiddlers/mytasks`, and also create subdirectory structures for all other non-draft tiddlers.
 
 Thus, $:/config/FileSystemPaths itself will end up in `tiddlers/_system/config/FileSystemPaths.tid` or `tiddlers\_system\config\FileSystemPaths.tid`, depending on the platform.
+
+The final `[!has[draft.of]]` will match all remaining non-draft tiddlers. Because there was a match, any `/` or `\` in the tiddler title is mapped to a path separator. Thus, `some/thing/entirely/new` will be saved to `tiddlers/some/thing/entirely/new.tid` (ie, the file `new.tid` in a directory called `entirely`).
diff --git a/plugins/tiddlywiki/filesystem/filesystemadaptor.js b/plugins/tiddlywiki/filesystem/filesystemadaptor.js
@@ -35,113 +35,107 @@ FileSystemAdaptor.prototype.getTiddlerInfo = function(tiddler) {
 	return {};
 };
 
-$tw.config.typeInfo = {
-	"text/vnd.tiddlywiki": {
-		fileType: "application/x-tiddler",
-		extension: ".tid"
-	}
-};
+/*
+Return a fileInfo object for a tiddler, creating it if necessary:
+  filepath: the absolute path to the file containing the tiddler
+  type: the type of the tiddler file (NOT the type of the tiddler -- see below)
+  hasMetaFile: true if the file also has a companion .meta file
 
+The boot process populates $tw.boot.files for each of the tiddler files that it loads. The type is found by looking up the extension in $tw.config.fileExtensionInfo (eg "application/x-tiddler" for ".tid" files).
+
+It is the responsibility of the filesystem adaptor to update $tw.boot.files for new files that are created.
+*/
 FileSystemAdaptor.prototype.getTiddlerFileInfo = function(tiddler,callback) {
 	// See if we've already got information about this file
 	var self = this,
 		title = tiddler.fields.title,
 		fileInfo = $tw.boot.files[title];
-	// Get information about how to save tiddlers of this type
-	var type = tiddler.fields.type || "text/vnd.tiddlywiki";
-	var typeInfo = $tw.config.typeInfo[type] ||
-		$tw.config.contentTypeInfo[type] ||
-		$tw.config.typeInfo["text/vnd.tiddlywiki"];
-	var extension = typeInfo.extension || "";
-	if(!fileInfo) {
-		// If not, we'll need to generate it
+	if(fileInfo) {
+		// If so, just invoke the callback
+		callback(null,fileInfo);
+	} else {
+		// Otherwise, we'll need to generate it
+		fileInfo = {};
+		var tiddlerType = tiddler.fields.type || "text/vnd.tiddlywiki";
+		// Get the content type info
+		var contentTypeInfo = $tw.config.contentTypeInfo[tiddlerType] || {};
+		// Get the file type by looking up the extension
+		var extension = contentTypeInfo.extension || ".tid";
+		fileInfo.type = $tw.config.fileExtensionInfo[extension].type;
+		// Use a .meta file unless we're saving a .tid file.
+		// (We would need more complex logic if we supported other template rendered tiddlers besides .tid)
+		fileInfo.hasMetaFile = (fileInfo.type !== "application/x-tiddler");
+		// Generate the base filepath and ensure the directories exist
+		var baseFilepath = path.resolve($tw.boot.wikiTiddlersPath,this.generateTiddlerBaseFilepath(title));
+		$tw.utils.createDirectory(baseFilepath);
 		// Start by getting a list of the existing files in the directory
-		fs.readdir($tw.boot.wikiTiddlersPath,function(err,files) {
+		fs.readdir(path.dirname(baseFilepath),function(err,files) {
 			if(err) {
 				return callback(err);
 			}
-			// Assemble the new fileInfo
-			fileInfo = {};
-			fileInfo.filepath = $tw.boot.wikiTiddlersPath + path.sep + self.generateTiddlerFilename(title,extension,files);
-			fileInfo.type = typeInfo.fileType || tiddler.fields.type;
-			fileInfo.hasMetaFile = typeInfo.hasMetaFile;
-			// Save the newly created fileInfo
+			// Start with the base filename plus the extension
+			var filepath = baseFilepath;
+			if(filepath.substr(-extension.length).toLocaleLowerCase() !== extension.toLocaleLowerCase()) {
+				filepath = filepath + extension;
+			}
+			var filename = path.basename(filepath),
+				count = 1;
+			// Add a discriminator if we're clashing with an existing filename while
+			// handling case-insensitive filesystems (NTFS, FAT/FAT32, etc.)
+			while(files.some(function(value) {return value.toLocaleLowerCase() === filename.toLocaleLowerCase();})) {
+				filepath = baseFilepath + " " + (count++) + extension;
+				filename = path.basename(filepath);
+			}
+			// Set the final fileInfo
+			fileInfo.filepath = filepath;
+console.log("\x1b[1;35m" + "For " + title + ", type is " + fileInfo.type + " hasMetaFile is " + fileInfo.hasMetaFile + " filepath is " + fileInfo.filepath + "\x1b[0m");
 			$tw.boot.files[title] = fileInfo;
 			// Pass it to the callback
 			callback(null,fileInfo);
 		});
-	} else {
-		// Otherwise just invoke the callback
-		callback(null,fileInfo);
 	}
 };
 
-/*
-Transliterate string from cyrillic russian to latin
-*/
- var transliterate = function(cyrillyc) {
-	var a = {"Ё":"YO","Й":"I","Ц":"TS","У":"U","К":"K","Е":"E","Н":"N","Г":"G","Ш":"SH","Щ":"SCH","З":"Z","Х":"H","Ъ":"'","ё":"yo","й":"i","ц":"ts","у":"u","к":"k","е":"e","н":"n","г":"g","ш":"sh","щ":"sch","з":"z","х":"h","ъ":"'","Ф":"F","Ы":"I","В":"V","А":"a","П":"P","Р":"R","О":"O","Л":"L","Д":"D","Ж":"ZH","Э":"E","ф":"f","ы":"i","в":"v","а":"a","п":"p","р":"r","о":"o","л":"l","д":"d","ж":"zh","э":"e","Я":"Ya","Ч":"CH","С":"S","М":"M","И":"I","Т":"T","Ь":"'","Б":"B","Ю":"YU","я":"ya","ч":"ch","с":"s","м":"m","и":"i","т":"t","ь":"'","б":"b","ю":"yu"};
-	return cyrillyc.split("").map(function (char) {
-		return a[char] || char;
-	}).join("");
-};
-
 /*
 Given a list of filters, apply every one in turn to source, and return the first result of the first filter with non-empty result.
 */
 FileSystemAdaptor.prototype.findFirstFilter = function(filters,source) {
-	var numFilters = filters.length;
-	for(var i=0; i<numFilters; i++) {
+	for(var i=0; i<filters.length; i++) {
 		var result = this.wiki.filterTiddlers(filters[i],null,source);
 		if(result.length > 0) {
 			return result[0];
 		}
 	}
+	return null;
 };
 
-/*
-Add file extension to a file path if it doesn't already exist.
-*/
-FileSystemAdaptor.addFileExtension = function(file,extension) {
-	return $tw.utils.strEndsWith(file,extension) ? file : file + extension;
-};
-
-
 /*
 Given a tiddler title and an array of existing filenames, generate a new legal filename for the title, case insensitively avoiding the array of existing filenames
 */
-FileSystemAdaptor.prototype.generateTiddlerFilename = function(title,extension,existingFilenames) {
+FileSystemAdaptor.prototype.generateTiddlerBaseFilepath = function(title) {
 	var baseFilename;
 	// Check whether the user has configured a tiddler -> pathname mapping
 	var pathNameFilters = this.wiki.getTiddlerText("$:/config/FileSystemPaths");
 	if(pathNameFilters) {
 		var source = this.wiki.makeTiddlerIterator([title]);
-		var result = this.findFirstFilter(pathNameFilters.split("\n"),source);
-		if(result) {
-			// interpret "/" as path separator
-			baseFilename = result.replace(/\//g,path.sep);
+		baseFilename = this.findFirstFilter(pathNameFilters.split("\n"),source);
+console.log("baseFilename",baseFilename)
+		if(baseFilename) {
+			// Interpret "/" and "\" as path separator
+			baseFilename = baseFilename.replace(/\/|\\/g,path.sep);
 		}
 	}
 	if(!baseFilename) {
-		// no mapping configured, or it did not match this tiddler
-		// in this case, we fall back to legacy behaviour
-		baseFilename = title.replace(/\//g,"_");
+		// No mappings provided, or failed to match this tiddler so we use title as filename
+		baseFilename = title.replace(/\/|\\/g,"_");
 	}
 	// Remove any of the characters that are illegal in Windows filenames
-	var baseFilename = transliterate(baseFilename.replace(/<|>|\:|\"|\\|\||\?|\*|\^|\s/g,"_"));
+	var baseFilename = $tw.utils.transliterate(baseFilename.replace(/<|>|\:|\"|\||\?|\*|\^/g,"_"));
 	// Truncate the filename if it is too long
 	if(baseFilename.length > 200) {
 		baseFilename = baseFilename.substr(0,200);
 	}
-	// Start with the base filename plus the extension
-	var filename = FileSystemAdaptor.addFileExtension(baseFilename,extension),
-		count = 1;
-	// Add a discriminator if we're clashing with an existing filename while
-	// handling case-insensitive filesystems (NTFS, FAT/FAT32, etc.)
-	while(existingFilenames.some(function(value) {return value.toLocaleLowerCase() === filename.toLocaleLowerCase();})) {
-		filename = baseFilename + " " + (count++) + extension;
-	}
-	return filename;
+	return baseFilename;
 };
 
 /*
@@ -150,45 +144,39 @@ Save a tiddler and invoke the callback with (err,adaptorInfo,revision)
 FileSystemAdaptor.prototype.saveTiddler = function(tiddler,callback) {
 	var self = this;
 	this.getTiddlerFileInfo(tiddler,function(err,fileInfo) {
-		var template, content, encoding, filepath,
-			_finish = function() {
-				callback(null, {}, 0);
-			};
 		if(err) {
 			return callback(err);
 		}
-		var error = $tw.utils.createDirectory(path.dirname(fileInfo.filepath));
+		var filepath = fileInfo.filepath,
+			error = $tw.utils.createDirectory(path.dirname(filepath));
 		if(error) {
 			return callback(error);
 		}
-		var typeInfo = $tw.config.contentTypeInfo[fileInfo.type];
-		if(fileInfo.hasMetaFile || typeInfo.encoding === "base64") {
+		if(fileInfo.hasMetaFile) {
 			// Save the tiddler as a separate body and meta file
-			filepath = fileInfo.filepath;
+			var typeInfo = $tw.config.contentTypeInfo[tiddler.fields.type || "text/plain"] || {encoding: "utf8"};
 			fs.writeFile(filepath,tiddler.fields.text,{encoding: typeInfo.encoding},function(err) {
 				if(err) {
 					return callback(err);
 				}
 				content = self.wiki.renderTiddler("text/plain","$:/core/templates/tiddler-metadata",{variables: {currentTiddler: tiddler.fields.title}});
-				filepath = FileSystemAdaptor.addFileExtension(fileInfo.filepath,".meta");
-				fs.writeFile(filepath,content,{encoding: "utf8"},function (err) {
+				fs.writeFile(fileInfo.filepath + ".meta",content,{encoding: "utf8"},function (err) {
 					if(err) {
 						return callback(err);
 					}
 					self.logger.log("Saved file",filepath);
-					_finish();
+					return callback(null);
 				});
 			});
 		} else {
 			// Save the tiddler as a self contained templated file
-			content = self.wiki.renderTiddler("text/plain","$:/core/templates/tid-tiddler",{variables: {currentTiddler: tiddler.fields.title}});
-			filepath = FileSystemAdaptor.addFileExtension(fileInfo.filepath,".tid");
+			var content = self.wiki.renderTiddler("text/plain","$:/core/templates/tid-tiddler",{variables: {currentTiddler: tiddler.fields.title}});
 			fs.writeFile(filepath,content,{encoding: "utf8"},function (err) {
 				if(err) {
 					return callback(err);
 				}
 				self.logger.log("Saved file",filepath);
-				_finish();
+				return callback(null);
 			});
 		}
 	});
@@ -219,14 +207,14 @@ FileSystemAdaptor.prototype.deleteTiddler = function(title,callback,options) {
 			self.logger.log("Deleted file",fileInfo.filepath);
 			// Delete the metafile if present
 			if(fileInfo.hasMetaFile) {
-				fs.unlink(FileSystemAdaptor.addFileExtension(fileInfo.filepath,".meta"),function(err) {
+				fs.unlink(fileInfo.filepath + ".meta",function(err) {
 					if(err) {
 						return callback(err);
 					}
-					$tw.utils.deleteEmptyDirs(path.dirname(fileInfo.filepath),callback);
+					return $tw.utils.deleteEmptyDirs(path.dirname(fileInfo.filepath),callback);
 				});
 			} else {
-				$tw.utils.deleteEmptyDirs(path.dirname(fileInfo.filepath),callback);
+				return $tw.utils.deleteEmptyDirs(path.dirname(fileInfo.filepath),callback);
 			}
 		});
 	} else {