Skip to content

Commit

Permalink
Merge pull request #203 from wilzbach/add-check-for-public-unittests
Browse files Browse the repository at this point in the history
Add Phobos public tests extractor script
  • Loading branch information
andralex committed Dec 15, 2016
2 parents 3a32331 + d9762d7 commit 184f5e6
Show file tree
Hide file tree
Showing 2 changed files with 207 additions and 14 deletions.
29 changes: 15 additions & 14 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,20 +8,21 @@ D tools
This repository hosts various tools redistributed with DMD or used
internally during various build tasks.

Program | Scope | Description
--------------- | -------- | -----------------------------------------
catdoc | Build | Concatenates Ddoc files.
changed | Internal | Change log generator.
chmodzip | Build | ZIP file attributes editor.
ddemangle | Public | D symbol demangler.
detab | Internal | Replaces tabs with spaces.
dget | Internal | D source code downloader.
dman | Public | D documentation lookup tool.
dustmite | Public | [Test case minimization tool](https://github.com/CyberShadow/DustMite/wiki).
get_dlibcurl32 | Internal | Win32 libcurl downloader/converter.
rdmd | Public | [D build tool](http://dlang.org/rdmd.html).
rdmd_test | Internal | rdmd test suite.
tolf | Internal | Line endings converter.
Program | Scope | Description
---------------------- | -------- | -----------------------------------------
catdoc | Build | Concatenates Ddoc files.
changed | Internal | Change log generator.
chmodzip | Build | ZIP file attributes editor.
ddemangle | Public | D symbol demangler.
detab | Internal | Replaces tabs with spaces.
dget | Internal | D source code downloader.
dman | Public | D documentation lookup tool.
dustmite | Public | [Test case minimization tool](https://github.com/CyberShadow/DustMite/wiki).
get_dlibcurl32 | Internal | Win32 libcurl downloader/converter.
phobos_tests_extractor | Internal | Extracts public unittests from Phobos (requires DUB)
rdmd | Public | [D build tool](http://dlang.org/rdmd.html).
rdmd_test | Internal | rdmd test suite.
tolf | Internal | Line endings converter.

To report a problem or browse the list of open bugs, please visit the
[bug tracker](http://issues.dlang.org/).
Expand Down
192 changes: 192 additions & 0 deletions phobos_tests_extractor.d
Original file line number Diff line number Diff line change
@@ -0,0 +1,192 @@
#!/usr/bin/env dub
/+ dub.sdl:
name "check_phobos"
dependency "libdparse" version="~>0.7.0-beta.2"
+/
/*
* Parses all public unittests that are visible on dlang.org
* (= annotated with three slashes)
*
* Copyright (C) 2016 by D Language Foundation
*
* Distributed under the Boost Software License, Version 1.0.
* (See accompanying file LICENSE_1_0.txt or copy at
* http://www.boost.org/LICENSE_1_0.txt)
*/
// Written in the D programming language.

import dparse.ast;
import std.algorithm;
import std.conv;
import std.exception;
import std.experimental.logger;
import std.file;
import std.path;
import std.range;
import std.regex;
import std.stdio;
import std.string;

class TestVisitor : ASTVisitor
{
File outFile;
ubyte[] sourceCode;
string moduleName;

this(string outFileName, string moduleName, ubyte[] sourceCode)
{
this.outFile = File(outFileName, "w");
this.moduleName = moduleName;
this.sourceCode = sourceCode;
}

alias visit = ASTVisitor.visit;

override void visit(const Unittest u)
{
// scan the previous line for ddoc header
auto prevLine = sourceCode[0 .. u.location].retro;
prevLine.findSkip("\n"); // skip forward to the previous line
auto ddocCommentSlashes = prevLine.until('\n').count('/');

// only look for comments annotated with three slashes (///)
if (ddocCommentSlashes != 3)
return;

// write the origin source code line
outFile.writefln("// Line %d", u.line);

// write the unittest block
outFile.write("unittest\n{\n");
scope(exit) outFile.writeln("}\n");

// add an import to the current module
outFile.writefln(" import %s;", moduleName);

// write the content of the unittest block (but skip the first brace)
auto k = cast(immutable(char)[]) sourceCode[u.blockStatement.startLocation .. u.blockStatement.endLocation];
k.findSkip("{");
outFile.write(k);

// if the last line contains characters, we want to add an extra line for increased visual beauty
if (k[$ - 1] != '\n')
outFile.writeln;
}
}

void parseTests(string fileName, string moduleName, string outFileName)
{
import dparse.lexer;
import dparse.parser;
import dparse.rollback_allocator;
import std.array : uninitializedArray;

assert(exists(fileName));

File f = File(fileName);

if (f.size == 0)
{
warningf("%s is empty", fileName);
return;
}

ubyte[] sourceCode = uninitializedArray!(ubyte[])(to!size_t(f.size));
f.rawRead(sourceCode);
LexerConfig config;
StringCache cache = StringCache(StringCache.defaultBucketCount);
auto tokens = getTokensForParser(sourceCode, config, &cache);
RollbackAllocator rba;
Module m = parseModule(tokens.array, fileName, &rba);
auto visitor = new TestVisitor(outFileName, moduleName, sourceCode);
visitor.visit(m);
}

void parseFile(string inputDir, string fileName, string outputDir, string modulePrefix = "")
{
import std.path : buildPath, dirSeparator, buildNormalizedPath;

// file name without its parent directory, e.g. std/uni.d
string fileNameNormalized = (inputDir == "." ? fileName : fileName.replace(inputDir, ""));

// remove leading dots or slashes
while (!fileNameNormalized.empty && fileNameNormalized[0] == '.')
fileNameNormalized = fileNameNormalized[1 .. $];
if (fileNameNormalized.length >= dirSeparator.length &&
fileNameNormalized[0 .. dirSeparator.length] == dirSeparator)
fileNameNormalized = fileNameNormalized[dirSeparator.length .. $];

// convert the file path to its module path, e.g. std/uni.d -> std.uni
string moduleName = modulePrefix ~ fileNameNormalized.replace(".d", "")
.replace(dirSeparator, ".")
.replace(".package", "");

// convert the file path to a nice output file, e.g. std/uni.d -> std_uni.d
string outName = fileNameNormalized.replace(dirSeparator, "_");

parseTests(fileName, moduleName, buildPath(outputDir, outName));
}

void main(string[] args)
{
import std.getopt;

string inputDir;
string outputDir = "./out";
string ignoredFilesStr;
string modulePrefix = "";

auto helpInfo = getopt(args, config.required,
"inputdir|i", "Folder to start the recursive search for unittest blocks (can be a single file)", &inputDir,
"outputdir|o", "Folder to which the extracted test files should be saved", &outputDir,
"moduleprefix", "Module prefix to use for all files (e.g. std.algorithm)", &modulePrefix,
"ignore", "Comma-separated list of files to exclude (partial matching is supported)", &ignoredFilesStr);

if (helpInfo.helpWanted)
{
return defaultGetoptPrinter(`phobos_tests_extractor
Searches the input directory recursively for public unittest blocks, i.e.
unittest blocks that are annotated with three slashes (///).
The tests will be extracted as one file for each source file
to in the output directory.
`, helpInfo.options);
}

inputDir = inputDir.asNormalizedPath.array;
outputDir = outputDir.asNormalizedPath.array;

if (!exists(outputDir))
mkdir(outputDir);

// if the module prefix is std -> add a dot for the next modules to follow
if (!modulePrefix.empty)
modulePrefix ~= '.';

DirEntry[] files;

if (inputDir.isFile)
{
files = [DirEntry(inputDir)];
inputDir = ".";
}
else
{
files = dirEntries(inputDir, SpanMode.depth).filter!(
a => a.name.endsWith(".d") && !a.name.canFind(".git")).array;
}

auto ignoringFiles = ignoredFilesStr.split(",");

foreach (file; files)
{
if (!ignoringFiles.any!(x => file.name.canFind(x)))
{
writeln("parsing ", file);
parseFile(inputDir, file, outputDir, modulePrefix);
}
else
{
writeln("ignoring ", file);
}
}
}

0 comments on commit 184f5e6

Please sign in to comment.