Permalink
Browse files

Support "filenames" key, mk. 2 (#115)

* Support "filenames" key for Makefile-style languages

* Remove debugging statement

* Add CMake

* Cleaner docs and tests

* Add scons, handle languages with no extensions
  • Loading branch information...
kazimuth authored and Aaronepower committed Jun 5, 2017
1 parent e4d06c9 commit 7849edfa34bbf5f62e30e9a0219b27881063769d
Showing with 281 additions and 38 deletions.
  1. +37 −0 CONTRIBUTING.md
  2. +4 −0 README.md
  3. +17 −11 build.rs
  4. +96 −2 languages.json
  5. +30 −8 src/language/language_type.hbs.rs
  6. +12 −17 src/utils/fs.rs
  7. +16 −0 tests/data/Dockerfile
  8. +24 −0 tests/data/Makefile
  9. +10 −0 tests/data/Rakefile
  10. +10 −0 tests/data/SConstruct
  11. +25 −0 tests/data/cmake.cmake
View
@@ -89,6 +89,43 @@ syntax.
- Does it only contain single line comments? Or only multi-line comments?
- Is just C style comments? `/* */, //`
Some languages have a single, standard filename, like Makefile or Dockerfile.
These can be defined with the `filenames` property:
```json
"Makefile":{
"filenames":[
"makefile"
],
"extensions":[
"makefile",
"mak",
"mk"
]
}
```
Filenames should be all-lowercase, whether or not the filename typically has capital letters included.
Note that filenames will *override* extensions, so with the following
configuration:
```json
"Text":{
"extensions":[
"txt"
]
},
"CMake":{
"filenames": [
"cmakelists.txt"
]
}
A file named `CMakeLists.txt` will be detected as a CMake file, not a text file.
```
# Bug Reports
Please include the error message, and a minimum working example including the file, or file structure.
View
@@ -258,6 +258,7 @@ C Header
C#
C Shell
Clojure
CMake
CoffeeScript
Cogent
ColdFusion
@@ -269,6 +270,7 @@ CSS
D
Dart
Device Tree
Dockerfile
Elixir
Elm
Erlang
@@ -317,13 +319,15 @@ PureScript
Python
QCL
R
Rakefile
Razor
ReStructuredText
Ruby
Ruby HTML
Rust
Sass
Scala
Scons
Standard ML
SQL
Swift
View
@@ -62,26 +62,32 @@ fn generate_tests(out_dir: &OsStr) {
let path = path.unwrap();
let path = path.path();
let name = path.file_stem().unwrap().to_str().unwrap();
let name = path.file_stem().unwrap().to_str().unwrap().to_lowercase();
string.push_str(&format!(r#"
#[test]
fn {0}() {{
let mut languages = Languages::new();
languages.get_statistics(vec!["{1}"], Vec::new());
if languages.len() != 1 {{
panic!("wrong languages detected: expected just {0}, found {{:?}}",
languages.into_iter().collect::<Vec<_>>());
}}
let (name, language) = languages.into_iter().next().unwrap();
let mut contents = String::new();
File::open("{1}").unwrap().read_to_string(&mut contents).unwrap();
for (name, language) in languages {{
assert_eq!(get_digit!(LINES, contents), language.lines);
println!("{{}} LINES MATCH", name);
assert_eq!(get_digit!(CODE, contents), language.code);
println!("{{}} CODE MATCH", name);
assert_eq!(get_digit!(COMMENTS, contents), language.comments);
println!("{{}} COMMENTS MATCH", name);
assert_eq!(get_digit!(BLANKS, contents), language.blanks);
println!("{{}} BLANKS MATCH", name);
}}
assert_eq!(get_digit!(LINES, contents), language.lines);
println!("{{}} LINES MATCH", name);
assert_eq!(get_digit!(CODE, contents), language.code);
println!("{{}} CODE MATCH", name);
assert_eq!(get_digit!(COMMENTS, contents), language.comments);
println!("{{}} COMMENTS MATCH", name);
assert_eq!(get_digit!(BLANKS, contents), language.blanks);
println!("{{}} BLANKS MATCH", name);
}}
"#, name, path.display()));
}
View
@@ -177,6 +177,23 @@
"clj"
]
},
"CMake": {
"single":[
"#"
],
"quotes":[
[
"\\\"",
"\\\""
]
],
"extensions":[
"cmake"
],
"filenames":[
"cmakelists.txt"
]
},
"CoffeeScript":{
"single":[
"#"
@@ -351,6 +368,28 @@
"dtsi"
]
},
"Dockerfile":{
"single":[
"#"
],
"extensions":[
"dockerfile",
"dockerignore"
],
"filenames":[
"dockerfile"
],
"quotes":[
[
"\\\"",
"\\\""
],
[
"'",
"'"
]
]
},
"Elixir":{
"single":[
"#"
@@ -829,6 +868,9 @@
"makefile",
"mak",
"mk"
],
"filenames":[
"makefile"
]
},
"Markdown":{
@@ -1048,6 +1090,33 @@
"r"
]
},
"Rakefile":{
"single":[
"#"
],
"multi":[
[
"=begin",
"=end"
]
],
"quotes":[
[
"\\\"",
"\\\""
],
[
"'",
"'"
]
],
"filenames":[
"rakefile"
],
"extensions":[
"rake"
]
},
"Razor":{
"multi":[
[
@@ -1084,8 +1153,7 @@
]
],
"extensions":[
"rb",
"rake"
"rb"
]
},
"RubyHtml":{
@@ -1143,6 +1211,32 @@
"scala"
]
},
"Scons":{
"base":"hash",
"quotes":[
[
"\\\"",
"\\\""
],
[
"'",
"'"
],
[
"\\\"\\\"\\\"",
"\\\"\\\"\\\""
],
[
"'''",
"'''"
]
],
"filenames":[
"sconstruct",
"sconscript"
]
},
"Sml":{
"name":"Standard ML (SML)",
"base":"func",
@@ -53,22 +53,44 @@ impl LanguageType {
]
}
/// Get language from it's file extension.
/// Get language from a file path. May open and read the file.
///
/// ```no_run
/// # use tokei::*;
/// let rust = LanguageType::from_extension("./main.rs");
/// let rust = LanguageType::from_path("./main.rs");
///
/// assert_eq!(rust, Some(LanguageType::Rust));
/// ```
pub fn from_extension<P: AsRef<Path>>(entry: P) -> Option<Self> {
if let Some(extension) = fs::get_extension(entry) {
pub fn from_path<P: AsRef<Path>>(entry: P) -> Option<Self> {
let filename = fs::get_filename(&entry);
if let Some(filename) = filename {
match &*filename {
{{~#each languages}}
{{~#if this.filenames}}
{{~#each this.filenames}}
"{{~this}}" {{~#unless @last}} | {{~/unless}}
{{~/each}}
=> return Some({{~@key}}),
{{~/if}}
{{~/each}}
_ => ()
}
}
let extension = fs::get_extension(&entry)
.or_else(|| get_filetype_from_shebang(&entry).map(String::from));
if let Some(extension) = extension {
match &*extension {
{{~#each languages}}
{{~#each this.extensions}}
"{{~this}}" {{~#unless @last}} | {{~/unless}}
{{~/each}}
=> Some({{~@key}}),
{{~#if this.extensions}}
{{~#each this.extensions}}
"{{~this}}" {{~#unless @last}} | {{~/unless}}
{{~/each}}
=> Some({{~@key}}),
{{~/if}}
{{~/each}}
extension => {
warn!("Unknown extension: {}", extension);
View
@@ -12,7 +12,6 @@ use ignore::overrides::OverrideBuilder;
use ignore::WalkState::*;
use language::{Language, Languages, LanguageType};
use language::LanguageType::*;
pub use language::get_filetype_from_shebang;
pub fn get_all_files(paths: Vec<&str>,
@@ -53,12 +52,7 @@ pub fn get_all_files(paths: Vec<&str>,
let entry = entry.path();
if entry.to_string_lossy().contains("Makefile") {
tx.send((Makefile, entry.to_owned())).unwrap();
return Continue;
}
if let Some(language) = LanguageType::from_extension(entry) {
if let Some(language) = LanguageType::from_path(entry) {
if let Ok(metadata) = entry.metadata() {
if metadata.is_file() {
tx.send((language, entry.to_owned())).unwrap();
@@ -82,19 +76,20 @@ pub fn get_extension<P: AsRef<Path>>(path: P) -> Option<String> {
match path.extension() {
Some(extension_os) => {
Some(extension_os.to_string_lossy().to_lowercase())
}
None => {
match get_filetype_from_shebang(path) {
// Using String::from here because all file extensions from
// get_filetype_from_shebang are guaranteed to be lowercase.
Some(extension) => Some(String::from(extension)),
None => None,
}
}
},
None => None
}
}
pub fn get_filename<P: AsRef<Path>>(path: P) -> Option<String> {
let path = path.as_ref();
match path.file_name() {
Some(filename_os) => {
Some(filename_os.to_string_lossy().to_lowercase())
},
None => None
}
}
#[cfg(test)]
mod test {
View
@@ -0,0 +1,16 @@
# 16 lines 6 code 3 comments 7 blanks
FROM netbsd:7.0.2
MAINTAINER Somebody version: 2.2
RUN curl -sSf https://static.rust-lang.org/rustup.sh | sh -s -- -y
# this part is important
VOLUME ["/project"]
WORKDIR "/project"
RUN cargo install tokei # not counted
# now you do your part
Oops, something went wrong.

0 comments on commit 7849edf

Please sign in to comment.