Dlang solution #16
Dlang solution #16
Conversation
|
Can you test this solution please? import std.stdio;
import std.outbuffer;
import std.ascii;
import std.algorithm;
import std.range;
import std.array;
import std.container.array;
import std.conv;
alias Dictionary = string[][ubyte[]];
ubyte char_to_digit(char ch)
{
final switch (ch.toLower)
{
case 'e':
return 0;
case 'j', 'n', 'q':
return 1;
case 'r', 'w', 'x':
return 2;
case 'd', 's', 'y':
return 3;
case 'f', 't':
return 4;
case 'a', 'm':
return 5;
case 'c', 'i', 'v':
return 6;
case 'b', 'k', 'u':
return 7;
case 'l', 'o', 'p':
return 8;
case 'g', 'h', 'z':
return 9;
}
}
immutable(ubyte[]) word_to_number(string word)
{
auto res = word
.filter!(a => a.isAlpha)
.map!(a => char_to_digit(cast(char)a))
.map!(a => cast(ubyte)(a + '0'))
.array;
return cast(immutable(ubyte[])) res;
}
Dictionary load_dict(string words_file)
{
Dictionary dict;
foreach (word; File(words_file).byLineCopy)
{
auto key = word_to_number(word);
dict[key] ~= word;
}
return dict;
}
bool lastItemIsDigit(Array!(string) words)
{
if (words.empty)
return false;
const back = words.back();
return back.length == 1 && back[0].isDigit;
}
void find_translation(
string res_opt,
ref size_t counter,
char[] num,
immutable(ubyte[]) digits,
Array!(string) words,
Dictionary dict,
OutBuffer writer)
{
if (digits.length == 0)
{
handle_solution(res_opt, counter, num, words, writer);
return;
}
bool found_word = false;
foreach (i; 0 .. digits.length)
{
auto key = digits[0 .. i + 1];
auto rest_of_digits = digits[i + 1 .. $];
auto found_words = key in dict;
if (found_words !is null)
{
found_word = true;
foreach (word; *found_words)
{
words.insertBack(word);
find_translation(res_opt, counter, num, rest_of_digits, words, dict, writer);
words.removeBack();
}
}
}
if (found_word)
return;
auto last_is_digit = words.lastItemIsDigit;
if (!last_is_digit)
{
auto digit = [(digits[0] - '0').to!string];
words.insertBack(digit);
find_translation(res_opt, counter, num, digits[1 .. $], words, dict, writer);
words.removeBack();
}
}
void handle_solution(string res_opt, ref size_t counter, char[] num, Array!(string) words, OutBuffer writer)
{
if (res_opt == "count")
{
counter++;
return;
}
writer.write(num);
if (words.empty)
{
writeln(writer.toString());
writer.clear();
return;
}
foreach (word; words)
{
writer.writef(" %s", word);
}
writeln(writer.toString());
writer.clear();
}
void main(string[] args)
{
auto res_opt = args.length > 1 ? args[1] : "print";
auto words_file = args.length > 2 ? args[2] : "tests/words.txt";
auto input_file = args.length > 3 ? args[3] : "tests/numbers.txt";
size_t counter;
auto dict = load_dict(words_file);
auto writer = new OutBuffer();
writer.reserve(4096);
Array!(string) words;
words.reserve(128);
foreach( num; File(input_file).byLine)
{
immutable(ubyte[]) digits = num
.filter!(a => a.isDigit)
.map!(a => cast(ubyte) a)
.array;
find_translation(res_opt, counter, num, digits, words, dict, writer);
}
if (res_opt == "count")
writeln(counter);
} |
|
Hi @cyrusmsk . There is some error in your solution as it does not pass the tests in |
|
It took me embarrassingly long to notice why it was wrong: the solutions were missing the semi-colon between the number and solutions :D. I've run your solutions but had to abort it as it was taking too long... here's the first few results: I am profiling it right now to see why it's slow, but I am guessing your code has a few tiny things that should be easy to fix - like allocating a string for each replacement digit, using a string instead of enum for |
|
Callgrind for your solution after 1min running (using the Nearly all the time is spent on FYI: the solution as posted (plus printing the missing I am compiling with GDC now because the code runs a bit faster than with LDC. |
|
Yes, I forgot to add small improvement. import std.stdio;
import std.outbuffer;
import std.ascii;
import std.algorithm;
import std.range;
import std.array;
import std.container.array;
import std.conv;
alias Dictionary = string[][ubyte[]];
ubyte char_to_digit(char ch)
{
final switch (ch.toLower)
{
case 'e':
return 0;
case 'j', 'n', 'q':
return 1;
case 'r', 'w', 'x':
return 2;
case 'd', 's', 'y':
return 3;
case 'f', 't':
return 4;
case 'a', 'm':
return 5;
case 'c', 'i', 'v':
return 6;
case 'b', 'k', 'u':
return 7;
case 'l', 'o', 'p':
return 8;
case 'g', 'h', 'z':
return 9;
}
}
immutable(ubyte[]) word_to_number(string word)
{
auto res = word
.filter!(a => a.isAlpha)
.map!(a => char_to_digit(cast(char)a))
.map!(a => cast(ubyte)(a + '0'))
.array;
return cast(immutable(ubyte[])) res;
}
Dictionary load_dict(string words_file)
{
Dictionary dict;
foreach (word; File(words_file).byLineCopy)
{
auto key = word_to_number(word);
dict[key] ~= word;
}
return dict;
}
bool lastItemIsDigit(Array!(string) words)
{
if (words.empty)
return false;
const back = words.back();
return back.length == 1 && back[0].isDigit;
}
void find_translation(
string res_opt,
ref size_t counter,
char[] num,
immutable(ubyte[]) digits,
ref Array!(string) words,
ref Dictionary dict,
ref OutBuffer writer)
{
if (digits.length == 0)
{
handle_solution(res_opt, counter, num, words, writer);
return;
}
bool found_word = false;
foreach (i; 0 .. digits.length)
{
auto key = digits[0 .. i + 1];
auto rest_of_digits = digits[i + 1 .. $];
auto found_words = key in dict;
if (found_words !is null)
{
found_word = true;
foreach (word; *found_words)
{
words.insertBack(word);
find_translation(res_opt, counter, num, rest_of_digits, words, dict, writer);
words.removeBack();
}
}
}
if (found_word)
return;
auto last_is_digit = words.lastItemIsDigit;
if (!last_is_digit)
{
auto digit = [(digits[0] - '0').to!string];
words.insertBack(digit);
find_translation(res_opt, counter, num, digits[1 .. $], words, dict, writer);
words.removeBack();
}
}
void handle_solution(string res_opt, ref size_t counter, char[] num, ref Array!(string) words, ref OutBuffer writer)
{
if (res_opt == "count")
{
counter++;
return;
}
writer.writef("%s:",num);
if (words.empty)
{
writeln(writer.toString());
writer.clear();
return;
}
foreach (word; words)
{
writer.writef(" %s", word);
}
writeln(writer.toString());
writer.clear();
}
void main(string[] args)
{
auto res_opt = args.length > 1 ? args[1] : "print";
auto words_file = args.length > 2 ? args[2] : "tests/words.txt";
auto input_file = args.length > 3 ? args[3] : "tests/numbers.txt";
size_t counter;
auto dict = load_dict(words_file);
auto writer = new OutBuffer();
Array!(string) words;
foreach( num; File(input_file).byLine)
{
immutable(ubyte[]) digits = num
.filter!(a => a.isDigit)
.map!(a => cast(ubyte) a)
.array;
find_translation(res_opt, counter, num, digits, words, dict, writer);
}
if (res_opt == "count")
writeln(counter);
}This code use : after number. The main difference with the previous version:
I also tried to change Array!string to string[] and I've got the same speed. If previous version was about 20s, so this one should be around 11 on my machine |
|
word_to_number is using a lot of casts.. which is unnecessary. Probably it is possible to rewrite this function to not use any of it. The issue was that isAlpha producing |
|
Because the code was spending a lot of time on hashing, I tried to make sure the hash was pre-computed by using a Key struct for the keys. import std.stdio;
import std.outbuffer;
import std.ascii;
import std.algorithm;
import std.range;
import std.array;
import std.container.array;
import std.conv;
struct Key {
const ubyte[] value;
const ulong hash;
this(const ubyte[] value) {
this.value = value;
this.hash = hashOf(value);
}
bool opEquals()(auto ref const Key other) const {
return this.value == other.value;
}
ulong toHash() nothrow @safe {
return hash;
}
}
alias Dictionary = string[][immutable(Key)];
ubyte char_to_digit(char ch)
{
final switch (ch.toLower)
{
case 'e':
return 0;
case 'j', 'n', 'q':
return 1;
case 'r', 'w', 'x':
return 2;
case 'd', 's', 'y':
return 3;
case 'f', 't':
return 4;
case 'a', 'm':
return 5;
case 'c', 'i', 'v':
return 6;
case 'b', 'k', 'u':
return 7;
case 'l', 'o', 'p':
return 8;
case 'g', 'h', 'z':
return 9;
}
}
immutable(Key) word_to_number(string word)
{
ubyte[] res = word
.filter!(a => a.isAlpha)
.map!(a => char_to_digit(cast(char)a))
.map!(a => cast(ubyte)(a + '0'))
.array;
return cast(immutable(Key)) Key(res);
}
Dictionary load_dict(string words_file)
{
Dictionary dict;
foreach (word; File(words_file).byLineCopy)
{
dict[word_to_number(word)] ~= word;
}
return dict;
}
bool lastItemIsDigit(Array!(string) words)
{
if (words.empty)
return false;
const back = words.back();
return back.length == 1 && back[0].isDigit;
}
void find_translation(
OutputOption res_opt,
ref size_t counter,
char[] num,
immutable(ubyte[]) digits,
Array!(string) words,
Dictionary dict,
OutBuffer writer)
{
if (digits.length == 0)
{
handle_solution(res_opt, counter, num, words, writer);
return;
}
bool found_word = false;
foreach (i; 0 .. digits.length)
{
auto key = digits[0 .. i + 1];
auto k = Key(key);
auto found_words = (cast(immutable(Key))k) in dict;
if (found_words !is null)
{
found_word = true;
auto rest_of_digits = digits[i + 1 .. $];
foreach (word; *found_words)
{
words.insertBack(word);
find_translation(res_opt, counter, num, rest_of_digits, words, dict, writer);
words.removeBack();
}
}
}
if (found_word)
return;
auto last_is_digit = words.lastItemIsDigit;
if (!last_is_digit)
{
auto digit = [(digits[0] - '0').to!string];
words.insertBack(digit);
find_translation(res_opt, counter, num, digits[1 .. $], words, dict, writer);
words.removeBack();
}
}
enum OutputOption { PRINT, COUNT }
void handle_solution(OutputOption res_opt, ref size_t counter, char[] num, Array!(string) words, OutBuffer writer)
{
if (res_opt == OutputOption.COUNT)
{
counter++;
return;
}
writer.write(num);
writer.write(':');
if (words.empty)
{
writeln(writer.toString());
writer.clear();
return;
}
foreach (word; words)
{
writer.writef(" %s", word);
}
writeln(writer.toString());
writer.clear();
}
void main(string[] args)
{
auto res_opt = args.length > 1 ? args[1] : "print";
auto words_file = args.length > 2 ? args[2] : "tests/words.txt";
auto input_file = args.length > 3 ? args[3] : "tests/numbers.txt";
OutputOption opt;
final switch(res_opt) {
case "print": opt = OutputOption.PRINT; break;
case "count": opt = OutputOption.COUNT; break;
};
size_t counter;
auto dict = load_dict(words_file);
auto writer = new OutBuffer();
writer.reserve(4096);
Array!(string) words;
words.reserve(128);
foreach( num; File(input_file).byLine)
{
immutable(ubyte[]) digits = num
.filter!(a => a.isDigit)
.map!(a => cast(ubyte) a)
.array;
find_translation(opt, counter, num, digits, words, dict, writer);
}
if (opt == OutputOption.COUNT)
writeln(counter);
}This made the code a little bit slower :(. The profiler still blames the same two functions: But I don't know where these are coming from... any idea?
Doesn't matter because that is only used in loading the dictionary... Nearly all of the time is spent on the find_translation function, not loading the dictionary. Your current solution (named |
|
Can you add |
|
@cyrusmsk unless you can make your solution run 5x faster, I'm sorry but it's not fast enough. |
|
I've run the current fastest D solution against Rust again, this time building D with It's a bit better, as using GDC also made it a bit better... but overall, nowhere near Rust: My conclusion is that for this task, D is at the same level as Java and Common Lisp, though it probably beats both by a small margin, but around 3 or 4 times slower than Rust. |
|
Maybe. Or algorithm is just should be changed for D with GC. I'm sure D could be as fast as Rust (at least in betterC mode) - the question is just in implementation of the algorithm. |
The Rust and D algorithms are as close as they can get. Please show me if you can make it even closer as I can't see how. I do agree that rewriting the D code in betterC would possibly make it reach Rust speed, but I am not going to do it because that would require a complete change in how the solution is written... this problem requires associative arrays and growable arrays, both of which are difficult to do in betterC, as well as either a BigInt impl or int128 (not sure if int128 would work in betterC?), and most of D wouldn't help (would need to use C libs probably - defeating the purpose of using D). |
|
You can also try https://gist.github.com/ssvb/147e7ca4a4fde729fe99e5f1c487aa8d and https://gist.github.com/ssvb/38a14d3d7323ecfaf580e5482f657068 The latter variant supports extreme types of input, such as generated by the following Ruby script: #!/usr/bin/env ruby
evildict_gen = Enumerator.new do |e|
def evil_helper(e, prefix, todo)
["c", "i", "v", "C", "I", "V"].each do |letter|
if todo > 0
evil_helper(e, prefix + letter, todo - 1)
else
e.yield prefix + letter
end
end
end
0.upto(50) {|n| evil_helper(e, "", n) }
end
File.write("evildict.txt", evildict_gen.take(75000).sort_by(&:downcase).join("\n") + "\n")
File.write("evilinput.txt", "6" * 50 + "\n") |
|
On M1 Mac I've generated on Ruby script 75000 dict and 8 '6' input. Results: |
|
The |
On my example with BigInt the speed is the same. But I don't have Java installed to try it on official tests |
The Rust solution uses I'll modify my solution to "switch gears" from |
|
@ssvb I was talking about the type of the keys, not the counter. The key must be at least 90 bits IIRC which is why I used |
|
@ssvb I see that you're implementing your own hash table and doing some pretty advanced stuff to get some speed... I hope you understand that doing that goes against the spirit of this exercise, which is to write a solution that's readable and idiomatic in each language, not just that runs as fast as possible regardless. Even if you could make this code run very very fast, there's no point once you go to great lengths like that. If you really want a very fast solution in D that's still readable, I suggest you use a Trie solution like I did in Java. With that algorithm, Java was faster than Rust (because Rust was using the hash table solution, like D in this PR), so I expect a D solution based on that will be faster than even the Java solution (though if you also write Rust using Trie, Rust will still lead by a large margin given all we've learned about performance so far). |
How does this go against the spirit of this exercise? The https://flownet.com/ron/papers/lisp-java/instructions.html page sets the rules:
So basically the idea was that each participant spends up to one week to implement their solution. People were expected to work alone instead of doing direct line-by-line conversions of the already existing C++ or Java solutions from the earlier experiment. And they were encouraged to operate similar to how one would implement this code in a professional commercial grade software rather than keeping it as a simplistic and idiomatic lazy exercise.
What makes you think so? I think that it's possible to achieve optimal balance between code readability and performance without dumbing it down too much.
Implementing slightly more sophisticated algorithms actually isn't too difficult. Why is it necessary to do a half-hearted work? I spent around ~2 hours to get the initial simplistic solution: https://gist.github.com/ssvb/abe821b3cdba7fcb7f3c3cecde864153
Thanks for this suggestion. But I still prefer to design my own solution based on my own judgement. The others are encouraged to try different algorithms in their favourite programming languages. Just like you did it with your optimized Trie solution for Java.
I think that this is a common stereotype and we can prove it wrong :-) Rust people can also port my D code to Rust if they want to. |
Int128 is not enough to handle longer words. They can be up to 50 characters (of course unless you relaxed this rule in your setup). And there's |
Your entry is ignoring D's standard data structures and just doing everything from scratch. I really hope you don't write code like that in the real world. Every entry needs to balance performance with readability and code should look like what people write in the real world, not what you write for absolute performance. Please read my blog post about the problem and the intended scope of this "challenge", or the actual Prechelt's paper if you want more details. Performance is just one of the things that were considered, but not even the main one (but understandably, most people who engage, like you, only care about showing how clever their solutions can be).
You're right, I messed up and unfortunately there is no test that picks this up (and I had forgotten about the maximum length of each solution, it's been a long time since I worked on this problem). I am using 4 bits for each character because only
You're only showing that D can run fast only if you're up to rolling out your own data structures in C-like D style. Now translate your solution to Rust, which can easily be done, and see how it goes. Don't get me wrong, you're using very clever stuff in your solution, specially the way you found to not compute a full hash for all digits when adding each digit - that speeds things up immensely and if performance if of utmost importance (it isn't for this challenge), that should've been done in all solutions... But in this "D analysis" I am doing here, that's the kind of stuff I don't care at all about - all I cared about was to find out how to port the Common Lisp and Rust solutions to D, and how that would perform as I was interested in learning where D is good and where it's bad (Which I definitely did), how readable the D code looks (very readable if you stick with conventional style) and how it performs using the approximately same code (much closer to Common Lisp and Java than to Rust). Every time you show a benchmark to people they immediately say "ah but that's not the same algorithm", or "it's comparing libraries not the language" or whatever... so when I compare languages, I try to avoid those problems by translating line by line, as close as possible, solutions in the different languages, and then try to change the code to use each language's idioms and stdlib as much as possible so it looks clean and idiomatic. That's where I stop as that's all that interests me. So, go ahead and call my work "half-hearted work", I don't really care... but don't expect me to give a damn about how clever your D solution was. |
|
I want to clarify something: I came back to this problem years after I actually "ran the study" just to use D on it as I got interested in D for random reasons. What I didn't ask was for people to participate in the study with their own entries, sorry @ssvb if you got that impression! Hope you at least had fun spending all these hours on this. |
|
For the record: I fixed my int128 solution by using a special So, here is my latest solution. Here's its performance compared with Rust: Here's how @ssvb 's solution performs: Not going to lie: this is freaking amazing. I am not sure how the solution works to be honest, as I can't really understand the code ... but if this is within the specifications of the problem, this is probably as fast as it gets :)! The last two runs use the |
Adding a D solution to the print-or-count version of the problem.