Permalink
Cannot retrieve contributors at this time
file_tokens_raw = './results/tokenization/tokens_raw.txt'; | |
file_tokens_filtered = './results/filtering/tokens_filtered.txt'; | |
fid_tokens_raw = fopen(file_tokens_raw, 'rt'); | |
lines = fscanf(fid_tokens_raw, '%c'); | |
fclose(fid_tokens_raw); | |
tokens = strsplit(lines); | |
num_token = length(tokens); | |
fid_tokens_filtered = fopen(file_tokens_filtered, 'wt'); | |
for id_token = 1:num_token | |
token_cur = tokens{id_token}; | |
idx_slash = find(token_cur == '/'); | |
category_cur = token_cur(idx_slash + 1:end); | |
if strcmp(category_cur, 'n') || strcmp(category_cur, 'a') || strcmp(category_cur, 'd') | |
fprintf(fid_tokens_filtered, [token_cur(1:idx_slash - 1), '\n']); | |
end | |
disp(['Processing Token ', num2str(id_token), '...']); | |
end | |
fclose(fid_tokens_filtered); |