Skip to content

Commit

Permalink
Refactor Tokenizer (#1605)
Browse files Browse the repository at this point in the history
  • Loading branch information
pivovarit authored and maibin committed Apr 7, 2017
1 parent e7d81e6 commit 196ea6b
Showing 1 changed file with 38 additions and 38 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -7,46 +7,46 @@
import java.util.Collections;
import java.util.List;
import java.util.StringTokenizer;
import java.util.stream.Collectors;

public class Application {

public List<String> getTokens(String str) {
List<String> tokens = new ArrayList<String>();
// StringTokenizer tokenizer = new StringTokenizer( str );
StringTokenizer tokenizer = new StringTokenizer(str, ",");
// StringTokenizer tokenizer = new StringTokenizer( str , "," , true );
while (tokenizer.hasMoreElements()) {
tokens.add(tokenizer.nextToken());
// tokens.add( tokenizer.nextToken("e") );
}
int tokenLength = tokens.size();
return tokens;
}

public List<String> getTokensWithCollection( String str ) {
StringTokenizer tokenizer = new StringTokenizer(str, ",");
List<String> tokens = new ArrayList<String>();
Collections.list(tokenizer).forEach(token -> tokens.add((String) token));
return tokens;
}

public List<String> getTokensFromFile(String path, String delim) {
List<String> tokens = new ArrayList<String>();
String currLine = "";
StringTokenizer tokenizer;
try (BufferedReader br = new BufferedReader(
new InputStreamReader(Application.class.getResourceAsStream("/" + path)))) {
while ((currLine = br.readLine()) != null) {
tokenizer = new StringTokenizer(currLine, delim);
while (tokenizer.hasMoreElements()) {
tokens.add(tokenizer.nextToken());
}
}
} catch (IOException e) {
e.printStackTrace();
}
return tokens;
}

public List<String> getTokens(String str) {
List<String> tokens = new ArrayList<String>();
// StringTokenizer tokenizer = new StringTokenizer( str );
StringTokenizer tokenizer = new StringTokenizer(str, ",");
// StringTokenizer tokenizer = new StringTokenizer( str , "," , true );
while (tokenizer.hasMoreElements()) {
tokens.add(tokenizer.nextToken());
// tokens.add( tokenizer.nextToken("e") );
}
int tokenLength = tokens.size();
return tokens;
}

public List<String> getTokensWithCollection(String str) {
StringTokenizer tokenizer = new StringTokenizer(str, ",");

return Collections.list(tokenizer).stream()
.map(token -> (String) token)
.collect(Collectors.toList());
}

public List<String> getTokensFromFile(String path, String delim) {
List<String> tokens = new ArrayList<>();
String currLine = "";
StringTokenizer tokenizer;
try (BufferedReader br = new BufferedReader(new InputStreamReader(Application.class.getResourceAsStream("/" + path)))) {
while ((currLine = br.readLine()) != null) {
tokenizer = new StringTokenizer(currLine, delim);
while (tokenizer.hasMoreElements()) {
tokens.add(tokenizer.nextToken());
}
}
} catch (IOException e) {
e.printStackTrace();
}
return tokens;
}

}

0 comments on commit 196ea6b

Please sign in to comment.