Skip to content

Commit

Permalink
Better logging, more testing, removed some code smells.
Browse files Browse the repository at this point in the history
  • Loading branch information
janothan committed Nov 8, 2021
1 parent 9e4565e commit d537864
Show file tree
Hide file tree
Showing 7 changed files with 89 additions and 49 deletions.
48 changes: 31 additions & 17 deletions src/main/java/de/uni_mannheim/informatik/dws/jrdf2vec/Main.java
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,11 @@ public class Main {
*/
public static final String DEFAULT_MERGE_FILE = "./mergedWalks.txt";

/**
* This variable is static merely for testing
*/
private static boolean isServerOk;

/**
* The main method that is executed when running the JAR.
*
Expand Down Expand Up @@ -194,11 +199,27 @@ public static void main(String[] args) {
return;
}

String portString = getValue("-port", args);
if (portString != null) {
try {
int intPort = Integer.parseInt(portString);
Gensim.setPort(intPort);
port = intPort;
} catch (NumberFormatException nfe) {
System.out.println("A problem occurred while trying to parse the following port number: " + portString
+ "\nUsing default port: " + Gensim.DEFAULT_PORT);
}
}
System.out.println("Using server port: " + port);

// check install
if (containsIgnoreCase("-checkInstall", args) || containsIgnoreCase("-check", args) || containsIgnoreCase(
"-checkRequirements", args) || containsIgnoreCase("-checkInstallation", args)) {
boolean isOk = Gensim.checkRequirements();
if (isOk) {
if (containsIgnoreCase("-checkInstall", args) ||
containsIgnoreCase("-check", args) ||
containsIgnoreCase("-checkRequirements", args) ||
containsIgnoreCase("-checkInstallation", args)) {

isServerOk = Gensim.getInstance().checkRequirements();
if (isServerOk) {
System.out.println("Installation is ok! [✔︎]");
} else {
System.out.println("Installation is not ok! [❌]\nIs Python 3 installed? Please check the log for " +
Expand Down Expand Up @@ -254,6 +275,8 @@ public static void main(String[] args) {
isEmbedText = true;
}



if (containsIgnoreCase("-onlyTraining", args)) {
isOnlyTraining = true;
String walksPath = getValue("-walkDirectory", args);
Expand All @@ -271,19 +294,6 @@ public static void main(String[] args) {
String knowledgeGraphFilePath = getValue("-graph", args);
if (knowledgeGraphFilePath == null) knowledgeGraphFilePath = getValue("-g", args);

String portString = getValue("-port", args);
if (portString != null) {
try {
int intPort = Integer.parseInt(portString);
Gensim.setPort(intPort);
port = intPort;
} catch (NumberFormatException nfe) {
System.out.println("A problem occurred while trying to parse the following port number: " + portString
+ "\nUsing default port: " + Gensim.DEFAULT_PORT);
}
}
System.out.println("Using server port: " + port);

isOnlyWalks = containsIgnoreCase("-onlyWalks", args);
// allowing a bit more...
if (!isOnlyWalks) isOnlyWalks = containsIgnoreCase("-walksOnly", args);
Expand Down Expand Up @@ -949,6 +959,10 @@ public static int getDepth() {
return depth;
}

public static boolean isIsServerOk() {
return isServerOk;
}

/**
* Obtain the arguments that were not (yet) parsed.
*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -189,19 +189,20 @@ public String train() {

Instant before = Instant.now();

WalkGenerationManager classicGenerator;
WalkGenerationManager walkGenerationManager;
if (useFile) {
classicGenerator = new WalkGenerationManager(getFile(this.knowledgeGraphUri).toURI(), isEmbedText(),
walkGenerationManager = new WalkGenerationManager(getFile(this.knowledgeGraphUri).toURI(), isEmbedText(),
true, existingWalkDirectory, this.walkDirectory);
} else {
classicGenerator = new WalkGenerationManager(this.ontModel, isEmbedText());
walkGenerationManager = new WalkGenerationManager(this.ontModel, isEmbedText());
}

classicGenerator.generateWalks(walkGenerationMode, numberOfThreads, numberOfWalksPerEntity, depth,
walkGenerationManager.generateWalks(walkGenerationMode, numberOfThreads, numberOfWalksPerEntity, depth,
configuration.getWindowSize(), getWalkDirectory());

Instant after = Instant.now();
this.requiredTimeForLastWalkGenerationString = Util.getDeltaTimeString(before, after);
LOGGER.info("Walks successfully generated. Starting training now...");

before = Instant.now();
Gensim gensim;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,7 @@ public void train() {

Instant after = Instant.now();
this.requiredTimeForLastWalkGenerationString = Util.getDeltaTimeString(before, after);
LOGGER.info("Walks successfully generated. Starting training now...");

before = Instant.now();
Gensim gensim;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -562,8 +562,7 @@ public static Gensim getInstance(File resourcesDirectory) {
*
* @return True if the server is fully functional, else false.
*/
public static boolean checkRequirements() {
Gensim gensim = Gensim.getInstance();
public boolean checkRequirements() {
HttpGet request = new HttpGet(serverUrl + "/check-requirements");
File requirementsFile = new File(DEFAULT_RESOURCES_DIRECTORY + "requirements.txt");
if (!requirementsFile.exists()) {
Expand All @@ -575,11 +574,7 @@ public static boolean checkRequirements() {
HttpEntity entity = response.getEntity();
String resultMessage = EntityUtils.toString(entity);
System.out.println(resultMessage);
if (resultMessage.contains("good to go")) {
return true;
} else {
return false;
}
return resultMessage.contains("good to go");
} catch (IOException ioe) {
LOGGER.error("Problem with http request.", ioe);
return false;
Expand All @@ -592,6 +587,7 @@ public static boolean checkRequirements() {
public static void shutDown() {
isShutDown = true;
instance = null;
port = DEFAULT_PORT;
try {
if (httpClient != null)
httpClient.close();
Expand Down Expand Up @@ -679,7 +675,7 @@ private boolean startServer() {
//pb.environment().put("PATH", "{FOLDER CONTAINING PYTHON EXE}" + File.pathSeparator + pb.environment().get("PATH"));
try {
pb.inheritIO();
this.serverProcess = pb.start();
serverProcess = pb.start();
final int maxTrials = 8;
for (int i = 0; i < maxTrials; i++) {
HttpGet request = new HttpGet(serverUrl + "/melt_ml.html");
Expand Down Expand Up @@ -720,6 +716,7 @@ private boolean startServer() {
}));
isHookStarted = true;
}
LOGGER.info("Server started.\nServer URL: " + getServerUrl() + "\nServer port: " + getPort());
return true;
}

Expand Down Expand Up @@ -939,6 +936,10 @@ public static int getPort() {
return port;
}

/**
* Set the port, must be performed before the server is started.
* @param port The port to be set.
*/
public static void setPort(int port) {
if (instance != null) {
LOGGER.error("Server is already running. The port cannot be changed.");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -539,6 +539,10 @@ public static void deleteDirectory(File directory) {
deleteDirectory(directory.getAbsolutePath());
}

/**
* Deletes the directory stated as string path.
* @param directoryPath The path to the directory which shall be deleted.
*/
public static void deleteDirectory(String directoryPath) {
try {
FileUtils.deleteDirectory(new File(directoryPath));
Expand Down
57 changes: 38 additions & 19 deletions src/test/java/MainTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -284,7 +284,7 @@ public void trainClassicWithOwlFileTextGeneration() {
String walkPath = "." + File.separator + "mainWalksOwlText" + File.separator;
File walkDirectory = new File(walkPath);
walkDirectory.deleteOnExit();
walkDirectory.mkdir();
assertTrue(walkDirectory.mkdir());
String graphFilePath = loadFile("pizza.owl.xml").getAbsolutePath();
String[] args = {"-graph", graphFilePath, "-walkDir", walkPath, "-sample", "0.01", "-embedText", "-window", "5"};
Main.main(args);
Expand Down Expand Up @@ -325,7 +325,7 @@ public void trainClassicWithNtFileTextGeneration() {
String walkPath = "." + File.separator + "mainWalksNtText" + File.separator;
File walkDirectory = new File(walkPath);
walkDirectory.deleteOnExit();
walkDirectory.mkdir();
assertTrue(walkDirectory.mkdir());
String graphFilePath = loadFile("dummyGraph_with_labels.nt").getAbsolutePath();
String[] args = {"-graph", graphFilePath, "-walkDir", walkPath, "-noVectorTextFileGeneration", "-sample", "0.01", "-embedText", "-window", "8"};
Main.main(args);
Expand Down Expand Up @@ -497,7 +497,7 @@ public void testTxtVectorGenerationLight() {
File vectorFile = new File(modelFile.getParentFile().getAbsolutePath(), "vectors.txt");
assertTrue(vectorFile.exists());
assertEquals(2, getNumberOfLines(vectorFile));
vectorFile.delete();
deleteFile(vectorFile);
assertEquals(0, Main.getIgnoredArguments().size());

// error case 1: directory
Expand All @@ -506,15 +506,15 @@ public void testTxtVectorGenerationLight() {
Main.main(args);
assertTrue(newVectorFile.exists());
assertTrue(getNumberOfLines(newVectorFile) > 5);
newVectorFile.delete();
deleteFile(newVectorFile);

// error case 2: non-existing file
args = new String[]{"-generateTxtVectorFile", modelFilePath, "-light", "./this_file_does_not_exist.txt"};
Main.main(args);
vectorFile = new File(modelFile.getParentFile().getAbsolutePath(), "vectors.txt");
assertTrue(vectorFile.exists());
assertTrue(getNumberOfLines(vectorFile) > 5);
vectorFile.delete();
deleteFile(vectorFile);
}

@Test
Expand Down Expand Up @@ -709,7 +709,7 @@ public void trainLightTdb(WalkGenerationMode walkGenerationMode) {
@Test
public void trainLightNoVectorTextFile() {
File lightWalks = new File("./mainLightWalks/");
lightWalks.mkdir();
assertTrue(lightWalks.mkdir());;
lightWalks.deleteOnExit();
String entityFilePath = loadFile("dummyEntities.txt").getAbsolutePath();
String graphFilePath = loadFile("dummyGraph.nt").getAbsolutePath();
Expand All @@ -718,9 +718,13 @@ public void trainLightNoVectorTextFile() {
Main.main(args);

assertEquals(RDF2VecLight.class, Main.getRdf2VecInstance().getClass());
assertTrue(lightWalks.listFiles().length > 0);
File[] lightWalksFiles = lightWalks.listFiles();
assertNotNull(lightWalksFiles);
assertTrue(lightWalksFiles.length > 0);
assertEquals(RDF2VecLight.DEFAULT_WALK_GENERATION_MODE, Main.getWalkGenerationMode());
HashSet<String> files = Sets.newHashSet(lightWalks.list());
String[] lightWalksFilesString = lightWalks.list();
assertNotNull(lightWalksFilesString);
HashSet<String> files = Sets.newHashSet(lightWalksFilesString);

// assert that all files are there
assertTrue(files.contains("model.kv"));
Expand Down Expand Up @@ -756,11 +760,24 @@ public void parameterCheck() {
@Test
void checkRequirements() {
Main.main(new String[]{"-checkInstallation"});
assertEquals(Gensim.DEFAULT_PORT, Gensim.getPort());
assertEquals("http://127.0.0.1:" + Gensim.DEFAULT_PORT, Gensim.getServerUrl());
assertTrue(Main.isIsServerOk());
}

@Test
void checkRequirementsWithDifferentPort() {
Main.main(new String[]{"-checkInstallation", "-port", "2025"});
assertEquals("http://127.0.0.1:2025", Gensim.getServerUrl());
assertEquals(2025, Gensim.getPort());
assertTrue(Main.isIsServerOk());
}

@Test
void writeVocabularyToFile() {
File modelFile = new File(getPathOfResource("test_model_vectors.kv"));
String resourcePath = getPathOfResource("test_model_vectors.kv");
assertNotNull(resourcePath);
File modelFile = new File(resourcePath);
Main.main(new String[]{"-generateVocabularyFile", modelFile.getAbsolutePath()});
File vocabFile = new File(modelFile.getParentFile().getAbsolutePath(), "vocabulary.txt");
vocabFile.deleteOnExit();
Expand Down Expand Up @@ -832,12 +849,13 @@ public void plainWalkGenerationLightHdtFile() {
File graphFileToUse = loadFile("swdf-2012-11-28.hdt");

// prepare directory
File walkDirectory = new File("./walksOnly/");
walkDirectory.mkdir();
File walkDirectory = new File("./walksOnlyHdt/");
assertTrue(walkDirectory.mkdir());
walkDirectory.deleteOnExit();

String lightFilePath = loadFile("./swdf_light_entities.txt").getAbsolutePath();
Main.main(new String[]{"-graph", graphFileToUse.getAbsolutePath(), "-numberOfWalks", "100", "-light", lightFilePath, "-onlyWalks", "-walkDir", "./walksOnly/"});
Main.main(new String[]{"-graph", graphFileToUse.getAbsolutePath(), "-numberOfWalks", "100", "-light",
lightFilePath, "-onlyWalks", "-walkDir", "./walksOnlyHdt/"});

// check ignored arguments
assertEquals(0, Main.getIgnoredArguments().size());
Expand Down Expand Up @@ -883,7 +901,7 @@ public void plainWalkGenerationLightHdtFile() {
}

// clean up
walkDirectory.delete();
deleteDirectory(walkDirectory);
}

@Test
Expand Down Expand Up @@ -1163,7 +1181,7 @@ public void lightWithDuplicateFreeMode() {

// prepare directory
File walkDirectory = new File("./walksOnly/");
walkDirectory.mkdir();
assertTrue(walkDirectory.mkdir());
walkDirectory.deleteOnExit();


Expand Down Expand Up @@ -1214,8 +1232,8 @@ public void lightWithDuplicateFreeMode() {
}

// clean up
graphFileToUse.delete();
walkDirectory.delete();
deleteFile(graphFileToUse);
deleteFile(walkDirectory);
}

/**
Expand All @@ -1238,7 +1256,7 @@ public void plainWalkGenerationLightNtFile() {
// prepare directory
String walkDirectoryName = "./walksOnly2/";
File walkDirectory = new File(walkDirectoryName);
walkDirectory.mkdir();
assertTrue(walkDirectory.mkdir());
walkDirectory.deleteOnExit();

String lightFilePath = loadFile("swdf_light_entities.txt").getAbsolutePath();
Expand Down Expand Up @@ -1288,8 +1306,8 @@ public void plainWalkGenerationLightNtFile() {
}

// clean up
graphFileToUse.delete();
walkDirectory.delete();
deleteFile(graphFileToUse);
deleteFile(walkDirectory);
}

@Test
Expand Down Expand Up @@ -1654,6 +1672,7 @@ static void cleanUp() {
deleteDirectory("./extClassic");
deleteDirectory("./extLight");
deleteDirectory("./walksOnly");
deleteDirectory("./walksOnlyHdt");
deleteDirectory("./walksOnly2");
deleteDirectory("./classicWalks/");
deleteDirectory("./mainWalks/");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ public static void tearDown(){

@Test
void checkRequirements(){
assertTrue(Gensim.checkRequirements());
assertTrue(Gensim.getInstance().checkRequirements());
}

/**
Expand Down

0 comments on commit d537864

Please sign in to comment.