From 05562cac002d3ba6b1d5bb24c9f286839d041a64 Mon Sep 17 00:00:00 2001 From: Joey Salazar Date: Thu, 20 Oct 2022 11:14:20 -0600 Subject: [PATCH 1/3] Change column names upon project start Fixes #75 --- .../extensions/commons/importer/CommonsImporter.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/main/java/org/openrefine/extensions/commons/importer/CommonsImporter.java b/src/main/java/org/openrefine/extensions/commons/importer/CommonsImporter.java index 745b12e..145ee80 100644 --- a/src/main/java/org/openrefine/extensions/commons/importer/CommonsImporter.java +++ b/src/main/java/org/openrefine/extensions/commons/importer/CommonsImporter.java @@ -97,6 +97,15 @@ static public void parse( 1); col.setReconStats(ReconStats.create(project, 0)); col.setReconConfig(cfg); + col.setName("File"); + if (mIdsColumn) { + project.columnModel.columns.get(1).setName("M-ids"); + if (categoriesColumn) { + project.columnModel.columns.get(2).setName("Categories"); + } + } else if (categoriesColumn) { + project.columnModel.columns.get(1).setName("Categories"); + } setProgress(job, categoriesWithDepth.get(categoriesWithDepth.size()-1).categoryName, 100); } From 98ebc11a8190353a51897a7e8518c425ba176981 Mon Sep 17 00:00:00 2001 From: Joey Salazar Date: Mon, 31 Oct 2022 07:15:09 -0600 Subject: [PATCH 2/3] Add test case --- .../commons/importer/CommonsImporterTest.java | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 src/test/java/org/openrefine/extensions/commons/importer/CommonsImporterTest.java diff --git a/src/test/java/org/openrefine/extensions/commons/importer/CommonsImporterTest.java b/src/test/java/org/openrefine/extensions/commons/importer/CommonsImporterTest.java new file mode 100644 index 0000000..0eba642 --- /dev/null +++ b/src/test/java/org/openrefine/extensions/commons/importer/CommonsImporterTest.java @@ -0,0 +1,56 @@ +package org.openrefine.extensions.commons.importer; + +import java.util.ArrayList; +import java.util.List; + +import org.mockito.Mockito; +import org.testng.Assert; +import org.testng.annotations.Test; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.refine.ProjectMetadata; +import com.google.refine.RefineServlet; +import com.google.refine.importing.ImportingJob; +import com.google.refine.importing.ImportingManager; +import com.google.refine.model.Cell; +import com.google.refine.model.Project; +import com.google.refine.util.ParsingUtilities; + +public class CommonsImporterTest { + + protected RefineServlet servlet; + + // dependencies + private Project project; + private ProjectMetadata metadata; + private ImportingJob job; + + /** + * Test column names upon project creation as well as reconciled cells + */ + @Test + public void testParse() throws Exception { + + servlet = new RefineServlet(); + ImportingManager.initialize(servlet); + project = new Project(); + metadata = new ProjectMetadata(); + metadata.setName("Commons Import Test Project"); + job = Mockito.mock(ImportingJob.class); + ObjectNode options = ParsingUtilities.evaluateJsonStringToObjectNode( + "{\"categoryJsonValue\":[{\"category\":\"Category:Costa Rica\",\"depth\":\"0\"}],\"skipDataLines\":0," + + "\"limit\":-1,\"disableAutoPreview\":false,\"categoriesColumn\":true,\"mIdsColumn\":true}"); + List exceptions = new ArrayList(); + + CommonsImporter.parse(project, metadata, job, 0, options, exceptions); + project.update(); + Cell cell = project.rows.get(0).cells.get(0); + + Assert.assertEquals(project.columnModel.columns.get(0).getName(), "File"); + Assert.assertEquals(project.columnModel.columns.get(1).getName(), "M-ids"); + Assert.assertEquals(project.columnModel.columns.get(2).getName(), "Categories"); + Assert.assertEquals(cell.recon.match.id, "M114916202"); + Assert.assertEquals(cell.recon.match.name, "File:2006-08-08 Vista Parque Nacional Volcan Arenal, Costa Rica.jpg"); + + } +} From 9fc7c3f41a8144220bde7eb44e7d660bcaee3de3 Mon Sep 17 00:00:00 2001 From: Joey Salazar Date: Mon, 31 Oct 2022 08:17:27 -0600 Subject: [PATCH 3/3] Use mock API call in test case --- .../commons/importer/CommonsImporter.java | 13 ++++- .../commons/importer/CommonsImporterTest.java | 55 ++++++++++++------- 2 files changed, 47 insertions(+), 21 deletions(-) diff --git a/src/main/java/org/openrefine/extensions/commons/importer/CommonsImporter.java b/src/main/java/org/openrefine/extensions/commons/importer/CommonsImporter.java index 145ee80..c39835b 100644 --- a/src/main/java/org/openrefine/extensions/commons/importer/CommonsImporter.java +++ b/src/main/java/org/openrefine/extensions/commons/importer/CommonsImporter.java @@ -21,6 +21,18 @@ public class CommonsImporter { + static String apiUrl = "https://commons.wikimedia.org/w/api.php"; + + + /** + * Utility method for testing with mock api calls + * + * @param limit + */ + public void setApiUrl(String apiUrlTest) { + apiUrl = apiUrlTest; + } + static public void parsePreview( Project project, ProjectMetadata metadata, @@ -60,7 +72,6 @@ static public void parse( categoriesWithDepth.add(new CategoryWithDepth(category.get("category").asText(), category.get("depth").asInt())); } - String apiUrl = "https://commons.wikimedia.org/w/api.php";//FIXME String service = "https://commonsreconcile.toolforge.org/en/api"; // initializes progress reporting with the name of the first category diff --git a/src/test/java/org/openrefine/extensions/commons/importer/CommonsImporterTest.java b/src/test/java/org/openrefine/extensions/commons/importer/CommonsImporterTest.java index 0eba642..0c457e6 100644 --- a/src/test/java/org/openrefine/extensions/commons/importer/CommonsImporterTest.java +++ b/src/test/java/org/openrefine/extensions/commons/importer/CommonsImporterTest.java @@ -16,6 +16,10 @@ import com.google.refine.model.Project; import com.google.refine.util.ParsingUtilities; +import okhttp3.HttpUrl; +import okhttp3.mockwebserver.MockResponse; +import okhttp3.mockwebserver.MockWebServer; + public class CommonsImporterTest { protected RefineServlet servlet; @@ -31,26 +35,37 @@ public class CommonsImporterTest { @Test public void testParse() throws Exception { - servlet = new RefineServlet(); - ImportingManager.initialize(servlet); - project = new Project(); - metadata = new ProjectMetadata(); - metadata.setName("Commons Import Test Project"); - job = Mockito.mock(ImportingJob.class); - ObjectNode options = ParsingUtilities.evaluateJsonStringToObjectNode( - "{\"categoryJsonValue\":[{\"category\":\"Category:Costa Rica\",\"depth\":\"0\"}],\"skipDataLines\":0," - + "\"limit\":-1,\"disableAutoPreview\":false,\"categoriesColumn\":true,\"mIdsColumn\":true}"); - List exceptions = new ArrayList(); - - CommonsImporter.parse(project, metadata, job, 0, options, exceptions); - project.update(); - Cell cell = project.rows.get(0).cells.get(0); - - Assert.assertEquals(project.columnModel.columns.get(0).getName(), "File"); - Assert.assertEquals(project.columnModel.columns.get(1).getName(), "M-ids"); - Assert.assertEquals(project.columnModel.columns.get(2).getName(), "Categories"); - Assert.assertEquals(cell.recon.match.id, "M114916202"); - Assert.assertEquals(cell.recon.match.name, "File:2006-08-08 Vista Parque Nacional Volcan Arenal, Costa Rica.jpg"); + try (MockWebServer server = new MockWebServer()) { + server.start(); + HttpUrl url = server.url("/w/api.php"); + String jsonResponse = "{\"batchcomplete\":\"\",\"query\":{\"categorymembers\":" + + "[{\"pageid\":127722,\"ns\":6,\"title\":\"File:3 Puppies.jpg\",\"type\":\"file\"}]}}"; + server.enqueue(new MockResponse().setBody(jsonResponse)); + servlet = new RefineServlet(); + ImportingManager.initialize(servlet); + project = new Project(); + metadata = new ProjectMetadata(); + metadata.setName("Commons Import Test Project"); + job = Mockito.mock(ImportingJob.class); + ObjectNode options = ParsingUtilities.evaluateJsonStringToObjectNode( + "{\"categoryJsonValue\":[{\"category\":\"Category:Costa Rica\",\"depth\":\"0\"}],\"skipDataLines\":0," + + "\"limit\":-1,\"disableAutoPreview\":false,\"categoriesColumn\":true,\"mIdsColumn\":true}"); + List exceptions = new ArrayList(); + CommonsImporter importer = new CommonsImporter(); + + importer.setApiUrl(url.toString()); + CommonsImporter.parse(project, metadata, job, 0, options, exceptions); + project.update(); + Cell cell = project.rows.get(0).cells.get(0); + + Assert.assertEquals(project.columnModel.columns.get(0).getName(), "File"); + Assert.assertEquals(project.columnModel.columns.get(1).getName(), "M-ids"); + Assert.assertEquals(project.columnModel.columns.get(2).getName(), "Categories"); + Assert.assertEquals(cell.recon.match.id, "M127722"); + Assert.assertEquals(cell.recon.match.name, "File:3 Puppies.jpg"); + + server.close(); + } } }