From 64c62a195885b90ca425b5a9643982c19cf3c4fd Mon Sep 17 00:00:00 2001 From: Damian Jansen Date: Mon, 13 Nov 2017 11:16:58 +1000 Subject: [PATCH] feat(ZNTA-2275): JSON parser for glossary Allow the import and export of a glossary in json format. --- .../org/zanata/rest/dto/GlossaryEntry.java | 21 ++- .../zanata/rest/service/GlossaryResource.java | 4 +- common/pom.xml | 6 +- .../adapter/glossary/GlossaryJsonReader.java | 164 ++++++++++++++++++ .../adapter/glossary/GlossaryJsonWriter.java | 95 ++++++++++ .../glossary/GlossaryJsonReaderTest.java | 91 ++++++++++ .../glossary/GlossaryJsonWriterTest.java | 74 ++++++++ .../src/test/resources/glossary/glossary.json | 16 ++ .../resources/glossary/glossaryEmpty.json | 1 + .../glossary/glossaryEmptyEntry.json | 1 + .../resources/glossary/glossaryNoDesc.json | 12 ++ .../resources/glossary/glossaryNoPos.json | 12 ++ server/services/pom.xml | 2 +- .../main/java/org/zanata/dao/GlossaryDAO.java | 12 ++ .../zanata/rest/service/GlossaryService.java | 60 +++++-- .../service/impl/GlossaryFileServiceImpl.java | 42 ++++- .../db/changelogs/db.changelog-4.4.xml | 16 +- .../impl/GlossaryFileServiceImplTest.java | 107 ++++++++++++ .../src/app/actions/glossary-actions.js | 2 +- .../app/containers/Glossary/ExportModal.js | 2 + .../app/containers/Glossary/ImportModal.js | 5 +- .../src/app/reducers/glossary-reducer.js | 3 +- .../java/org/zanata/model/HGlossaryEntry.java | 9 + server/zanata-war/pom.xml | 2 +- 24 files changed, 720 insertions(+), 39 deletions(-) create mode 100644 common/zanata-adapter-glossary/src/main/java/org/zanata/adapter/glossary/GlossaryJsonReader.java create mode 100644 common/zanata-adapter-glossary/src/main/java/org/zanata/adapter/glossary/GlossaryJsonWriter.java create mode 100644 common/zanata-adapter-glossary/src/test/java/org/zanata/adapter/glossary/GlossaryJsonReaderTest.java create mode 100644 common/zanata-adapter-glossary/src/test/java/org/zanata/adapter/glossary/GlossaryJsonWriterTest.java create mode 100644 common/zanata-adapter-glossary/src/test/resources/glossary/glossary.json create mode 100644 common/zanata-adapter-glossary/src/test/resources/glossary/glossaryEmpty.json create mode 100644 common/zanata-adapter-glossary/src/test/resources/glossary/glossaryEmptyEntry.json create mode 100644 common/zanata-adapter-glossary/src/test/resources/glossary/glossaryNoDesc.json create mode 100644 common/zanata-adapter-glossary/src/test/resources/glossary/glossaryNoPos.json diff --git a/api/zanata-common-api/src/main/java/org/zanata/rest/dto/GlossaryEntry.java b/api/zanata-common-api/src/main/java/org/zanata/rest/dto/GlossaryEntry.java index 655738e733..9082beb899 100644 --- a/api/zanata-common-api/src/main/java/org/zanata/rest/dto/GlossaryEntry.java +++ b/api/zanata-common-api/src/main/java/org/zanata/rest/dto/GlossaryEntry.java @@ -47,8 +47,9 @@ **/ @XmlRootElement(name = "glossaryEntry") @XmlType(name = "glossaryEntryType", propOrder = { "id", "pos", - "description", "sourceReference", "glossaryTerms", "termsCount", "qualifiedName" }) -@JsonPropertyOrder({ "id", "pos", "description", "srcLang", "sourceReference", "glossaryTerms", "termsCount", "qualifiedName" }) + "description", "externalId", "sourceReference", "glossaryTerms", "termsCount", "qualifiedName" }) +@JsonPropertyOrder({ "id", "pos", "description", "externalId", "srcLang", "sourceReference", + "glossaryTerms", "termsCount", "qualifiedName" }) @JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) @Label("Glossary Entry") public class GlossaryEntry implements Serializable, HasMediaType { @@ -63,6 +64,8 @@ public class GlossaryEntry implements Serializable, HasMediaType { private String description; + private String externalId; + private List glossaryTerms; private LocaleId srcLang; @@ -119,6 +122,20 @@ public void setDescription(String description) { this.description = description; } + /** + * An identifier for maintenance in external tools + */ + @XmlElement(name = "externalId", namespace = Namespaces.ZANATA_OLD) + @JsonProperty("externalId") + @DocumentationExample(value = "myterm-verb") + public String getExternalId() { + return externalId; + } + + public void setExternalId(String externalId) { + this.externalId = externalId; + } + /** * Number of translated terms. A term is the glossary entry's representation * for a specific locale diff --git a/api/zanata-common-api/src/main/java/org/zanata/rest/service/GlossaryResource.java b/api/zanata-common-api/src/main/java/org/zanata/rest/service/GlossaryResource.java index df3caf23a2..ea4f65348a 100644 --- a/api/zanata-common-api/src/main/java/org/zanata/rest/service/GlossaryResource.java +++ b/api/zanata-common-api/src/main/java/org/zanata/rest/service/GlossaryResource.java @@ -226,7 +226,7 @@ Response getDetails( /** * Download all glossary entries as a file * - * @param fileType 'po' or 'csv' (case insensitive) are currently supported + * @param fileType 'po', 'json' or 'csv' (case insensitive) are currently supported * @param locales optional comma separated list of languages required. * @param qualifiedName * Qualified name of glossary, default to {@link #GLOBAL_QUALIFIED_NAME} @@ -271,7 +271,7 @@ public Response post(List glossaryEntries, @DefaultValue(GLOBAL_QUALIFIED_NAME) @QueryParam("qualifiedName") String qualifiedName); /** - * Upload glossary file (currently supported formats: po, csv) + * Upload glossary file (currently supported formats: po, csv, json) * * * @param form Multi-part form with the following named parts:
diff --git a/common/pom.xml b/common/pom.xml index 684b6bdee0..09d97c496c 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -92,7 +92,11 @@ junit junit - + + org.json + json + 20160810 + diff --git a/common/zanata-adapter-glossary/src/main/java/org/zanata/adapter/glossary/GlossaryJsonReader.java b/common/zanata-adapter-glossary/src/main/java/org/zanata/adapter/glossary/GlossaryJsonReader.java new file mode 100644 index 0000000000..3d36105b72 --- /dev/null +++ b/common/zanata-adapter-glossary/src/main/java/org/zanata/adapter/glossary/GlossaryJsonReader.java @@ -0,0 +1,164 @@ +/* + * Copyright 2018, Red Hat, Inc. and individual contributors + * as indicated by the @author tags. See the copyright.txt file in the + * distribution for a full listing of individual contributors. + * + * This is free software; you can redistribute it and/or modify it + * under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation; either version 2.1 of + * the License, or (at your option) any later version. + * + * This software is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this software; if not, write to the Free + * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA + * 02110-1301 USA, or see the FSF site: http://www.fsf.org. + */ +package org.zanata.adapter.glossary; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import org.apache.commons.lang3.StringUtils; +import org.json.JSONException; +import org.zanata.common.LocaleId; +import org.zanata.rest.dto.GlossaryEntry; +import org.json.JSONObject; +import org.json.JSONArray; +import org.zanata.rest.dto.GlossaryTerm; +import org.zanata.rest.dto.QualifiedName; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.Reader; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import static org.apache.commons.lang3.ObjectUtils.firstNonNull; +import static org.apache.commons.lang3.StringUtils.isBlank; + +/** + * @author Damian Jansen djansen@redhat.com + */ +public class GlossaryJsonReader { + private final LocaleId srcLang; + + private final static String TERM = "term"; + private final static String TRANSLATIONS = "translations"; + private final static String[] POSSYNONYMS = + {"pos", "partofspeech", "part of speech"}; + private final static String[] DESCSYNONYMS = + {"desc", "description", "definition"}; + private final static String[] EXTERNALID = {"id", "externalid", "external id"}; + + public GlossaryJsonReader(LocaleId srcLang) { + this.srcLang = srcLang; + } + + /** + * Extract a glossary from a representative json file. + * The format of the glossary should be: + * {"terms": [ + * { + * "term": "hello", + * "id": "hello-verb" + * "desc": "testing of hello json", + * "pos": "verb", + * "translations": { "es": "Hola", "zh": "您好" } + * "synonyms": "Hi", + * ... + * }, + * term2... + * ]} + * @param reader input source for the json content + * @param qualifiedName name for the glossary, e.g. global, projectname + * @return a map of glossary entries + * @throws IOException if the file is not available + */ + public Map> extractGlossary(Reader reader, + String qualifiedName) throws IOException { + BufferedReader bufferedReader = new BufferedReader(reader); + String content = bufferedReader.lines().collect(Collectors.joining()); + reader.close(); + Map> results = Maps.newHashMap(); + + try { + JSONObject jsonObj = new JSONObject(content); + JSONArray termsArray = jsonObj.getJSONArray("terms"); + List empty = Lists.newArrayList(); + // Iterate through the terms + for (int current = 0; current < termsArray.length(); ++current) { + Object obj = termsArray.get(current); + if (!(obj instanceof JSONObject)) { + continue; + } + JSONObject entry = ((JSONObject) obj); + if (!entry.has(TERM)) { + continue; + } + String srcTerm = entry.getString(TERM); + GlossaryEntry glossaryEntry = new GlossaryEntry(); + String description = getValueOf(DESCSYNONYMS, entry); + if (!isBlank(description)) { + glossaryEntry.setDescription(getValueOf(DESCSYNONYMS, entry)); + } + String pos = getValueOf(POSSYNONYMS, entry); + if (!isBlank(pos)) { + glossaryEntry.setPos(pos); + } + glossaryEntry.setQualifiedName(new QualifiedName(qualifiedName)); + glossaryEntry.setSrcLang(srcLang); + glossaryEntry.setExternalId(getValueOf(EXTERNALID, entry)); + GlossaryTerm glossaryTerm = new GlossaryTerm(); + glossaryTerm.setLocale(srcLang); + glossaryTerm.setContent(srcTerm); + glossaryEntry.getGlossaryTerms().add(glossaryTerm); + // Iterate through the translations + if (entry.has(TRANSLATIONS) && + entry.get(TRANSLATIONS) instanceof JSONObject) { + JSONObject translations = (JSONObject) entry.get(TRANSLATIONS); + Iterator transKeys = translations.keys(); + + while (transKeys.hasNext()) { + String locale = (String) transKeys.next(); + if (translations.getString(locale) != null) { + LocaleId transLocaleId = new LocaleId(locale); + String transContent = translations.getString(locale); + + GlossaryTerm transTerm = new GlossaryTerm(); + transTerm.setLocale(transLocaleId); + transTerm.setContent(transContent); + glossaryEntry.getGlossaryTerms().add(transTerm); + } + } + } + List srcEntries = firstNonNull( + results.get(srcLang), empty); + srcEntries.add(glossaryEntry); + results.put(srcLang, srcEntries); + } + } catch (ClassCastException | JSONException exception) { + throw new RuntimeException("Invalid JSON glossary file: " + .concat(exception.getMessage())); + } + return results; + } + + /* + * Attempt to return a value from the json data based on a key synonym + */ + private String getValueOf(String[] synonyms, JSONObject data) { + for (String option : synonyms) { + if (data.has(option)) { + return data.getString(option); + } else if (data.has(option.toUpperCase())) { + return data.getString(option.toUpperCase()); + } + } + return StringUtils.EMPTY; + } +} diff --git a/common/zanata-adapter-glossary/src/main/java/org/zanata/adapter/glossary/GlossaryJsonWriter.java b/common/zanata-adapter-glossary/src/main/java/org/zanata/adapter/glossary/GlossaryJsonWriter.java new file mode 100644 index 0000000000..54e3703b65 --- /dev/null +++ b/common/zanata-adapter-glossary/src/main/java/org/zanata/adapter/glossary/GlossaryJsonWriter.java @@ -0,0 +1,95 @@ +/* + * Copyright 2017, Red Hat, Inc. and individual contributors + * as indicated by the @author tags. See the copyright.txt file in the + * distribution for a full listing of individual contributors. + * + * This is free software; you can redistribute it and/or modify it + * under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation; either version 2.1 of + * the License, or (at your option) any later version. + * + * This software is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this software; if not, write to the Free + * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA + * 02110-1301 USA, or see the FSF site: http://www.fsf.org. + */ +package org.zanata.adapter.glossary; + +import com.google.common.base.Charsets; +import org.json.JSONArray; +import org.json.JSONObject; +import org.zanata.common.LocaleId; +import org.zanata.rest.dto.GlossaryEntry; +import org.zanata.rest.dto.GlossaryTerm; + +import javax.annotation.Nonnull; +import java.io.IOException; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.Writer; +import java.util.List; + +public class GlossaryJsonWriter extends AbstractGlossaryPullWriter { + + public GlossaryJsonWriter() { + } + + /** + * @see {@link #write(Writer, List, LocaleId, List)} + */ + public void write(@Nonnull OutputStream stream, + @Nonnull final List entries, + @Nonnull final LocaleId srcLocale, + @Nonnull final List transLocales) throws IOException { + OutputStreamWriter osWriter = + new OutputStreamWriter(stream, Charsets.UTF_8); + write(osWriter, entries, srcLocale, transLocales); + } + + /** + * This outputs a json file of given transLocales. + */ + public void write(@Nonnull final Writer fileWriter, + @Nonnull final List entries, + @Nonnull final LocaleId srcLocale, + @Nonnull final List transLocales) throws IOException { + + JSONObject root = new JSONObject(); + try { + + JSONArray entriesOut = new JSONArray(); + + for (GlossaryEntry entry : entries) { + GlossaryTerm srcTerm = + getGlossaryTerm(entry.getGlossaryTerms(), srcLocale); + + JSONObject newEntry = new JSONObject(); + newEntry.put("id", entry.getExternalId()); + newEntry.put("term", srcTerm.getContent()); + newEntry.put("description", entry.getDescription()); + newEntry.put("pos", entry.getPos()); + + JSONObject translations = new JSONObject(); + for (LocaleId transLocale : transLocales) { + GlossaryTerm transTerm = + getGlossaryTerm(entry.getGlossaryTerms(), transLocale); + if (transTerm != null) { + translations.put(transTerm.getLocale().toJavaName(), transTerm.getContent()); + } + } + newEntry.put("translations", translations); + entriesOut.put(newEntry); + } + root.put("terms", entriesOut); + } finally { + fileWriter.write(root.toString(2)); + fileWriter.close(); + } + } + +} diff --git a/common/zanata-adapter-glossary/src/test/java/org/zanata/adapter/glossary/GlossaryJsonReaderTest.java b/common/zanata-adapter-glossary/src/test/java/org/zanata/adapter/glossary/GlossaryJsonReaderTest.java new file mode 100644 index 0000000000..144697e260 --- /dev/null +++ b/common/zanata-adapter-glossary/src/test/java/org/zanata/adapter/glossary/GlossaryJsonReaderTest.java @@ -0,0 +1,91 @@ +/* + * Copyright 2017, Red Hat, Inc. and individual contributors + * as indicated by the @author tags. See the copyright.txt file in the + * distribution for a full listing of individual contributors. + * + * This is free software; you can redistribute it and/or modify it + * under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation; either version 2.1 of + * the License, or (at your option) any later version. + * + * This software is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this software; if not, write to the Free + * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA + * 02110-1301 USA, or see the FSF site: http://www.fsf.org. + */ +package org.zanata.adapter.glossary; + +import org.junit.Test; +import org.zanata.common.LocaleId; +import org.zanata.rest.dto.GlossaryEntry; +import org.zanata.rest.service.GlossaryResource; +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.*; +import java.util.List; +import java.util.Map; + +public class GlossaryJsonReaderTest { + + @Test + public void extractGlossary() throws IOException { + Map> glossaries = getGlossaries("glossary"); + + assertThat(glossaries.entrySet()).hasSize(1); + + for (Map.Entry> entry : glossaries.entrySet()) { + assertThat(entry.getValue().get(0).getDescription()).isEqualTo("testing of hello json"); + assertThat(entry.getValue().get(0).getExternalId()).isEqualTo("hello-verb"); + assertThat(entry.getValue().get(1).getDescription()).isEqualTo("testing of morning json"); + assertThat(entry.getValue().get(1).getExternalId()).isEmpty(); + } + } + + @Test + public void emptyGlossary() throws IOException { + Map> glossaries = getGlossaries("glossaryEmpty"); + assertThat(glossaries.keySet()).hasSize(0); + } + + @Test + public void emptyGlossaryEntry() throws IOException { + Map> glossaries = getGlossaries("glossaryEmptyEntry"); + assertThat(glossaries.keySet()).hasSize(0); + } + + @Test + public void glossaryEntryMissingDesc() throws IOException { + Map> glossaries = getGlossaries("glossaryNoDesc"); + assertThat(glossaries.keySet()).hasSize(1); + + for (Map.Entry> entry : glossaries.entrySet()) { + assertThat(entry.getValue().get(0).getPos()).isEqualTo("verb"); + assertThat(entry.getValue().get(0).getDescription()).isNull(); + } + } + + @Test + public void glossaryEntryMissingPos() throws IOException { + Map> glossaries = getGlossaries("glossaryNoPos"); + assertThat(glossaries.keySet()).hasSize(1); + + for (Map.Entry> entry : glossaries.entrySet()) { + assertThat(entry.getValue().get(0).getPos()).isNull(); + assertThat(entry.getValue().get(0).getDescription()).isEqualTo("testing of hello json"); + } + } + + private Map> getGlossaries(String filename) throws IOException { + GlossaryJsonReader glossaryJsonReader = new GlossaryJsonReader(LocaleId.EN_US); + String fullPath = "src/test/resources/glossary/" + filename + ".json"; + BufferedReader br = new BufferedReader( + new InputStreamReader(new FileInputStream(new File(fullPath)), "UTF-8")); + + return glossaryJsonReader.extractGlossary(br, GlossaryResource.GLOBAL_QUALIFIED_NAME); + } +} diff --git a/common/zanata-adapter-glossary/src/test/java/org/zanata/adapter/glossary/GlossaryJsonWriterTest.java b/common/zanata-adapter-glossary/src/test/java/org/zanata/adapter/glossary/GlossaryJsonWriterTest.java new file mode 100644 index 0000000000..c9be7a1318 --- /dev/null +++ b/common/zanata-adapter-glossary/src/test/java/org/zanata/adapter/glossary/GlossaryJsonWriterTest.java @@ -0,0 +1,74 @@ +package org.zanata.adapter.glossary; + +import org.junit.Test; +import org.zanata.common.LocaleId; +import org.zanata.rest.dto.GlossaryEntry; +import org.zanata.rest.service.GlossaryResource; + +import java.io.*; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +public class GlossaryJsonWriterTest extends AbstractGlossaryWriterTest { + + @Test + public void glossaryWriteTest() throws IOException { + GlossaryJsonWriter writer = new GlossaryJsonWriter(); + String filePath = "target/output.json"; + + FileWriter fileWriter = new FileWriter(filePath); + LocaleId srcLocale = LocaleId.EN_US; + + List entries = new ArrayList<>(); + GlossaryEntry entry1 = + generateGlossaryEntry(srcLocale, "pos", "desc"); + entry1.setExternalId("1.content-en-us"); + entry1.getGlossaryTerms().add(generateGlossaryTerm("1.content-en-us", LocaleId.EN_US)); + entry1.getGlossaryTerms().add(generateGlossaryTerm("1.content-de", LocaleId.DE)); + entry1.getGlossaryTerms().add(generateGlossaryTerm("1.content-es", LocaleId.ES)); + entries.add(entry1); + + GlossaryEntry entry2 = + generateGlossaryEntry(srcLocale, "pos", "desc"); + entry2.setExternalId("2.content-en-us"); + entry2.getGlossaryTerms().add(generateGlossaryTerm("2.content-en-us", LocaleId.EN_US)); + entry2.getGlossaryTerms().add(generateGlossaryTerm("2.content-de", LocaleId.DE)); + entry2.getGlossaryTerms().add(generateGlossaryTerm("2.content-es", LocaleId.ES)); + entries.add(entry2); + + GlossaryEntry entry3 = + generateGlossaryEntry(srcLocale, "pos", "desc"); + entry3.setExternalId("3.content-en-us"); + entry3.getGlossaryTerms().add(generateGlossaryTerm("3.content-en-us", LocaleId.EN_US)); + entry3.getGlossaryTerms().add(generateGlossaryTerm("3.content-de", LocaleId.DE)); + entry3.getGlossaryTerms().add(generateGlossaryTerm("3.content-es", LocaleId.ES)); + entries.add(entry3); + + List transLocales = new ArrayList<>(); + transLocales.add(LocaleId.DE); + transLocales.add(LocaleId.ES); + + writer.write(fileWriter, entries, srcLocale, transLocales); + + GlossaryJsonReader reader = new GlossaryJsonReader(srcLocale); + File sourceFile = new File(filePath); + + Reader inputStreamReader = + new InputStreamReader(new FileInputStream(sourceFile), "UTF-8"); + BufferedReader br = new BufferedReader(inputStreamReader); + + Map> glossaries = reader + .extractGlossary(br, GlossaryResource.GLOBAL_QUALIFIED_NAME); + br.close(); + assertThat(glossaries).hasSize(1); + assertThat(glossaries.get(LocaleId.EN_US)).hasSize(3); + for (int entry = 0; entry < 3; ++entry) { + GlossaryEntry thisEntry = glossaries.get(LocaleId.EN_US).get(entry); + assertThat(thisEntry.getGlossaryTerms().get(0).getContent() + .equals(String.valueOf(entry).concat(".content-en-us"))); + } + } +} diff --git a/common/zanata-adapter-glossary/src/test/resources/glossary/glossary.json b/common/zanata-adapter-glossary/src/test/resources/glossary/glossary.json new file mode 100644 index 0000000000..8bd201d3a6 --- /dev/null +++ b/common/zanata-adapter-glossary/src/test/resources/glossary/glossary.json @@ -0,0 +1,16 @@ +{"terms":[ + { + "id": "hello-verb", + "term": "hello", + "description": "testing of hello json", + "pos": "verb", + "translations": { "es": "Hola", "zh": "您好" } + }, + { + "term": "morning", + "definition": "testing of morning json", + "partofspeech": "noun", + "translations": { "es": "mañana", "zh": "上午" } + } + ] +} diff --git a/common/zanata-adapter-glossary/src/test/resources/glossary/glossaryEmpty.json b/common/zanata-adapter-glossary/src/test/resources/glossary/glossaryEmpty.json new file mode 100644 index 0000000000..20ca3e97ad --- /dev/null +++ b/common/zanata-adapter-glossary/src/test/resources/glossary/glossaryEmpty.json @@ -0,0 +1 @@ +{"terms":[]} diff --git a/common/zanata-adapter-glossary/src/test/resources/glossary/glossaryEmptyEntry.json b/common/zanata-adapter-glossary/src/test/resources/glossary/glossaryEmptyEntry.json new file mode 100644 index 0000000000..4000682d2a --- /dev/null +++ b/common/zanata-adapter-glossary/src/test/resources/glossary/glossaryEmptyEntry.json @@ -0,0 +1 @@ +{"terms":[{}]} diff --git a/common/zanata-adapter-glossary/src/test/resources/glossary/glossaryNoDesc.json b/common/zanata-adapter-glossary/src/test/resources/glossary/glossaryNoDesc.json new file mode 100644 index 0000000000..6bf72a30a9 --- /dev/null +++ b/common/zanata-adapter-glossary/src/test/resources/glossary/glossaryNoDesc.json @@ -0,0 +1,12 @@ +{"terms":[ + { + "term": "hello", + "pos": "verb", + "translations": { "es": "Hola", "zh": "您好" } + }, + { + "term": "morning", + "partofspeech": "noun", + "translations": { "es": "mañana", "zh": "上午" } + }] +} diff --git a/common/zanata-adapter-glossary/src/test/resources/glossary/glossaryNoPos.json b/common/zanata-adapter-glossary/src/test/resources/glossary/glossaryNoPos.json new file mode 100644 index 0000000000..4902ca6f48 --- /dev/null +++ b/common/zanata-adapter-glossary/src/test/resources/glossary/glossaryNoPos.json @@ -0,0 +1,12 @@ +{"terms":[ + { + "term": "hello", + "description": "testing of hello json", + "translations": { "es": "Hola", "zh": "您好" } + }, + { + "term": "morning", + "definition": "testing of morning json", + "translations": { "es": "mañana", "zh": "上午" } + }] +} diff --git a/server/services/pom.xml b/server/services/pom.xml index 1b03ced99c..c859a6be88 100644 --- a/server/services/pom.xml +++ b/server/services/pom.xml @@ -505,7 +505,7 @@ org.json json - 20140107 + 20160810 diff --git a/server/services/src/main/java/org/zanata/dao/GlossaryDAO.java b/server/services/src/main/java/org/zanata/dao/GlossaryDAO.java index 3fd50026fc..8ad70682f3 100644 --- a/server/services/src/main/java/org/zanata/dao/GlossaryDAO.java +++ b/server/services/src/main/java/org/zanata/dao/GlossaryDAO.java @@ -251,6 +251,18 @@ public HGlossaryEntry getEntryByContentHash(String contentHash, return (HGlossaryEntry) query.uniqueResult(); } + public HGlossaryEntry getEntryByExternalId(String externalId, String qualifiedName) { + String queryBuilder = "from HGlossaryEntry as e " + + "WHERE e.externalId =:externalId " + + "AND e.glossary.qualifiedName =:qualifiedName"; + + Query query = getSession().createQuery(queryBuilder) + .setParameter("externalId", externalId) + .setParameter("qualifiedName", qualifiedName); + query.setComment("GlossaryDAO.getEntryByExternalId"); + return (HGlossaryEntry) query.uniqueResult(); + } + public List findTermByIdList(List idList) { if (idList == null || idList.isEmpty()) { return Lists.newArrayList(); diff --git a/server/services/src/main/java/org/zanata/rest/service/GlossaryService.java b/server/services/src/main/java/org/zanata/rest/service/GlossaryService.java index c71e847824..68a17e3ebf 100644 --- a/server/services/src/main/java/org/zanata/rest/service/GlossaryService.java +++ b/server/services/src/main/java/org/zanata/rest/service/GlossaryService.java @@ -22,11 +22,7 @@ import java.io.IOException; import java.io.OutputStream; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; +import java.util.*; import java.util.stream.Collectors; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; @@ -48,6 +44,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.zanata.adapter.glossary.GlossaryCSVWriter; +import org.zanata.adapter.glossary.GlossaryJsonWriter; import org.zanata.adapter.glossary.GlossaryPoWriter; import org.zanata.common.GlossarySortField; import org.zanata.common.LocaleId; @@ -127,10 +124,8 @@ public Response getInfo( .forEach(locale -> { LocaleDetails localeDetails = LocaleService.convertHLocaleToDTO(locale); - int count = transMap.containsKey(locale.getLocaleId()) - ? transMap.get(locale.getLocaleId()) : 0; - transLocale - .add(new GlossaryLocaleInfo(localeDetails, count)); + transLocale.add(new GlossaryLocaleInfo(localeDetails, + transMap.getOrDefault(locale.getLocaleId(), 0))); }); GlossaryInfo glossaryInfo = new GlossaryInfo(srcGlossaryLocale, transLocale); @@ -254,10 +249,10 @@ public Response downloadFile(@DefaultValue("csv") String fileType, if (response != null) { return response; } - if (!fileType.equalsIgnoreCase("csv") - && !fileType.equalsIgnoreCase("po")) { + + if (Arrays.stream(new String[]{"csv", "po", "json"}).noneMatch(x -> x.equalsIgnoreCase(fileType))) { return Response.status(Response.Status.BAD_REQUEST) - .entity("Not supported file type-" + fileType).build(); + .entity("Not supported file type: " + fileType).build(); } LocaleId srcLocaleId = getSourceLocale().getLocaleId(); // use commaSeparatedLanguage is exist, @@ -281,10 +276,18 @@ public Response downloadFile(@DefaultValue("csv") String fileType, List entries = Lists.newArrayList(); transferEntriesResource(glossaryDAO.getEntries(qualifiedName), entries); try { - GlossaryStreamingOutput output = fileType.equalsIgnoreCase("csv") - ? new CSVStreamingOutput(entries, srcLocaleId, transLocales) - : new PotStreamingOutput(entries, srcLocaleId, - transLocales); + GlossaryStreamingOutput output; + switch (fileType.toLowerCase()) { + case "csv": output = new CSVStreamingOutput(entries, srcLocaleId, transLocales); + break; + case "po": output = new PotStreamingOutput(entries, srcLocaleId, transLocales); + break; + case "json": output = new JsonStreamingOutput(entries, srcLocaleId, transLocales); + break; + default: return Response.status(Response.Status.BAD_REQUEST) + .entity("Invalid glossary file type: " + fileType) + .build(); + } String filename = getFileName(qualifiedName, fileType); return Response.ok() .header("Content-Disposition", @@ -305,8 +308,9 @@ public Response downloadFile(@DefaultValue("csv") String fileType, private String getFileName(String qualifiedName, String type) { String filePrefix = isProjectGlossary(qualifiedName) ? getProjectSlug(qualifiedName) + "_" : ""; - return type.equalsIgnoreCase("csv") ? filePrefix + "glossary.csv" - : filePrefix + "glossary.zip"; + return type.equalsIgnoreCase("po") + ? filePrefix + "glossary.zip" + : filePrefix + "glossary.".concat(type); } @Override @@ -551,6 +555,7 @@ public GlossaryEntry generateGlossaryEntry(HGlossaryEntry hGlossaryEntry) { glossaryEntry.setQualifiedName(new QualifiedName( hGlossaryEntry.getGlossary().getQualifiedName())); glossaryEntry.setTermsCount(hGlossaryEntry.getGlossaryTerms().size()); + glossaryEntry.setExternalId(hGlossaryEntry.getExternalId()); return glossaryEntry; } @@ -628,4 +633,23 @@ public void write(OutputStream output) } } } + + private class JsonStreamingOutput extends GlossaryStreamingOutput { + + public JsonStreamingOutput(List entries, + LocaleId srcLocaleId, List transLocales) { + super(entries, srcLocaleId, transLocales); + } + + @Override + public void write(OutputStream output) + throws IOException, WebApplicationException { + try { + GlossaryJsonWriter writer = new GlossaryJsonWriter(); + writer.write(output, entries, srcLocaleId, transLocales); + } finally { + output.close(); + } + } + } } diff --git a/server/services/src/main/java/org/zanata/service/impl/GlossaryFileServiceImpl.java b/server/services/src/main/java/org/zanata/service/impl/GlossaryFileServiceImpl.java index a51829a315..b49515e3ba 100644 --- a/server/services/src/main/java/org/zanata/service/impl/GlossaryFileServiceImpl.java +++ b/server/services/src/main/java/org/zanata/service/impl/GlossaryFileServiceImpl.java @@ -29,6 +29,7 @@ import java.util.Map; import java.util.Optional; import java.util.TreeSet; +import java.util.UUID; import java.util.stream.Collectors; import org.apache.commons.io.FilenameUtils; import org.apache.commons.lang3.StringUtils; @@ -40,6 +41,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.zanata.adapter.glossary.GlossaryCSVReader; +import org.zanata.adapter.glossary.GlossaryJsonReader; import org.zanata.adapter.glossary.GlossaryPoReader; import org.zanata.common.LocaleId; import org.zanata.dao.GlossaryDAO; @@ -90,6 +92,8 @@ public Map> parseGlossaryFile( } else if (FilenameUtils.getExtension(fileName).equals("po")) { return parsePoFile(inputStream, sourceLang, transLang, qualifiedName); + } else if (FilenameUtils.getExtension(fileName).equals("json")) { + return parseJsonFile(sourceLang, qualifiedName, inputStream); } throw new ZanataServiceException( "Unsupported Glossary file: " + fileName); @@ -116,8 +120,17 @@ public GlossaryProcessed saveOrUpdateGlossary( } continue; } - message = checkForDuplicateEntry(entry); boolean onlyTransferTransTerm = false; + if (StringUtils.isBlank(entry.getExternalId())) { + entry.setExternalId(UUID.randomUUID().toString()); + while(entryExists(entry)) { + entry.setExternalId(UUID.randomUUID().toString()); + } + } else { + onlyTransferTransTerm = entryExists(entry); + } + + message = checkForDuplicateEntry(entry); if (message.isPresent()) { // only update transTerm warnings.add(message.get()); @@ -177,12 +190,8 @@ private Optional validateGlossaryEntry(GlossaryEntry entry) { } private Optional getSourceTerm(GlossaryEntry entry) { - for (GlossaryTerm term : entry.getGlossaryTerms()) { - if (term.getLocale().equals(entry.getSrcLang())) { - return Optional.of(term); - } - } - return Optional.empty(); + return entry.getGlossaryTerms().stream() + .filter(p -> p.getLocale().equals(entry.getSrcLang())).findFirst(); } public static class GlossaryProcessed { @@ -221,6 +230,13 @@ private Map> parseCsvFile(LocaleId sourceLang, Charsets.UTF_8.displayName()), qualifiedName); } + private Map> parseJsonFile(LocaleId sourceLang, + String qualifiedName, InputStream inputStream) throws IOException { + GlossaryJsonReader jsonReader = new GlossaryJsonReader(sourceLang); + return jsonReader.extractGlossary(new InputStreamReader(inputStream, + Charsets.UTF_8.displayName()), qualifiedName); + } + private Map> parsePoFile( InputStream inputStream, LocaleId sourceLang, LocaleId transLang, String qualifiedName) throws IOException { @@ -253,6 +269,9 @@ private HGlossaryEntry getOrCreateGlossaryEntry(GlossaryEntry from, HGlossaryEntry hGlossaryEntry; if (id != null) { hGlossaryEntry = glossaryDAO.findById(id); + } else if (entryExists(from)) { + hGlossaryEntry = glossaryDAO.getEntryByExternalId(from.getExternalId(), + from.getQualifiedName().getName()); } else { hGlossaryEntry = glossaryDAO.getEntryByContentHash(contentHash, from.getQualifiedName().getName()); @@ -266,6 +285,14 @@ private HGlossaryEntry getOrCreateGlossaryEntry(GlossaryEntry from, return hGlossaryEntry; } + /** + * Check if entry exists + */ + private boolean entryExists(GlossaryEntry entry) { + return null != glossaryDAO.getEntryByExternalId(entry.getExternalId(), + entry.getQualifiedName().getName()); + } + /** * Check if request save/update entry have duplication with same source * content, pos, and description @@ -303,6 +330,7 @@ private HGlossaryEntry transferGlossaryEntryAndSave(GlossaryEntry from, to.setSourceRef(from.getSourceReference()); to.setPos(from.getPos()); to.setDescription(from.getDescription()); + to.setExternalId(from.getExternalId()); String qualifiedName = GlossaryUtil.GLOBAL_QUALIFIED_NAME; if (from.getQualifiedName() != null && StringUtils.isNotBlank(from.getQualifiedName().getName())) { diff --git a/server/services/src/main/resources/db/changelogs/db.changelog-4.4.xml b/server/services/src/main/resources/db/changelogs/db.changelog-4.4.xml index 8a63017a18..84ad49561e 100644 --- a/server/services/src/main/resources/db/changelogs/db.changelog-4.4.xml +++ b/server/services/src/main/resources/db/changelogs/db.changelog-4.4.xml @@ -36,14 +36,24 @@ referencedTableName="ReviewCriteria" referencedColumnNames="id" /> + + + Add externalId column to HGlossaryEntry + + + + + + + - + + + columnName="description" columnDataType="varchar(255)"/> - diff --git a/server/services/src/test/java/org/zanata/service/impl/GlossaryFileServiceImplTest.java b/server/services/src/test/java/org/zanata/service/impl/GlossaryFileServiceImplTest.java index 35a28cec1a..192cceaffc 100644 --- a/server/services/src/test/java/org/zanata/service/impl/GlossaryFileServiceImplTest.java +++ b/server/services/src/test/java/org/zanata/service/impl/GlossaryFileServiceImplTest.java @@ -60,6 +60,7 @@ import javax.enterprise.inject.Produces; import javax.inject.Inject; import javax.persistence.EntityManager; +import org.zanata.service.impl.GlossaryFileServiceImpl.GlossaryProcessed; import static org.assertj.core.api.Assertions.assertThat; @@ -152,6 +153,112 @@ public void parseGlossaryFilePoTest() throws UnsupportedEncodingException { .contains(srcLocaleId, transLocaleId); } + @Test + @InRequestScope + public void parseGlossaryFileJsonTest() throws UnsupportedEncodingException { + String jsonSample = "{\"terms\":[" + + "{ \"term\":\"test\"," + + "\"id\":\"test-noun\"," + + "\"description\":\"something that verifies\"," + + "\"pos\":\"verb\"," + + "\"translations\": { \"de\":\"ttttt\" }" + + "}]}"; + + InputStream stubInputStream = IOUtils.toInputStream(jsonSample); + + String fileName = "fileName.json"; + LocaleId srcLocaleId = LocaleId.EN_US; + LocaleId transLocaleId = LocaleId.DE; + + Map> result = + glossaryFileService.parseGlossaryFile(stubInputStream, + fileName, srcLocaleId, + null, GlossaryResource.GLOBAL_QUALIFIED_NAME); + + assertThat(result).hasSize(1); + + List entries = result.get(srcLocaleId); + + assertThat(entries).hasSize(1); + + GlossaryEntry entry = entries.get(0); + assertThat(entry.getSrcLang()).isEqualTo(srcLocaleId); + assertThat(entry.getExternalId()).isEqualTo("test-noun"); + assertThat(entry.getGlossaryTerms()).hasSize(2) + .extracting("locale") + .contains(srcLocaleId, transLocaleId); + } + + @Test + @InRequestScope + public void glossaryPosAttributeBoundaryTest() throws UnsupportedEncodingException { + String jsonSample = "{\"terms\":[" + + "{ \"term\":\"test\"," + + "\"id\":\"test-noun\"," + + "\"description\":\"something that verifies\"," + + "\"pos\":\"FoCAvyFATGWGBqZNsS7wsyO56f5pfyrU8DhoNbCy9IbUZEBEfmItC8s7SDVg" + + "VKJi8Nb7EzzRUvP8o XfzoiGIeBX4IZSO7hZb2LjkqM64fGnmohxOhO1fAOvGtjXSgDFoe" + + "Iw7GvtBFtaJv6sa8Xw8ZotLkvcE2ie4yQ1w tBm58aoaGyT2apnHGv6YaNXHWygwjGmI2M" + + "LemjEf1lkB03QvNhjPqxXpeakE2iTe0Po1n2DAXXBst6ERz7j8clD3BouX\"," + + "\"translations\": { \"de\":\"ttttt\" }" + + "}]}"; + + GlossaryProcessed glossaryProcessed = glossaryFileService.saveOrUpdateGlossary( + glossaryFileService.parseGlossaryFile(IOUtils.toInputStream(jsonSample), + "fileName.json", LocaleId.EN_US, + null, GlossaryResource.GLOBAL_QUALIFIED_NAME) + .get(LocaleId.EN_US), Optional.ofNullable(LocaleId.EN_US)); + + assertThat(glossaryProcessed.getGlossaryEntries()).hasSize(0); + assertThat(glossaryProcessed.getWarnings()) + .contains("Glossary part of speech too long, maximum 255 character"); + } + + @Test + @InRequestScope + public void glossaryDescAttributeBoundaryTest() throws UnsupportedEncodingException { + String jsonSample = "{\"terms\":[" + + "{ \"term\":\"test\"," + + "\"id\":\"test-noun\"," + + "\"pos\":\"noun\"," + + "\"description\":\"FoCAvyFATGWGBqZNsS7wsyO56f5pfyrU8DhoNbCy9IbUZEBEfmItC8" + + "s7SDVgVKJi8Nb7EzzRUvP8oXfzoiGIeBX4IZSO7hZb2LjkqM64fGnmohxOhO1fAOvGtjXSgD" + + "FoeIw7GvtBFtaJv6sa8Xw8ZotLkvcE2ie4yQ1wtBm58aoaGyT2apnHGv6YaNXHWygwjGmI2M" + + "LemjEf1lkB03QvNhjPqxXpeakE2iTe0Po1n2DAXXBst6ERz7j8clD3BouX\"," + + "\"translations\": { \"de\":\"ttttt\" }" + + "}]}"; + + GlossaryProcessed glossaryProcessed = glossaryFileService.saveOrUpdateGlossary( + glossaryFileService.parseGlossaryFile(IOUtils.toInputStream(jsonSample), + "fileName.json", LocaleId.EN_US, + null, GlossaryResource.GLOBAL_QUALIFIED_NAME) + .get(LocaleId.EN_US), Optional.ofNullable(LocaleId.EN_US)); + + assertThat(glossaryProcessed.getGlossaryEntries()).hasSize(0); + assertThat(glossaryProcessed.getWarnings()).contains("Glossary description too long, maximum " + + "255 character"); + } + + @Test + @InRequestScope + public void saveGlossaryFileJsonTest() throws UnsupportedEncodingException { + String jsonSample = "{\"terms\":[" + + "{ \"term\":\"test\"," + + "\"description\":\"something that verifies\"," + + "\"pos\":\"verb\"," + + "\"translations\": { \"de\":\"ttttt\" }" + + "}]}"; + + GlossaryProcessed glossaryProcessed = glossaryFileService.saveOrUpdateGlossary( + glossaryFileService.parseGlossaryFile(IOUtils.toInputStream(jsonSample), + "fileName.json", LocaleId.EN_US, + null, GlossaryResource.GLOBAL_QUALIFIED_NAME) + .get(LocaleId.EN_US), Optional.ofNullable(LocaleId.EN_US)); + + assertThat(glossaryProcessed.getGlossaryEntries()).hasSize(1); + assertThat(glossaryProcessed.getGlossaryEntries().get(0).getExternalId()).isNotBlank(); + } + @Test @InRequestScope public void saveOrUpdateGlossaryTest() { diff --git a/server/zanata-frontend/src/app/actions/glossary-actions.js b/server/zanata-frontend/src/app/actions/glossary-actions.js index a6280adf8c..b7b9e4c700 100644 --- a/server/zanata-frontend/src/app/actions/glossary-actions.js +++ b/server/zanata-frontend/src/app/actions/glossary-actions.js @@ -15,7 +15,7 @@ import { } from './common-actions' import { apiUrl } from '../config' -export const FILE_TYPES = ['csv', 'po'] +export const FILE_TYPES = ['csv', 'po', 'json'] export const PAGE_SIZE_SELECTION = [20, 50, 100, 300, 500] // 500 by default export const PAGE_SIZE_DEFAULT = last(PAGE_SIZE_SELECTION) diff --git a/server/zanata-frontend/src/app/containers/Glossary/ExportModal.js b/server/zanata-frontend/src/app/containers/Glossary/ExportModal.js index d54baefc31..2c965106cb 100644 --- a/server/zanata-frontend/src/app/containers/Glossary/ExportModal.js +++ b/server/zanata-frontend/src/app/containers/Glossary/ExportModal.js @@ -48,6 +48,8 @@ class ExportModal extends Component { message = This will download glossary entries in all languages into csv format. } else if (type.value === FILE_TYPES[1]) { message = This will download a zip file of glossary entries in all languages in po format. + } else if (type.value === FILE_TYPES[2]) { + message = This will download glossary entries in all languages in json format. } /* eslint-enable max-len */ const exportGlossaryUrl = diff --git a/server/zanata-frontend/src/app/containers/Glossary/ImportModal.js b/server/zanata-frontend/src/app/containers/Glossary/ImportModal.js index b264185fee..4e25e78b02 100644 --- a/server/zanata-frontend/src/app/containers/Glossary/ImportModal.js +++ b/server/zanata-frontend/src/app/containers/Glossary/ImportModal.js @@ -112,8 +112,9 @@ class ImportModal extends Component { : langSelection }

- CSV and PO files are supported. The source language should - be in {locale}. For more details on how to prepare glossary + CSV, JSON and PO files are supported. + The source language should be in {locale}. + For more details on how to prepare glossary files, see our glossary import documentation.

diff --git a/server/zanata-frontend/src/app/reducers/glossary-reducer.js b/server/zanata-frontend/src/app/reducers/glossary-reducer.js index bbdc1ddd8d..346e6cb9db 100644 --- a/server/zanata-frontend/src/app/reducers/glossary-reducer.js +++ b/server/zanata-frontend/src/app/reducers/glossary-reducer.js @@ -722,7 +722,8 @@ const glossary = handleActions({ status: -1, types: [ {value: FILE_TYPES[0], label: FILE_TYPES[0]}, - {value: FILE_TYPES[1], label: FILE_TYPES[1]} + {value: FILE_TYPES[1], label: FILE_TYPES[1]}, + {value: FILE_TYPES[2], label: FILE_TYPES[2]} ] }, newEntry: { diff --git a/server/zanata-model/src/main/java/org/zanata/model/HGlossaryEntry.java b/server/zanata-model/src/main/java/org/zanata/model/HGlossaryEntry.java index 8a540b3ec3..e1e10e8c4d 100644 --- a/server/zanata-model/src/main/java/org/zanata/model/HGlossaryEntry.java +++ b/server/zanata-model/src/main/java/org/zanata/model/HGlossaryEntry.java @@ -59,6 +59,7 @@ public class HGlossaryEntry extends ModelEntityBase { private String contentHash; private String pos; private String description; + private String externalId; private HLocale srcLocale; private Glossary glossary; @@ -107,6 +108,10 @@ public String getDescription() { return description; } + public String getExternalId() { + return externalId; + } + public static class EntityListener { @PreUpdate @@ -170,6 +175,10 @@ public void setGlossary(final Glossary glossary) { this.glossary = glossary; } + public void setExternalId(final String externalId) { + this.externalId = externalId; + } + @Override public boolean equals(final Object o) { if (o == this) diff --git a/server/zanata-war/pom.xml b/server/zanata-war/pom.xml index 96752e2ca2..3f77b828e5 100644 --- a/server/zanata-war/pom.xml +++ b/server/zanata-war/pom.xml @@ -920,7 +920,7 @@ org.json json - 20140107 + 20160810