Permalink
Browse files

Feature/repo 1066 tika upgrade (#72)

* REPO-1066: update tika lib dependencies, pdfbox and poi

* [maven-release-plugin] prepare release alfresco-repository-6.21_feature_repo-1066_tika_upgrade

* [maven-release-plugin] prepare for next development iteration

* REPO-1066: fix the preferences service code to work with the new json getString method

* REPO-1066: last commit did not add the new file

* [maven-release-plugin] prepare release alfresco-repository-6.21_feature_repo-1066_tika_upgrade

* REPO-1066: update pom as it was broken by the failed release attempt

* [maven-release-plugin] prepare release alfresco-repository-6.21_feature_repo-1066_tika_upgrade_3

* [maven-release-plugin] prepare for next development iteration

* REPO-1066: test out tika without  com.tdunning json lib

* [maven-release-plugin] prepare release alfresco-repository-6.21_feature_repo-1066_tika_upgrade_4

* [maven-release-plugin] prepare for next development iteration

* REPO-1322 Upgrade poi to 3.17

* REPO-1322 Remove unused ooxml-schemas lib

* REPO-1322 Remove unused test in PoiMetadataExtracterTest

* REPO-1066 Fix metadata exctraction for compressed JPEG (MNT-13920)

* REPO-1066: fix tika breaking change from TIKA-2096

* REPO-1066: run tests with the patched tika version

* [maven-release-plugin] prepare release alfresco-repository-6.21_feature_repo-1066_tika_upgrade_13

* [maven-release-plugin] prepare for next development iteration

* REPO-1066: upgrade dependencies

* REPO-1066: fixed some formatting

* REPO-1066: clean up not useful classes and clean formatting
  • Loading branch information...
andrei-rebegea committed Feb 1, 2018
1 parent 8510896 commit 5de7c1d9746a6ba08e9a8b7c0ad886503d4b5cc3
View
33 pom.xml
@@ -37,9 +37,9 @@
<dependency.alfresco-legacy-lucene.version>6.2</dependency.alfresco-legacy-lucene.version>
<dependency.alfresco-mbeans.version>6.0</dependency.alfresco-mbeans.version>
<dependency.alfresco-core.version>6.16</dependency.alfresco-core.version>
<dependency.alfresco-core.version>6.17</dependency.alfresco-core.version>
<dependency.alfresco-greenmail.version>6.1</dependency.alfresco-greenmail.version>
<dependency.alfresco-data-model.version>7.1</dependency.alfresco-data-model.version>
<dependency.alfresco-data-model.version>7.2</dependency.alfresco-data-model.version>
<dependency.alfresco-jlan.version>6.4</dependency.alfresco-jlan.version>
<dependency.alfresco-pdf-renderer.version>1.0</dependency.alfresco-pdf-renderer.version>
<dependency.alfresco-hb-data-sender.version>1.0.4</dependency.alfresco-hb-data-sender.version>
@@ -48,7 +48,7 @@
<dependency.spring.version>3.2.17.RELEASE</dependency.spring.version>
<dependency.httpcomponents.version>4.5.2</dependency.httpcomponents.version>
<dependency.truezip.version>7.7.9</dependency.truezip.version>
<dependency.poi.version>3.10.1-20151016-alfresco-patched</dependency.poi.version>
<dependency.poi.version>3.17</dependency.poi.version>
<dependency.webscripts.version>6.13</dependency.webscripts.version>
<dependency.opencmis.version>1.0.0</dependency.opencmis.version>
<dependency.activiti-engine.version>5.22.0</dependency.activiti-engine.version>
@@ -145,7 +145,7 @@
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-compress</artifactId>
<version>1.12</version>
<version>1.15</version>
</dependency>
<dependency>
<groupId>commons-codec</groupId>
@@ -295,12 +295,12 @@
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>1.7.21</version>
<version>1.7.24</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<version>1.7.21</version>
<version>1.7.25</version>
</dependency>
<dependency>
<groupId>org.beanshell</groupId>
@@ -371,17 +371,23 @@
<dependency>
<groupId>org.apache.pdfbox</groupId>
<artifactId>pdfbox</artifactId>
<version>1.8.10-alfresco-patched</version>
<version>2.0.8</version>
</dependency>
<dependency>
<groupId>org.apache.pdfbox</groupId>
<artifactId>fontbox</artifactId>
<version>1.8.10</version>
<version>2.0.8</version>
</dependency>
<dependency>
<!-- we need this for TextToPdfContentTransformer -->
<groupId>org.apache.pdfbox</groupId>
<artifactId>pdfbox-tools</artifactId>
<version>2.0.8</version>
</dependency>
<dependency>
<groupId>org.apache.pdfbox</groupId>
<artifactId>jempbox</artifactId>
<version>1.8.10</version>
<version>1.8.13</version>
</dependency>
<dependency>
<groupId>org.bouncycastle</groupId>
@@ -396,12 +402,12 @@
<dependency>
<groupId>com.googlecode.mp4parser</groupId>
<artifactId>isoparser</artifactId>
<version>1.1.21</version>
<version>1.1.22</version>
</dependency>
<dependency>
<groupId>com.drewnoakes</groupId>
<artifactId>metadata-extractor</artifactId>
<version>2.7.2</version>
<version>2.10.1</version>
</dependency>
<dependency>
<groupId>org.ccil.cowan.tagsoup</groupId>
@@ -512,11 +518,6 @@
<artifactId>poi-scratchpad</artifactId>
<version>${dependency.poi.version}</version>
</dependency>
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>ooxml-schemas</artifactId>
<version>1.1</version>
</dependency>
<dependency>
<groupId>org.livetribe</groupId>
<artifactId>livetribe-jsr223</artifactId>
@@ -1,28 +1,28 @@
/*
* #%L
* Alfresco Repository
* %%
* Copyright (C) 2005 - 2016 Alfresco Software Limited
* %%
* This file is part of the Alfresco software.
* If the software was purchased under a paid Alfresco license, the terms of
* the paid license agreement will prevail. Otherwise, the software is
* provided under the following open source license terms:
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
* #L%
*/
/*
* #%L
* Alfresco Repository
* %%
* Copyright (C) 2005 - 2016 Alfresco Software Limited
* %%
* This file is part of the Alfresco software.
* If the software was purchased under a paid Alfresco license, the terms of
* the paid license agreement will prevail. Otherwise, the software is
* provided under the following open source license terms:
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
* #L%
*/
package org.alfresco.filesys.repo;
import java.io.BufferedInputStream;
@@ -42,7 +42,7 @@
import org.alfresco.util.TempFileProvider;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.poi.hslf.HSLFSlideShow;
import org.apache.poi.hslf.usermodel.HSLFSlideShowImpl;
import org.apache.poi.hssf.usermodel.HSSFWorkbook;
import org.apache.poi.poifs.filesystem.DirectoryEntry;
import org.apache.poi.poifs.filesystem.EntryUtils;
@@ -412,8 +412,8 @@ public boolean isContentEqual(ContentReader existingContent, File newFile)
excludes.add("Current User");
leftIs = existingContent.getContentInputStream();
HSLFSlideShow slideShow1 = new HSLFSlideShow(leftIs);
HSLFSlideShow slideShow2 = new HSLFSlideShow(new FileInputStream(newFile));
HSLFSlideShowImpl slideShow1 = new HSLFSlideShowImpl(leftIs);
HSLFSlideShowImpl slideShow2 = new HSLFSlideShowImpl(new FileInputStream(newFile));
String lastEditUsername1 = slideShow1.getCurrentUserAtom().getLastEditUsername();
String lastEditUsername2 = slideShow2.getCurrentUserAtom().getLastEditUsername();
@@ -30,7 +30,6 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.TikaMetadataKeys;
/**
* Default implementation for the MetadataExtracterConfig;
@@ -53,7 +52,7 @@ public void prepareMetadataWithConfigParams(Metadata metadata)
return;
}
boolean shouldParseShapes = getBooleanProperty(PARSE_SHAPE_PROP_STRING, TIKA_PARSER_PARSE_SHAPES_DEFAULT_VALUE);
metadata.add(TikaMetadataKeys.TIKA_PARSER_PARSE_SHAPES_KEY, Boolean.toString(shouldParseShapes));
//TODO decide in REPO-3251 if this method remains or not.
if (logger.isDebugEnabled())
{
@@ -1,28 +1,28 @@
/*
* #%L
* Alfresco Repository
* %%
* Copyright (C) 2005 - 2016 Alfresco Software Limited
* %%
* This file is part of the Alfresco software.
* If the software was purchased under a paid Alfresco license, the terms of
* the paid license agreement will prevail. Otherwise, the software is
* provided under the following open source license terms:
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
* #L%
*/
/*
* #%L
* Alfresco Repository
* %%
* Copyright (C) 2005 - 2016 Alfresco Software Limited
* %%
* This file is part of the Alfresco software.
* If the software was purchased under a paid Alfresco license, the terms of
* the paid license agreement will prevail. Otherwise, the software is
* provided under the following open source license terms:
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
* #L%
*/
package org.alfresco.repo.content.metadata;
import java.util.ArrayList;
@@ -31,7 +31,6 @@
import org.alfresco.repo.content.MimetypeMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.poi.patch.AlfrescoPoiPatchUtils;
import org.apache.tika.parser.Parser;
import org.apache.tika.parser.microsoft.ooxml.OOXMLParser;
import org.springframework.beans.factory.InitializingBean;
@@ -47,112 +46,31 @@
* <b>Any custom property:</b> -- [not mapped]
* </pre>
*
* Uses Apache Tika<br />
* <br />
* Configures {@link AlfrescoPoiPatchUtils} to resolve the following issues:
* <ul>
* <li><a href="https://issues.alfresco.com/jira/browse/MNT-577">MNT-577</a></li>
* <li><a href="https://issues.alfresco.com/jira/browse/MNT-11823">MNT-11823</a></li>
* </ul>
*
* Uses Apache Tika
*
* @author Nick Burch
* @author Neil McErlean
* @author Dmitry Velichkevich
*/
public class PoiMetadataExtracter extends TikaPoweredMetadataExtracter implements InitializingBean
public class PoiMetadataExtracter extends TikaPoweredMetadataExtracter
{
protected static Log logger = LogFactory.getLog(PoiMetadataExtracter.class);
public static ArrayList<String> SUPPORTED_MIMETYPES = buildSupportedMimetypes(
public static ArrayList<String> SUPPORTED_MIMETYPES = buildSupportedMimetypes(
new String[] {MimetypeMap.MIMETYPE_OPENXML_WORDPROCESSING,
MimetypeMap.MIMETYPE_OPENXML_SPREADSHEET,
MimetypeMap.MIMETYPE_OPENXML_PRESENTATION},
new OOXMLParser()
new OOXMLParser()
);
private Integer poiFootnotesLimit;
private Boolean poiExtractPropertiesOnly = false;
private Set<String> poiAllowableXslfRelationshipTypes;
public PoiMetadataExtracter()
{
super(PoiMetadataExtracter.class.getName(), SUPPORTED_MIMETYPES);
}
@Override
protected Parser getParser()
protected Parser getParser()
{
return new OOXMLParser();
}
/**
* MNT-577: Alfresco is running 100% CPU for over 10 minutes while extracting metadata for Word office document <br />
* <br />
*
* @param poiFootnotesLimit - {@link Integer} value which specifies limit of amount of footnotes of XWPF documents
*/
public void setPoiFootnotesLimit(Integer poiFootnotesLimit)
{
this.poiFootnotesLimit = poiFootnotesLimit;
}
/**
* MNT-11823: Upload of PPTX causes very high memory usage leading to system instability<br />
* <br />
*
* @param poiExtractPropertiesOnly - {@link Boolean} value which indicates that POI extractor must avoid building of the full document parts hierarchy and reading content of
* the parts
*/
public void setPoiExtractPropertiesOnly(Boolean poiExtractPropertiesOnly)
{
this.poiExtractPropertiesOnly = poiExtractPropertiesOnly;
}
public Boolean isPoiExtractPropertiesOnly()
{
return (poiExtractPropertiesOnly == null) ? (false) : (poiExtractPropertiesOnly);
}
/**
* MNT-11823: Upload of PPTX causes very high memory usage leading to system instability<br />
* <br />
*
* @param poiAllowableXslfRelationshipTypes - {@link Set}&lt;{@link String}&gt; instance which determines the list of allowable relationship types for traversing during
* analyzing of XSLF document
*/
public void setPoiAllowableXslfRelationshipTypes(Set<String> poiAllowableXslfRelationshipTypes)
{
this.poiAllowableXslfRelationshipTypes = poiAllowableXslfRelationshipTypes;
}
public Set<String> getPoiAllowableXslfRelationshipTypes()
{
return poiAllowableXslfRelationshipTypes;
}
/**
* MNT-11823: Upload of PPTX causes very high memory usage leading to system instability<br />
* <br />
* Initialization of {@link AlfrescoPoiPatchUtils} properties for {@link PoiMetadataExtracter#getExtractorContext()} context
*/
@Override
public void afterPropertiesSet() throws Exception
{
if (null == poiExtractPropertiesOnly)
{
poiExtractPropertiesOnly = false;
}
String context = getExtractorContext();
if (null != poiFootnotesLimit)
{
AlfrescoPoiPatchUtils.setPoiFootnotesLimit(context, poiFootnotesLimit);
}
AlfrescoPoiPatchUtils.setPoiExtractPropertiesOnly(context, poiExtractPropertiesOnly);
AlfrescoPoiPatchUtils.setPoiAllowableXslfRelationshipTypes(context, poiAllowableXslfRelationshipTypes);
}
}
Oops, something went wrong.

0 comments on commit 5de7c1d

Please sign in to comment.