Skip to content
Permalink
Browse files Browse the repository at this point in the history
OO-5549: fix the wiki import and add some unit tests
  • Loading branch information
lainsr committed Jun 22, 2021
1 parent b39c9ee commit 699490b
Show file tree
Hide file tree
Showing 13 changed files with 164 additions and 15 deletions.
14 changes: 11 additions & 3 deletions src/main/java/org/olat/core/util/PathUtils.java
Expand Up @@ -22,6 +22,7 @@
import java.io.File;
import java.io.IOException;
import java.nio.file.FileSystems;
import java.nio.file.FileVisitOption;
import java.nio.file.FileVisitResult;
import java.nio.file.FileVisitor;
import java.nio.file.Files;
Expand All @@ -32,9 +33,11 @@
import java.nio.file.SimpleFileVisitor;
import java.nio.file.StandardCopyOption;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.EnumSet;
import java.util.ServiceConfigurationError;

import org.apache.commons.io.IOUtils;
import org.olat.core.logging.OLATRuntimeException;

/**
*
Expand Down Expand Up @@ -62,7 +65,8 @@ public static boolean copyFileToDir(Path source, File targetDir, String path) th
}

/**
* Use the closeSubsequentFS method to close the file system.
* Use the closeSubsequentFS method to close the file system. The method doesn't
* follow sym. links and its depth is limited.
*
* @param file The file to visit
* @param filename The filename
Expand Down Expand Up @@ -90,14 +94,14 @@ public static Path visit(File file, String filename, FileVisitor<Path> visitor)
fPath = file.toPath();
}
if(fPath != null) {
Files.walkFileTree(fPath, visitor);
Files.walkFileTree(fPath, EnumSet.noneOf(FileVisitOption.class), 32, visitor);
}
return fPath;
}

public static void closeSubsequentFS(Path path) {
if(path != null && FileSystems.getDefault() != path.getFileSystem()) {
IOUtils.closeQuietly(path.getFileSystem());
IOUtils.closeQuietly(path.getFileSystem(), null);
}
}

Expand Down Expand Up @@ -125,6 +129,10 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
throws IOException {
Path relativeFile = source.relativize(file);
final Path destFile = Paths.get(destDir.toString(), relativeFile.toString());
Path normalizedPath = destFile.normalize();
if(!normalizedPath.startsWith(destDir)) {
throw new OLATRuntimeException("Invalid ZIP");
}
if(filter.matches(file)) {
Files.copy(file, destFile, StandardCopyOption.REPLACE_EXISTING);
}
Expand Down
8 changes: 5 additions & 3 deletions src/main/java/org/olat/fileresource/types/FileResource.java
Expand Up @@ -30,20 +30,22 @@
import java.nio.charset.Charset;
import java.nio.file.FileSystem;
import java.nio.file.FileSystems;
import java.nio.file.FileVisitOption;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.PathMatcher;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.nio.file.spi.FileSystemProvider;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.Map;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;

import org.olat.core.id.OLATResourceable;
import org.apache.logging.log4j.Logger;
import org.olat.core.id.OLATResourceable;
import org.olat.core.logging.Tracing;
import org.olat.core.util.CodeHelper;
import org.olat.core.util.PathUtils;
Expand Down Expand Up @@ -193,7 +195,7 @@ private static boolean isEncodingOk(File file, String encoding) {
protected static RootSearcher searchRootDirectory(Path fPath)
throws IOException {
RootSearcher rootSearcher = new RootSearcher();
Files.walkFileTree(fPath, rootSearcher);
Files.walkFileTree(fPath, EnumSet.noneOf(FileVisitOption.class), 16, rootSearcher);
return rootSearcher;
}

Expand All @@ -209,7 +211,7 @@ public static boolean copyResource(File file, String filename, File targetDirect
}

Path destDir = targetDirectory.toPath();
Files.walkFileTree(path, new CopyVisitor(path, destDir, filter));
Files.walkFileTree(path, EnumSet.noneOf(FileVisitOption.class), 24, new CopyVisitor(path, destDir, filter));
PathUtils.closeSubsequentFS(path);
return true;
} catch (IOException e) {
Expand Down
Expand Up @@ -27,11 +27,13 @@

import java.io.File;
import java.io.IOException;
import java.nio.file.FileVisitOption;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
Expand Down Expand Up @@ -75,7 +77,7 @@ public static ResourceEvaluation evaluate(File file, String filename) {
Path manifestPath = fPath.resolve(realManifestPath);

RootSearcher rootSearcher = new RootSearcher();
Files.walkFileTree(fPath, rootSearcher);
Files.walkFileTree(fPath, EnumSet.noneOf(FileVisitOption.class), 16, rootSearcher);
if(rootSearcher.foundRoot()) {
manifestPath = rootSearcher.getRoot().resolve(IMS_MANIFEST);
} else {
Expand Down
Expand Up @@ -29,11 +29,13 @@
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.nio.file.FileVisitOption;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.EnumSet;

import org.apache.logging.log4j.Logger;
import org.olat.core.logging.Tracing;
Expand Down Expand Up @@ -79,7 +81,7 @@ public static ResourceEvaluation evaluate(File file, String filename) {
Path manifestPath = fPath.resolve(realManifestPath);

RootSearcher rootSearcher = new RootSearcher();
Files.walkFileTree(fPath, rootSearcher);
Files.walkFileTree(fPath, EnumSet.noneOf(FileVisitOption.class), 16, rootSearcher);
if(rootSearcher.foundRoot()) {
manifestPath = rootSearcher.getRoot().resolve(IMS_MANIFEST);
} else {
Expand Down
Expand Up @@ -27,11 +27,13 @@

import java.io.File;
import java.io.IOException;
import java.nio.file.FileVisitOption;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
Expand Down Expand Up @@ -79,7 +81,7 @@ public static ResourceEvaluation evaluate(File file, String filename) {
Path manifestPath = fPath.resolve(realManifestPath);

RootSearcher rootSearcher = new RootSearcher();
Files.walkFileTree(fPath, rootSearcher);
Files.walkFileTree(fPath, EnumSet.noneOf(FileVisitOption.class), 16, rootSearcher);
if(rootSearcher.foundRoot()) {
manifestPath = rootSearcher.getRoot().resolve(IMS_MANIFEST);
} else {
Expand Down
20 changes: 14 additions & 6 deletions src/main/java/org/olat/modules/wiki/WikiManager.java
Expand Up @@ -30,13 +30,15 @@
import java.io.InputStream;
import java.io.OutputStream;
import java.io.UnsupportedEncodingException;
import java.nio.file.FileVisitOption;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.StandardCopyOption;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.EnumSet;
import java.util.Iterator;
import java.util.List;
import java.util.Properties;
Expand Down Expand Up @@ -162,7 +164,7 @@ public boolean importWiki(File file, String filename, File targetDirectory) {
}

Path destDir = targetDirectory.toPath();
Files.walkFileTree(path, new ImportVisitor(destDir));
Files.walkFileTree(path, EnumSet.noneOf(FileVisitOption.class), 16, new ImportVisitor(destDir));
PathUtils.closeSubsequentFS(path);
return true;
} catch (IOException e) {
Expand Down Expand Up @@ -299,26 +301,32 @@ public ImportVisitor(Path destDir) throws IOException {
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
throws IOException {
String filename = file.getFileName().toString();
Path normalizedPath = file.normalize();
if(!normalizedPath.startsWith(destDir)) {
throw new IOException("Invalid ZIP");
}


if(filename.endsWith(WikiManager.WIKI_PROPERTIES_SUFFIX)) {
String f = convertAlternativeFilename(file.toString());
final Path destFile = Paths.get(wikiDir.toString(), f);
checkDestinationFile(destFile);
resetAndCopyProperties(file, destFile);
} else if (filename.endsWith(WIKI_FILE_SUFFIX)) {
String f = convertAlternativeFilename(file.toString());
final Path destFile = Paths.get(wikiDir.toString(), f);
checkDestinationFile(destFile);
Files.copy(file, destFile, StandardCopyOption.REPLACE_EXISTING);
} else if (!filename.contains(WIKI_FILE_SUFFIX + "-")
&& !filename.contains(WIKI_PROPERTIES_SUFFIX + "-")) {
final Path destFile = Paths.get(mediaDir.toString(), file.toString());
checkDestinationFile(destFile);
Files.copy(file, destFile, StandardCopyOption.REPLACE_EXISTING);
}
return FileVisitResult.CONTINUE;
}

private void checkDestinationFile(Path destFile) throws IOException {
Path normalizedPath = destFile.normalize();
if(!normalizedPath.startsWith(destDir)) {
throw new OLATRuntimeException("Invalid ZIP");
}
}

@Override
public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs)
Expand Down
61 changes: 61 additions & 0 deletions src/test/java/org/olat/modules/ims/cp/ImsCPHandlerTest.java
@@ -0,0 +1,61 @@
package org.olat.modules.ims.cp;

import java.io.File;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.Locale;

import org.junit.Assert;
import org.junit.Test;
import org.olat.core.id.Identity;
import org.olat.core.logging.OLATRuntimeException;
import org.olat.fileresource.FileResourceManager;
import org.olat.fileresource.types.ImsCPFileResource;
import org.olat.repository.RepositoryEntry;
import org.olat.repository.handlers.RepositoryHandler;
import org.olat.repository.handlers.RepositoryHandlerFactory;
import org.olat.test.JunitTestHelper;
import org.olat.test.OlatTestCase;
import org.springframework.beans.factory.annotation.Autowired;

/**
*
* Initial date: 22 juin 2021<br>
* @author srosse, stephane.rosse@frentix.com, http://www.frentix.com
*
*/
public class ImsCPHandlerTest extends OlatTestCase {

@Autowired
private RepositoryHandlerFactory handlerFactory;

@Test
public void importImsCP() throws URISyntaxException {
Identity author = JunitTestHelper.createAndPersistIdentityAsRndAuthor("ims-cp-1");
URL imsCpUrl = ImsCPHandlerTest.class.getResource("imscp.zip");
File imsCpFile = new File(imsCpUrl.toURI());

RepositoryHandler cpHandler = handlerFactory.getRepositoryHandler(ImsCPFileResource.TYPE_NAME);
RepositoryEntry entry = cpHandler.importResource(author, null, "IMS CP", null, false, null, Locale.ENGLISH, imsCpFile, imsCpFile.getName());
Assert.assertNotNull(entry);

File cpRoot = FileResourceManager.getInstance().unzipFileResource(entry.getOlatResource());
File image = new File(cpRoot, "IMG_1482.jpg");
Assert.assertTrue(image.exists());
File manifestXml = new File(cpRoot, "imsmanifest.xml");
Assert.assertTrue(manifestXml.exists());
File page = new File(cpRoot, "new.html");
Assert.assertTrue(page.exists());
}

@Test(expected=OLATRuntimeException.class)
public void importImsCPSlide() throws URISyntaxException {
Identity author = JunitTestHelper.createAndPersistIdentityAsRndAuthor("ims-cp-1");
URL imsCpUrl = ImsCPHandlerTest.class.getResource("imscp_alt.zip");
File imsCpFile = new File(imsCpUrl.toURI());

RepositoryHandler cpHandler = handlerFactory.getRepositoryHandler(ImsCPFileResource.TYPE_NAME);
cpHandler.importResource(author, null, "IMS CP", null, false, null, Locale.ENGLISH, imsCpFile, imsCpFile.getName());
}

}
Binary file added src/test/java/org/olat/modules/ims/cp/imscp.zip
Binary file not shown.
Binary file not shown.
62 changes: 62 additions & 0 deletions src/test/java/org/olat/modules/wiki/WikiManagerTest.java
@@ -0,0 +1,62 @@
package org.olat.modules.wiki;

import java.io.File;
import java.net.URISyntaxException;
import java.net.URL;

import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.olat.core.logging.OLATRuntimeException;
import org.olat.core.util.CodeHelper;
import org.olat.core.util.FileUtils;
import org.olat.core.util.WebappHelper;
import org.olat.test.OlatTestCase;
import org.springframework.beans.factory.annotation.Autowired;
import org.wildfly.common.Assert;

/**
*
* Initial date: 22 juin 2021<br>
* @author srosse, stephane.rosse@frentix.com, http://www.frentix.com
*
*/
public class WikiManagerTest extends OlatTestCase {

@Autowired
private WikiManager wikiManager;

private File tmpWikiDir;

@Before
public void createTmpDir() {
tmpWikiDir = new File(WebappHelper.getTmpDir(), "wiki" + CodeHelper.getForeverUniqueID());
}

@After
public void deleteTmpDir() {
FileUtils.deleteDirsAndFiles(tmpWikiDir, true, true);
}

@Test
public void importWiki() throws URISyntaxException {
URL wikiUrl = WikiManagerTest.class.getResource("wiki.zip");
File wikiFile = new File(wikiUrl.toURI());
wikiManager.importWiki(wikiFile, null, tmpWikiDir);

File image = new File(tmpWikiDir, "media/IMG_1482.jpg");
Assert.assertTrue(image.exists());
File imageMetadata = new File(tmpWikiDir, "media/IMG_1482.jpg.metadata");
Assert.assertTrue(imageMetadata.exists());
File indexPage = new File(tmpWikiDir, "wiki/SW5kZXg=.wp");
Assert.assertTrue(indexPage.exists());
}

@Test(expected=OLATRuntimeException.class)
public void importWikiSlide() throws URISyntaxException {
URL wikiUrl = WikiManagerTest.class.getResource("wiki_alt.zip");
File wikiFile = new File(wikiUrl.toURI());
wikiManager.importWiki(wikiFile, null, tmpWikiDir);
}

}
Binary file added src/test/java/org/olat/modules/wiki/wiki.zip
Binary file not shown.
Binary file added src/test/java/org/olat/modules/wiki/wiki_alt.zip
Binary file not shown.
2 changes: 2 additions & 0 deletions src/test/java/org/olat/test/AllTestsJunit4.java
Expand Up @@ -236,6 +236,7 @@
org.olat.modules.iq.IQManagerTest.class,
org.olat.modules.fo.ForumManagerTest.class,//fail
org.olat.modules.wiki.WikiUnitTest.class,
org.olat.modules.wiki.WikiManagerTest.class,
org.olat.modules.wiki.versioning.diff.CookbookDiffTest.class,
org.olat.modules.wiki.gui.components.wikiToHtml.FilterUtilTest.class,
org.olat.modules.coach.manager.CoachingDAOTest.class,
Expand Down Expand Up @@ -359,6 +360,7 @@
org.olat.course.nodes.projectbroker.ProjectBrokerManagerTest.class,
org.olat.core.commons.persistence.DBTest.class,
org.olat.modules.ims.cp.CPManagerTest.class,
org.olat.modules.ims.cp.ImsCPHandlerTest.class,
org.olat.modules.ims.qti.fileresource.FileResourceValidatorTest.class,
org.olat.ims.qti.QTIResultManagerTest.class,
org.olat.ims.qti.qpool.QTIImportProcessorTest.class,
Expand Down

0 comments on commit 699490b

Please sign in to comment.