Skip to content

Commit

Permalink
InitialDataImport total overhaul, uses resources, not ZipFS/files
Browse files Browse the repository at this point in the history 
This works for JAR, WAR executable, deployed WAR and IDEA the same way.
  • Loading branch information
@virgo47
virgo47 committed Oct 4, 2020
1 parent bfbb628 commit 3f25d21
Showing 1 changed file with 95 additions and 236 deletions.
331 changes: 95 additions & 236 deletions gui/admin-gui/src/main/java/com/evolveum/midpoint/init/InitialDataImport.java
View file
@@ -1,12 +1,25 @@
/*
* Copyright (c) 2010-2013 Evolveum and contributors
* Copyright (C) 2010-2020 Evolveum and contributors
*
* This work is dual-licensed under the Apache License 2.0
* and European Union Public License. See LICENSE file for details.
*/

package com.evolveum.midpoint.init;

import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Comparator;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;

import org.apache.commons.io.IOUtils;
import org.springframework.core.io.Resource;
import org.springframework.core.io.support.PathMatchingResourcePatternResolver;
import org.springframework.security.core.context.SecurityContext;

import com.evolveum.midpoint.model.api.ModelExecuteOptions;
import com.evolveum.midpoint.prism.PrismObject;
import com.evolveum.midpoint.prism.delta.DeltaFactory;
Expand All @@ -25,283 +38,129 @@
import com.evolveum.midpoint.util.logging.TraceManager;
import com.evolveum.midpoint.xml.ns._public.common.common_3.ObjectType;
import com.evolveum.midpoint.xml.ns._public.common.common_3.ReportType;
import org.apache.commons.io.FileUtils;
import org.springframework.security.core.context.SecurityContext;

import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.file.*;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.*;

/**
* @author lazyman
* Imports initial data objects as needed, ignoring already imported objects.
* Initial objects are found on the classpath using defined pattern to match
* resources under {@code initial-objects} directory.
*/
public class InitialDataImport extends DataImport {

private static final Trace LOGGER = TraceManager.getTrace(InitialDataImport.class);

private static final String INITIAL_OBJECTS_RESOURCE_PATTERN = "classpath*:/initial-objects/*";

public void init() throws SchemaException {
LOGGER.info("Starting initial object import (if necessary).");

OperationResult mainResult = new OperationResult(OPERATION_INITIAL_OBJECTS_IMPORT);
Task task = taskManager.createTaskInstance(OPERATION_INITIAL_OBJECTS_IMPORT);
task.setChannel(SchemaConstants.CHANNEL_INIT_URI);

int count = 0;
int errors = 0;

File[] files = getInitialImportObjects();
LOGGER.debug("Files to be imported: {}.", Arrays.toString(files));

SecurityContext securityContext = provideFakeSecurityContext();

for (File file : files) {
try {
LOGGER.debug("Considering initial import of file {}.", file.getName());
PrismObject object = prismContext.parseObject(file);
if (ReportType.class.equals(object.getCompileTimeClass())) {
ReportTypeUtil.applyDefinition(object, prismContext);
}

Boolean importObject = importObject(object, file, task, mainResult);
if (importObject == null) {
continue;
}
if (importObject) {
count++;
} else {
errors++;
}
} catch (Exception ex) {
LoggingUtils.logUnexpectedException(LOGGER, "Couldn't import file {}", ex, file.getName());
mainResult.recordFatalError("Couldn't import file '" + file.getName() + "'", ex);
}
}

securityContext.setAuthentication(null);

Map<ImportResult, AtomicInteger> importStats = new LinkedHashMap<>();
importStats.put(ImportResult.IMPORTED, new AtomicInteger());
importStats.put(ImportResult.ERROR, new AtomicInteger());
importStats.put(ImportResult.SKIPPED, new AtomicInteger());
try {
cleanup();
} catch (IOException ex) {
LOGGER.error("Couldn't cleanup tmp folder with imported files", ex);
Resource[] resources = new PathMatchingResourcePatternResolver()
.getResources(INITIAL_OBJECTS_RESOURCE_PATTERN);
Arrays.sort(resources, Comparator.comparing(Resource::getFilename));

SecurityContext securityContext = provideFakeSecurityContext();
for (Resource resource : resources) {
ImportResult result = importInitialObjectsResource(resource, task, mainResult);
importStats.get(result).incrementAndGet();
}
securityContext.setAuthentication(null);
} catch (IOException e) {
LoggingUtils.logUnexpectedException(LOGGER, "Couldn't list initial-objects resources", e);
mainResult.recordFatalError("Couldn't list initial-objects resources", e);
}

mainResult.recomputeStatus("Couldn't import objects.");

LOGGER.info("Initial object import finished ({} objects imported, {} errors)", count, errors);
LOGGER.info("Initial object import finished ({} objects imported, {} errors, {} skipped)",
importStats.get(ImportResult.IMPORTED),
importStats.get(ImportResult.ERROR),
importStats.get(ImportResult.SKIPPED));
if (LOGGER.isTraceEnabled()) {
LOGGER.trace("Initialization status:\n" + mainResult.debugDump());
}
}

/**
* @param object
* @param task
* @param mainResult
* @return null if nothing was imported, true if it was success, otherwise false
*/
private <O extends ObjectType> Boolean importObject(PrismObject<O> object, File file, Task task, OperationResult mainResult) {
private ImportResult importInitialObjectsResource(
Resource resource, Task task, OperationResult mainResult) {

try {
LOGGER.debug("Considering initial import of file {}.", resource.getFilename());
PrismObject<? extends ObjectType> object;
try (InputStream resourceInputStream = resource.getInputStream()) {
String objectText = IOUtils.toString(resourceInputStream, StandardCharsets.UTF_8);
object = prismContext.parseObject(objectText);
}
if (ReportType.class.equals(object.getCompileTimeClass())) {
//noinspection unchecked
ReportTypeUtil.applyDefinition((PrismObject<ReportType>) object, prismContext);
}

return importObject(object, resource.getFilename(), task, mainResult);
} catch (Exception ex) {
LoggingUtils.logUnexpectedException(LOGGER,
"Couldn't import file {}", ex, resource.getFilename());
mainResult.recordFatalError(
"Couldn't import file '" + resource.getFilename() + "'", ex);
return ImportResult.ERROR;
}
}

private ImportResult importObject(PrismObject<? extends ObjectType> object,
String fileName, Task task, OperationResult mainResult) {
OperationResult result = mainResult.createSubresult(OPERATION_IMPORT_OBJECT);

boolean importObject = true;
try {
model.getObject(object.getCompileTimeClass(), object.getOid(), SelectorOptions.createCollection(GetOperationOptions.createAllowNotFound()), task, result);
importObject = false;
// returns not-null or throws, we don't care about the returned object
model.getObject(
object.getCompileTimeClass(),
object.getOid(),
SelectorOptions.createCollection(GetOperationOptions.createAllowNotFound()),
task,
result);
result.recordSuccess();
return ImportResult.SKIPPED;
} catch (ObjectNotFoundException ex) {
importObject = true;
// this is OK, we're going to import missing object
} catch (Exception ex) {
if (!importObject){
LoggingUtils.logUnexpectedException(LOGGER, "Couldn't get object with oid {} from model", ex,
object.getOid());
result.recordWarning("Couldn't get object with oid '" + object.getOid() + "' from model",
ex);
}
}

if (!importObject) {
return null;
// unexpected, but we'll try to import the object and see what happens
LoggingUtils.logUnexpectedException(LOGGER,
"Couldn't get object with oid {} from model", ex, object.getOid());
result.recordWarning(
"Couldn't get object with oid '" + object.getOid() + "' from model", ex);
}

preImportUpdate(object);

ObjectDelta delta = DeltaFactory.Object.createAddDelta(object);
ObjectDelta<? extends ObjectType> delta = DeltaFactory.Object.createAddDelta(object);
try {
LOGGER.info("Starting initial import of file {}.", file.getName());
model.executeChanges(MiscUtil.createCollection(delta), ModelExecuteOptions.create(prismContext).setIsImport(), task, result);
LOGGER.info("Starting initial import of file {}.", fileName);
model.executeChanges(
MiscUtil.createCollection(delta),
ModelExecuteOptions.create(prismContext).setIsImport(),
task,
result);
result.recordSuccess();
LOGGER.info("Created {} as part of initial import", object);
return true;
return ImportResult.IMPORTED;
} catch (Exception e) {
LoggingUtils.logUnexpectedException(LOGGER, "Couldn't import {} from file {}: ", e, object,
file.getName(), e.getMessage());
LoggingUtils.logUnexpectedException(LOGGER, "Couldn't import {} from file {}: ",
e, object, fileName, e.getMessage());
result.recordFatalError(e);

LOGGER.info("\n" + result.debugDump());
return false;
}
}

private File getResource(String name) {
URI path;
try {
LOGGER.trace("getResource: name = {}", name);
path = InitialDataImport.class.getClassLoader().getResource(name).toURI();
LOGGER.trace("getResource: path = {}", path);
//String updatedPath = path.toString().replace("zip:/", "jar:/");
//LOGGER.trace("getResource: path updated = {}", updatedPath);
//path = new URI(updatedPath);
} catch (URISyntaxException e) {
throw new IllegalArgumentException("parameter name = " + name, e);
}
return new File(path);
}

private void cleanup() throws IOException {
Path destDir = Paths.get(configuration.getMidpointHome(), "tmp/initial-objects");
FileUtils.deleteDirectory(destDir.toFile());
}

/**
* @param path jar:file:/<ABSOLUTE_PATH>/midpoint.war!/WEB-INF/classes!/initial-objects
* @param tmpDir
*/
private void copyInitialImportObjectsFromFatJar(URL path, File tmpDir) throws IOException, URISyntaxException {
String warPath = path.toString().split("!/")[0];

URI src = URI.create(warPath);

Map<String, String> env = new HashMap<>();
env.put("create", "false");
try (FileSystem zipfs = FileSystems.newFileSystem(src, env)) {
Path pathInZipfile = zipfs.getPath("/WEB-INF/classes/initial-objects");
final Path destDir = Paths.get(configuration.getMidpointHome(), "tmp");

Files.walkFileTree(pathInZipfile, new SimpleFileVisitor<Path>() {

@Override
public FileVisitResult visitFile(Path file,
BasicFileAttributes attrs) throws IOException {
String f = file.subpath(2, file.getNameCount()).toString(); // strip /WEB-INF/classes
final Path destFile = Paths.get(destDir.toString(), f);
LOGGER.trace("Extracting file {} to {}", file, destFile);
Files.copy(file, destFile, StandardCopyOption.REPLACE_EXISTING);
return FileVisitResult.CONTINUE;
}

@Override
public FileVisitResult preVisitDirectory(Path dir,
BasicFileAttributes attrs) throws IOException {
String folder = dir.subpath(2, dir.getNameCount()).toString(); // strip /WEB-INF/classes
final Path dirToCreate = Paths.get(destDir.toString(), folder);
if (Files.notExists(dirToCreate)) {
LOGGER.trace("Creating directory {}", dirToCreate);
Files.createDirectory(dirToCreate);
}
return FileVisitResult.CONTINUE;
}
});
}
}

/**
* file:/<ABSOLUTE_PATH>/midpoint/WEB-INF/classes/initial-objects/
*/
private void copyInitialImportObjectsFromJar() throws IOException, URISyntaxException {
URI src = InitialDataImport.class.getProtectionDomain().getCodeSource().getLocation().toURI();
LOGGER.trace("InitialDataImport code location: {}", src);
Map<String, String> env = new HashMap<>();
env.put("create", "false");
URI normalizedSrc = new URI(src.toString().replaceFirst("file:", "jar:file:"));
LOGGER.trace("InitialDataImport normalized code location: {}", normalizedSrc);
try (FileSystem zipfs = FileSystems.newFileSystem(normalizedSrc, env)) {
Path pathInZipfile = zipfs.getPath("/initial-objects");
final Path destDir = Paths.get(configuration.getMidpointHome(), "tmp");
Files.walkFileTree(pathInZipfile, new SimpleFileVisitor<Path>() {

@Override
public FileVisitResult visitFile(Path file,
BasicFileAttributes attrs) throws IOException {
final Path destFile = Paths.get(destDir.toString(), file.toString());
LOGGER.trace("Extracting file {} to {}", file, destFile);
Files.copy(file, destFile, StandardCopyOption.REPLACE_EXISTING);
return FileVisitResult.CONTINUE;
}

@Override
public FileVisitResult preVisitDirectory(Path dir,
BasicFileAttributes attrs) throws IOException {
final Path dirToCreate = Paths.get(destDir.toString(), dir.toString());
if (Files.notExists(dirToCreate)) {
LOGGER.trace("Creating directory {}", dirToCreate);
Files.createDirectory(dirToCreate);
}
return FileVisitResult.CONTINUE;
}
});
return ImportResult.ERROR;
}
}

private File setupInitialObjectsTmpFolder() {
File tmpDir = new File(configuration.getMidpointHome(), "tmp");
if (!tmpDir.mkdir()) {
LOGGER.warn("Failed to create temporary directory for initial objects {}. Maybe it already exists",
configuration.getMidpointHome()+"/tmp");
}

tmpDir = new File(configuration.getMidpointHome()+"/tmp/initial-objects");
if (!tmpDir.mkdir()) {
LOGGER.warn("Failed to create temporary directory for initial objects {}. Maybe it already exists",
configuration.getMidpointHome()+"/tmp/initial-objects");
}

return tmpDir;
}

protected File[] getInitialImportObjects() {
URL path = InitialDataImport.class.getClassLoader().getResource("initial-objects");
String resourceType = path.getProtocol();

File folder = null;

try {
if (path.toString().split("!/").length == 3) {
File tmpDir = setupInitialObjectsTmpFolder();

copyInitialImportObjectsFromFatJar(path, tmpDir);

folder = tmpDir;
} else if ("zip".equals(resourceType) || "jar".equals(resourceType)) {

File tmpDir = setupInitialObjectsTmpFolder();
copyInitialImportObjectsFromJar();

folder = tmpDir;
}
} catch (IOException ex) {
throw new RuntimeException("Failed to copy initial objects file out of the archive to the temporary directory", ex);
} catch (URISyntaxException ex) {
throw new RuntimeException("Failed get URI for the source code bundled with initial objects", ex);
}

if ("file".equals(resourceType)) {
folder = getResource("initial-objects");
}

File[] files = folder.listFiles(pathname -> {
if (pathname.isDirectory()) {
return false;
}

return true;
});

sortFiles(files);

return files;
private enum ImportResult {
SKIPPED, IMPORTED, ERROR
}
}

0 comments on commit 3f25d21

Please sign in to comment.