BackupServiceIntegrationTest.java

/*
 * Copyright 2017 ThoughtWorks, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.thoughtworks.go.server.service;

import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Semaphore;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
import javax.sql.DataSource;

import com.thoughtworks.go.config.CaseInsensitiveString;
import com.thoughtworks.go.config.GoConfigDao;
import com.thoughtworks.go.config.GoMailSender;
import com.thoughtworks.go.config.ServerConfig;
import com.thoughtworks.go.config.materials.SubprocessExecutionContext;
import com.thoughtworks.go.config.materials.git.GitMaterial;
import com.thoughtworks.go.database.Database;
import com.thoughtworks.go.domain.materials.Modification;
import com.thoughtworks.go.domain.materials.mercurial.StringRevision;
import com.thoughtworks.go.domain.materials.RevisionContext;
import com.thoughtworks.go.i18n.Localizable;
import com.thoughtworks.go.i18n.Localizer;
import com.thoughtworks.go.security.CipherProvider;
import com.thoughtworks.go.server.dao.DatabaseAccessHelper;
import com.thoughtworks.go.server.domain.Username;
import com.thoughtworks.go.server.messaging.SendEmailMessage;
import com.thoughtworks.go.server.persistence.ServerBackupRepository;
import com.thoughtworks.go.server.service.result.HttpLocalizedOperationResult;
import com.thoughtworks.go.server.util.ServerVersion;
import com.thoughtworks.go.service.ConfigRepository;
import com.thoughtworks.go.util.*;
import com.thoughtworks.go.util.command.InMemoryStreamConsumer;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.io.filefilter.NameFileFilter;
import org.apache.commons.io.filefilter.TrueFileFilter;
import org.bouncycastle.crypto.InvalidCipherTextException;
import org.joda.time.DateTime;
import org.joda.time.format.DateTimeFormatter;
import org.joda.time.format.ISODateTimeFormat;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;

import static org.hamcrest.Matchers.greaterThan;
import static org.hamcrest.core.Is.is;
import static org.hamcrest.core.IsNull.nullValue;
import static org.junit.Assert.assertThat;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoMoreInteractions;
import static org.mockito.Mockito.when;

@RunWith(SpringJUnit4ClassRunner.class)
@ContextConfiguration(locations = {
        "classpath:WEB-INF/applicationContext-global.xml",
        "classpath:WEB-INF/applicationContext-dataLocalAccess.xml",
        "classpath:WEB-INF/applicationContext-acegi-security.xml"
})
public class BackupServiceIntegrationTest {

    @Autowired BackupService backupService;
    @Autowired
    GoConfigService goConfigService;
    @Autowired DataSource dataSource;
    @Autowired ArtifactsDirHolder artifactsDirHolder;
    @Autowired DatabaseAccessHelper dbHelper;
    @Autowired
    GoConfigDao goConfigDao;
    @Autowired
    ServerBackupRepository backupInfoRepository;
    @Autowired TimeProvider timeProvider;
    @Autowired Localizer localizer;
    @Autowired SystemEnvironment systemEnvironment;
    @Autowired ServerVersion serverVersion;
    @Autowired ConfigRepository configRepository;
    @Autowired private SubprocessExecutionContext subprocessExecutionContext;
    @Autowired Database databaseStrategy;

    private GoConfigFileHelper configHelper = new GoConfigFileHelper();

    private File backupsDirectory;
    private TempFiles tempFiles;
    private byte[] originalCipher;
    private Username admin;

    @Before
    public void setUp() throws Exception {
        configHelper.onSetUp();
        dbHelper.onSetUp();
        admin = new Username(new CaseInsensitiveString("admin"));
        configHelper.enableSecurity();
        configHelper.addAdmins(CaseInsensitiveString.str(admin.getUsername()));
        goConfigDao.forceReload();
        backupsDirectory = new File(artifactsDirHolder.getArtifactsDir(), ServerConfig.SERVER_BACKUPS);
        cleanupBackups();
        tempFiles = new TempFiles();
        originalCipher = new CipherProvider(systemEnvironment).getKey();

        FileUtil.writeContentToFile("invalid crapy config", new File(systemEnvironment.getConfigDir(), "cruise-config.xml"));
        FileUtil.writeContentToFile("invalid crapy cipher", new File(systemEnvironment.getConfigDir(), "cipher"));
    }

    @After
    public void tearDown() throws Exception {
        tempFiles.cleanUp();
        dbHelper.onTearDown();
        cleanupBackups();
        FileUtil.writeContentToFile(goConfigService.xml(), new File(systemEnvironment.getConfigDir(), "cruise-config.xml"));
        FileUtil.writeContentToFile(originalCipher, systemEnvironment.getCipherFile());
        configHelper.onTearDown();
    }

    @Test
    public void shouldFailIfUserIsNotAnAdmin() {
        HttpLocalizedOperationResult result = new HttpLocalizedOperationResult();
        backupService.startBackup(new Username(new CaseInsensitiveString("loser")), result);
        assertThat(result.isSuccessful(), is(false));
        assertThat(result.message(localizer), is("Unauthorized to initiate Go backup as you are not a Go administrator"));
    }

    @Test
    public void shouldPerformConfigBackupForAllConfigFiles() throws Exception {
        try {
            HttpLocalizedOperationResult result = new HttpLocalizedOperationResult();

            createConfigFile("foo", "foo_foo");
            createConfigFile("bar", "bar_bar");
            createConfigFile("baz", "hazar_bar");
            createConfigFile("hello/world/file", "hello world!");
            createConfigFile("some_dir/cruise-config.xml", "some-other-cruise-config");
            createConfigFile("some_dir/cipher", "some-cipher");

            backupService.startBackup(admin, result);
            assertThat(result.isSuccessful(), is(true));
            assertThat(result.message(localizer), is("Backup completed successfully."));

            File configZip = backedUpFile("config-dir.zip");
            assertThat(fileContents(configZip, "foo"), is("foo_foo"));
            assertThat(fileContents(configZip, "bar"), is("bar_bar"));
            assertThat(fileContents(configZip, "baz"), is("hazar_bar"));

            assertThat(fileContents(configZip, FilenameUtils.separatorsToSystem("hello/world/file")), is("hello world!"));
            assertThat(fileContents(configZip, FilenameUtils.separatorsToSystem("some_dir/cruise-config.xml")), is("some-other-cruise-config"));
            assertThat(fileContents(configZip, FilenameUtils.separatorsToSystem("some_dir/cipher")), is("some-cipher"));

            assertThat(fileContents(configZip, "cruise-config.xml"), is(goConfigService.xml()));
            byte[] realCipher = (byte[]) ReflectionUtil.invoke(new CipherProvider(systemEnvironment), "getKey");
            assertThat(fileContents(configZip, "cipher").getBytes(), is(realCipher));
        } finally {
            deleteConfigFileIfExists("foo", "bar", "baz", "hello", "some_dir");
        }
    }

    @Test
    public void shouldBackupConfigRepository() throws IOException {
        configHelper.addPipeline("too-unique-to-be-present", "stage-name");

        HttpLocalizedOperationResult result = new HttpLocalizedOperationResult();

        backupService.startBackup(admin, result);
        assertThat(result.isSuccessful(), is(true));
        assertThat(result.message(localizer), is("Backup completed successfully."));

        File repoZip = backedUpFile("config-repo.zip");
        File repoDir = tempFiles.createUniqueFolder("expanded-config-repo-backup");
        TestUtils.extractZipToDir(repoZip, repoDir);
        File cloneDir = tempFiles.createUniqueFolder("cloned-config-repo-backup");
        GitMaterial git = new GitMaterial(repoDir.getAbsolutePath());

        List<Modification> modifications = git.latestModification(cloneDir, subprocessExecutionContext);
        String latestChangeRev = modifications.get(0).getRevision();
        git.checkout(cloneDir, new StringRevision(latestChangeRev), subprocessExecutionContext);
        assertThat(FileUtil.readContentFromFile(new File(cloneDir, "cruise-config.xml")).indexOf("too-unique-to-be-present"), greaterThan(0));
        StringRevision revision = new StringRevision(latestChangeRev + "~1");
        git.updateTo(new InMemoryStreamConsumer(), cloneDir, new RevisionContext(revision), subprocessExecutionContext);
        assertThat(FileUtil.readContentFromFile(new File(cloneDir, "cruise-config.xml")).indexOf("too-unique-to-be-present"), is(-1));
    }

    @Test
    public void shouldCaptureVersionForEveryBackup() throws IOException {
        HttpLocalizedOperationResult result = new HttpLocalizedOperationResult();
        ServerVersion serverVersion = mock(ServerVersion.class);
        when(serverVersion.version()).thenReturn("some-test-version-007");
        BackupService backupService = new BackupService(dataSource, artifactsDirHolder, goConfigService, timeProvider, backupInfoRepository, systemEnvironment, serverVersion, configRepository, databaseStrategy);
        backupService.initialize();
        backupService.startBackup(admin, result);
        assertThat(result.isSuccessful(), is(true));
        assertThat(result.message(localizer), is("Backup completed successfully."));
        File version = backedUpFile("version.txt");
        assertThat(FileUtil.readContentFromFile(version), is("some-test-version-007"));
    }

    @Test
    public void shouldSendEmailToAdminAfterTakingBackup() throws InvalidCipherTextException {
        GoConfigService configService = mock(GoConfigService.class);
        GoMailSender goMailSender = mock(GoMailSender.class);
        when(configService.getMailSender()).thenReturn(goMailSender);
        when(configService.adminEmail()).thenReturn("mail@admin.com");
        when(configService.isUserAdmin(admin)).thenReturn(true);

        TimeProvider timeProvider = mock(TimeProvider.class);
        DateTime now = new DateTime();
        when(timeProvider.currentDateTime()).thenReturn(now);

        BackupService service = new BackupService(dataSource, artifactsDirHolder, configService, timeProvider, backupInfoRepository, systemEnvironment, serverVersion, configRepository,
                databaseStrategy);
        service.initialize();
        service.startBackup(admin, new HttpLocalizedOperationResult());

        String ipAddress = SystemUtil.getFirstLocalNonLoopbackIpAddress();
        String body = String.format("Backup of the Go server at '%s' was successfully completed. The backup is stored at location: %s. This backup was triggered by 'admin'.", ipAddress, backupDir(now).getAbsolutePath());

        verify(goMailSender).send(new SendEmailMessage("Server Backup Completed Successfully", body, "mail@admin.com"));
        verifyNoMoreInteractions(goMailSender);
    }

    @Test
    public void shouldSendEmailToAdminWhenTheBackupFails() throws Exception {
        GoConfigService configService = mock(GoConfigService.class);
        when(configService.adminEmail()).thenReturn("mail@admin.com");

        GoMailSender goMailSender = mock(GoMailSender.class);
        when(configService.getMailSender()).thenReturn(goMailSender);
        when(configService.isUserAdmin(admin)).thenReturn(true);

        DateTime now = new DateTime();
        TimeProvider timeProvider = mock(TimeProvider.class);
        when(timeProvider.currentDateTime()).thenReturn(now);

        HttpLocalizedOperationResult result = new HttpLocalizedOperationResult();

        Database databaseStrategyMock = mock(Database.class);
        doThrow(new RuntimeException("Oh no!")).when(databaseStrategyMock).backup(any(File.class));
        BackupService service = new BackupService(dataSource, artifactsDirHolder, configService, timeProvider, backupInfoRepository, systemEnvironment, serverVersion, configRepository,
                databaseStrategyMock);
        service.initialize();
        service.startBackup(admin, result);

        String ipAddress = SystemUtil.getFirstLocalNonLoopbackIpAddress();
        String body = String.format("Backup of the Go server at '%s' has failed. The reason is: %s", ipAddress, "Oh no!");

        assertThat(result.isSuccessful(), is(false));
        assertThat(result.message(localizer), is("Failed to perform backup. Reason: Oh no!"));
        verify(goMailSender).send(new SendEmailMessage("Server Backup Failed", body, "mail@admin.com"));
        verifyNoMoreInteractions(goMailSender);

        assertThat(FileUtils.listFiles(backupsDirectory, TrueFileFilter.TRUE, TrueFileFilter.TRUE).isEmpty(), is(true));
    }

    @Test
    public void shouldReturnBackupRunningSinceValue_inISO8601_format() throws InterruptedException {
        assertThat(backupService.backupRunningSinceISO8601(), is(nullValue()));

        final Semaphore waitForBackupToStart = new Semaphore(1);
        final Semaphore waitForAssertionToCompleteWhileBackupIsOn = new Semaphore(1);
        final HttpLocalizedOperationResult result = new HttpLocalizedOperationResult() {
            @Override public void setMessage(Localizable message) {
                waitForBackupToStart.release();
                super.setMessage(message);
                try {
                    waitForAssertionToCompleteWhileBackupIsOn.acquire();
                } catch (InterruptedException e) {
                    throw new RuntimeException(e);
                }

            }
        };
        waitForAssertionToCompleteWhileBackupIsOn.acquire();
        waitForBackupToStart.acquire();
        Thread backupThd = new Thread(new Runnable() {
            public void run() {
                backupService.startBackup(admin, result);
            }
        });

        backupThd.start();
        waitForBackupToStart.acquire();
        String backupStartedTimeString = backupService.backupRunningSinceISO8601();
        DateTimeFormatter dateTimeFormatter = ISODateTimeFormat.dateTime();
        DateTime dateTime = dateTimeFormatter.parseDateTime(backupStartedTimeString);
        assertThat(ReflectionUtil.getField(backupService, "backupRunningSince"), is(dateTime));
        waitForAssertionToCompleteWhileBackupIsOn.release();
        backupThd.join();
    }

     @Test
    public void shouldReturnBackupStartedBy() throws InterruptedException {
        assertThat(backupService.backupStartedBy(), is(nullValue()));

        final Semaphore waitForBackupToStart = new Semaphore(1);
        final Semaphore waitForAssertionToCompleteWhileBackupIsOn = new Semaphore(1);
        final HttpLocalizedOperationResult result = new HttpLocalizedOperationResult() {
            @Override public void setMessage(Localizable message) {
                waitForBackupToStart.release();
                super.setMessage(message);
                try {
                    waitForAssertionToCompleteWhileBackupIsOn.acquire();
                } catch (InterruptedException e) {
                    throw new RuntimeException(e);
                }

            }
        };
        waitForAssertionToCompleteWhileBackupIsOn.acquire();
        waitForBackupToStart.acquire();
        Thread backupThd = new Thread(new Runnable() {
            public void run() {
                backupService.startBackup(admin, result);
            }
        });

        backupThd.start();
        waitForBackupToStart.acquire();
        String backupStartedBy = backupService.backupStartedBy();
        assertThat(ReflectionUtil.getField(backupService, "backupStartedBy"), is(backupStartedBy));
        waitForAssertionToCompleteWhileBackupIsOn.release();
        backupThd.join();
    }

    private void deleteConfigFileIfExists(String ...fileNames) {
        for (String fileName : fileNames) {
            FileUtils.deleteQuietly(new File(configDir(), fileName));
        }
    }

    private String fileContents(File location, String filename) throws IOException {
        ZipInputStream zipIn = null;
        ByteArrayOutputStream out = new ByteArrayOutputStream();
        try {
            zipIn = new ZipInputStream(new FileInputStream(location));
            while (zipIn.available() > 0) {
                ZipEntry nextEntry = zipIn.getNextEntry();
                if (nextEntry.getName().equals(filename)) {
                    IOUtils.copy(zipIn, out);
                }
            }
        } finally {
            if (zipIn != null) {
                zipIn.close();
            }
        }
        return out.toString();
    }

    private void createConfigFile(String fileName, String content) throws IOException {
        FileOutputStream fos = null;
        try {
            File file = new File(configDir(), fileName);
            FileUtils.forceMkdir(file.getParentFile());
            fos = new FileOutputStream(file);
            fos.write(content.getBytes());
        } finally {
            if (fos != null) {
                fos.close();
            }
        }
    }

    @Test
    public void shouldReturnIfBackupIsInProgress() throws SQLException, InterruptedException {
        final Semaphore waitForBackupToBegin = new Semaphore(1);
        final Semaphore waitForAssertion_whichHasToHappen_whileBackupIsRunning = new Semaphore(1);

        Database databaseStrategyMock = mock(Database.class);
        doAnswer(new Answer<Object>() {
            @Override
            public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
                waitForBackupToBegin.release();
                waitForAssertion_whichHasToHappen_whileBackupIsRunning.acquire();
                return null;
            }
        }).when(databaseStrategyMock).backup(any(File.class));


        final BackupService backupService = new BackupService(dataSource, artifactsDirHolder, goConfigService, new TimeProvider(), backupInfoRepository, systemEnvironment,
                serverVersion, configRepository, databaseStrategyMock);

        waitForBackupToBegin.acquire();
        Thread thd = new Thread(new Runnable() {
            public void run() {
                backupService.startBackup(admin, new HttpLocalizedOperationResult());
            }
        });
        thd.start();

        waitForAssertion_whichHasToHappen_whileBackupIsRunning.acquire();
        waitForBackupToBegin.acquire();
        assertThat(backupService.isBackingUp(), is(true));
        waitForAssertion_whichHasToHappen_whileBackupIsRunning.release();

        thd.join();
    }

    private File configDir() {
        return new File(new SystemEnvironment().getConfigDir());
    }

    private File backupDir(DateTime now) {
        return new File(backupsDirectory, BackupService.BACKUP + now.toString("YYYYMMdd-HHmmss"));
    }

    private File backedUpFile(final String filename) {
        return new ArrayList<>(FileUtils.listFiles(backupsDirectory, new NameFileFilter(filename), TrueFileFilter.TRUE)).get(0);
    }

    private void cleanupBackups() throws IOException {
        FileUtils.deleteQuietly(artifactsDirHolder.getArtifactsDir());
    }
}