From 8bf9dbc9e02dc6561d511d0f1a5023bfa2a9d8a9 Mon Sep 17 00:00:00 2001 From: Michel Jung Date: Wed, 8 Mar 2017 18:47:06 +0100 Subject: [PATCH 1/9] Improve API doc --- .../faforever/api/featuredmods/FeaturedModsController.java | 2 +- .../com/faforever/api/leaderboard/LeaderboardController.java | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/faforever/api/featuredmods/FeaturedModsController.java b/src/main/java/com/faforever/api/featuredmods/FeaturedModsController.java index 0ca6762f7..d10f32107 100644 --- a/src/main/java/com/faforever/api/featuredmods/FeaturedModsController.java +++ b/src/main/java/com/faforever/api/featuredmods/FeaturedModsController.java @@ -31,7 +31,7 @@ public FeaturedModsController(FeaturedModService featuredModService) { @Async @RequestMapping(path = "/{modId}/files/{version}") - @ApiOperation(value = "Lists the required files for a specific featured mod version") + @ApiOperation("Lists the required files for a specific featured mod version") public CompletableFuture getFiles(@PathVariable("modId") int modId, @PathVariable("version") String version) { ImmutableMap mods = Maps.uniqueIndex(featuredModService.getFeaturedMods(), FeaturedMod::getId); diff --git a/src/main/java/com/faforever/api/leaderboard/LeaderboardController.java b/src/main/java/com/faforever/api/leaderboard/LeaderboardController.java index 1b0f25f94..9062463f7 100644 --- a/src/main/java/com/faforever/api/leaderboard/LeaderboardController.java +++ b/src/main/java/com/faforever/api/leaderboard/LeaderboardController.java @@ -5,6 +5,7 @@ import com.yahoo.elide.jsonapi.models.Data; import com.yahoo.elide.jsonapi.models.JsonApiDocument; import com.yahoo.elide.jsonapi.models.Resource; +import io.swagger.annotations.ApiOperation; import org.springframework.scheduling.annotation.Async; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestMapping; @@ -28,6 +29,7 @@ public LeaderboardController(LeaderboardService leaderboardService) { @Async @RequestMapping(path = "/ladder1v1") + @ApiOperation("Lists the ladder1v1 leaderboard") public CompletableFuture getLadder1v1() { List values = leaderboardService.getLadder1v1Leaderboard().stream() .map(entry -> new Resource(LADDER_1V1_LEADERBOARD_ENTRY, String.valueOf(entry.getId()), @@ -48,6 +50,7 @@ public CompletableFuture getLadder1v1() { @Async @RequestMapping(path = "/global") + @ApiOperation("Lists the global leaderboard") public CompletableFuture getGlobal() { List values = leaderboardService.getGlobalLeaderboard().stream() .map(entry -> new Resource(GLOBAL_LEADERBOARD_ENTRY, String.valueOf(entry.getId()), @@ -67,6 +70,7 @@ public CompletableFuture getGlobal() { @Async @RequestMapping(path = "/ladder1v1/{playerId}") + @ApiOperation("Lists the ladder1v1 leaderboard for the specified player") public CompletableFuture getSingleLadder1v1(@PathVariable("playerId") String playerId) { Ladder1v1LeaderboardEntry entry = leaderboardService.getLadder1v1Entry(Integer.valueOf(playerId)); if (entry == null) { @@ -89,6 +93,7 @@ public CompletableFuture getSingleLadder1v1(@PathVariable("play @Async @RequestMapping(path = "/global/{playerId}") + @ApiOperation("Lists the global leaderboard for the specified player") public CompletableFuture getSingleGlobal(@PathVariable("playerId") String playerId) { GlobalLeaderboardEntry entry = leaderboardService.getGlobalEntry(Integer.valueOf(playerId)); if (entry == null) { From befcde9e4d4e35a87e3707427c81f080a2bf099b Mon Sep 17 00:00:00 2001 From: Michel Jung Date: Wed, 8 Mar 2017 18:47:58 +0100 Subject: [PATCH 2/9] Temporarily disable achievements --- .../api/achievements/PlayerAchievementsController.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/com/faforever/api/achievements/PlayerAchievementsController.java b/src/main/java/com/faforever/api/achievements/PlayerAchievementsController.java index 93bb2fa77..7701a8954 100644 --- a/src/main/java/com/faforever/api/achievements/PlayerAchievementsController.java +++ b/src/main/java/com/faforever/api/achievements/PlayerAchievementsController.java @@ -10,6 +10,7 @@ import org.springframework.web.bind.annotation.RestController; import javax.inject.Inject; +import java.time.Instant; import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; @@ -29,6 +30,10 @@ public PlayerAchievementsController(AchievementsService achievementsService) { @ApiOperation(value = "Updates the state and progress of one or multiple achievements.") public List update(@RequestBody AchievementUpdateRequest[] updateRequests, @AuthenticationPrincipal FafUserDetails userDetails) { + // FIXME protect using OAuth scope, so that only the server is allowed to send requests + if (Instant.now().toEpochMilli() > 0) { + throw new UnsupportedOperationException("Not yet supported"); + } int playerId = userDetails.getId(); return Arrays.stream(updateRequests).map(achievementUpdateRequest -> { switch (achievementUpdateRequest.getOperation()) { From 6e73c6c5c86550ac578b1fef9202c43f7da58163 Mon Sep 17 00:00:00 2001 From: Michel Jung Date: Wed, 8 Mar 2017 18:50:12 +0100 Subject: [PATCH 3/9] Mark endTime @Nullable It's possible that no player score has been reported --- src/main/java/com/faforever/api/data/domain/Game.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/com/faforever/api/data/domain/Game.java b/src/main/java/com/faforever/api/data/domain/Game.java index bd96b8e95..6c423cb8f 100644 --- a/src/main/java/com/faforever/api/data/domain/Game.java +++ b/src/main/java/com/faforever/api/data/domain/Game.java @@ -4,6 +4,7 @@ import lombok.Setter; import org.hibernate.annotations.Formula; import org.hibernate.annotations.Immutable; +import org.jetbrains.annotations.Nullable; import javax.persistence.Column; import javax.persistence.Entity; @@ -86,6 +87,7 @@ public List getPlayerStats() { } @Formula(value = "(SELECT game_player_stats.scoreTime FROM game_player_stats WHERE game_player_stats.gameId = id ORDER BY game_player_stats.scoreTime DESC LIMIT 1)") + @Nullable public OffsetDateTime getEndTime() { return endTime; } From 745825cfd66ff16b5288b74be8b6e2afae2e15f4 Mon Sep 17 00:00:00 2001 From: Michel Jung Date: Wed, 8 Mar 2017 18:57:42 +0100 Subject: [PATCH 4/9] Temporarily disable achievement test --- .../PlayerAchievementsControllerTest.java | 58 ++++++++----------- 1 file changed, 25 insertions(+), 33 deletions(-) diff --git a/src/test/java/com/faforever/api/achievements/PlayerAchievementsControllerTest.java b/src/test/java/com/faforever/api/achievements/PlayerAchievementsControllerTest.java index e148b397f..0bd2120c6 100644 --- a/src/test/java/com/faforever/api/achievements/PlayerAchievementsControllerTest.java +++ b/src/test/java/com/faforever/api/achievements/PlayerAchievementsControllerTest.java @@ -1,22 +1,13 @@ package com.faforever.api.achievements; import com.faforever.api.achievements.AchievementUpdateRequest.Operation; -import com.faforever.api.data.domain.AchievementState; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; import org.mockito.runners.MockitoJUnitRunner; -import java.util.List; - import static com.faforever.api.TestUser.testUser; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.hasSize; -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.anyInt; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; @RunWith(MockitoJUnitRunner.class) @@ -32,30 +23,31 @@ public void setUp() throws Exception { instance = new PlayerAchievementsController(achievementService); } - @Test - public void update() throws Exception { - AchievementUpdateRequest[] updateRequests = new AchievementUpdateRequest[]{ - new AchievementUpdateRequest("111", Operation.INCREMENT, 7), - new AchievementUpdateRequest("222", Operation.INCREMENT, 19), - new AchievementUpdateRequest("333", Operation.SET_STEPS_AT_LEAST, 9), - new AchievementUpdateRequest("444", Operation.SET_STEPS_AT_LEAST, 13), - new AchievementUpdateRequest("555", Operation.UNLOCK, 11), - new AchievementUpdateRequest("666", Operation.UNLOCK, 17), - }; - when(achievementService.increment(any(), anyInt(), anyInt())).thenAnswer(invocation -> - new AchievementUpdateResponse(true, AchievementState.UNLOCKED, invocation.getArgumentAt(1, int.class))); - - List result = instance.update(updateRequests, testUser()); - - verify(achievementService).increment("111", 7, 1); - verify(achievementService).increment("222", 19, 1); - verify(achievementService).setStepsAtLeast("333", 9, 1); - verify(achievementService).setStepsAtLeast("444", 13, 1); - verify(achievementService).unlock("555", 1); - verify(achievementService).unlock("666", 1); - - assertThat(result, hasSize(6)); - } + // FIXME add back as soon as OAuth scope protected +// @Test +// public void update() throws Exception { +// AchievementUpdateRequest[] updateRequests = new AchievementUpdateRequest[]{ +// new AchievementUpdateRequest("111", Operation.INCREMENT, 7), +// new AchievementUpdateRequest("222", Operation.INCREMENT, 19), +// new AchievementUpdateRequest("333", Operation.SET_STEPS_AT_LEAST, 9), +// new AchievementUpdateRequest("444", Operation.SET_STEPS_AT_LEAST, 13), +// new AchievementUpdateRequest("555", Operation.UNLOCK, 11), +// new AchievementUpdateRequest("666", Operation.UNLOCK, 17), +// }; +// when(achievementService.increment(any(), anyInt(), anyInt())).thenAnswer(invocation -> +// new AchievementUpdateResponse(true, AchievementState.UNLOCKED, invocation.getArgumentAt(1, int.class))); +// +// List result = instance.update(updateRequests, testUser()); +// +// verify(achievementService).increment("111", 7, 1); +// verify(achievementService).increment("222", 19, 1); +// verify(achievementService).setStepsAtLeast("333", 9, 1); +// verify(achievementService).setStepsAtLeast("444", 13, 1); +// verify(achievementService).unlock("555", 1); +// verify(achievementService).unlock("666", 1); +// +// assertThat(result, hasSize(6)); +// } @Test(expected = UnsupportedOperationException.class) public void updateReveledUnsupported() throws Exception { From 87a01af1a50a8bd7abc0dae5e37810ba31642a69 Mon Sep 17 00:00:00 2001 From: Michel Jung Date: Tue, 7 Feb 2017 12:43:42 +0100 Subject: [PATCH 5/9] Add legacy deployment --- build.gradle | 10 +- gradle.properties | 3 + .../api/config/FafApiProperties.java | 38 ++- .../faforever/api/config/GitHubConfig.java | 15 ++ .../config/security/WebSecurityConfig.java | 1 + .../api/deployment/GitHubController.java | 72 ++++++ .../deployment/GitHubDeploymentService.java | 81 ++++++ .../LegacyFeaturedModDeploymentTask.java | 243 ++++++++++++++++++ .../api/deployment/git/GitWrapper.java | 13 + .../api/deployment/git/JGitWrapper.java | 48 ++++ .../api/featuredmods/FeaturedModFile.java | 12 +- .../FeaturedModFileRepository.java | 55 ---- .../api/featuredmods/FeaturedModService.java | 19 +- .../featuredmods/FeaturedModsController.java | 8 +- .../LegacyFeaturedModFileRepository.java | 93 +++++++ .../com/faforever/api/map/MapService.java | 15 +- .../api/utils/ByteCountListener.java | 7 - .../java/com/faforever/api/utils/LuaUtil.java | 37 --- .../java/com/faforever/api/utils/MapData.java | 14 - .../faforever/api/utils/PreviewGenerator.java | 190 -------------- .../com/faforever/api/utils/Unzipper.java | 105 -------- .../java/com/faforever/api/utils/Zipper.java | 133 ---------- src/main/resources/config/application-dev.yml | 22 +- .../resources/config/application-prod.yml | 31 ++- src/main/resources/config/application.yml | 4 - .../GitHubDeploymentServiceTest.java | 149 +++++++++++ .../LegacyFeaturedModDeploymentTaskTest.java | 123 +++++++++ .../com/faforever/api/map/MapServiceTest.java | 10 +- src/test/resources/featured_mod/mod_info.lua | 28 ++ .../resources/featured_mod/someDir/someFile | 0 30 files changed, 993 insertions(+), 586 deletions(-) create mode 100644 src/main/java/com/faforever/api/config/GitHubConfig.java create mode 100644 src/main/java/com/faforever/api/deployment/GitHubController.java create mode 100644 src/main/java/com/faforever/api/deployment/GitHubDeploymentService.java create mode 100644 src/main/java/com/faforever/api/deployment/LegacyFeaturedModDeploymentTask.java create mode 100644 src/main/java/com/faforever/api/deployment/git/GitWrapper.java create mode 100644 src/main/java/com/faforever/api/deployment/git/JGitWrapper.java delete mode 100644 src/main/java/com/faforever/api/featuredmods/FeaturedModFileRepository.java create mode 100644 src/main/java/com/faforever/api/featuredmods/LegacyFeaturedModFileRepository.java delete mode 100644 src/main/java/com/faforever/api/utils/ByteCountListener.java delete mode 100644 src/main/java/com/faforever/api/utils/LuaUtil.java delete mode 100644 src/main/java/com/faforever/api/utils/MapData.java delete mode 100644 src/main/java/com/faforever/api/utils/PreviewGenerator.java delete mode 100644 src/main/java/com/faforever/api/utils/Unzipper.java delete mode 100644 src/main/java/com/faforever/api/utils/Zipper.java create mode 100644 src/test/java/com/faforever/api/deployment/GitHubDeploymentServiceTest.java create mode 100644 src/test/java/com/faforever/api/deployment/LegacyFeaturedModDeploymentTaskTest.java create mode 100644 src/test/resources/featured_mod/mod_info.lua create mode 100644 src/test/resources/featured_mod/someDir/someFile diff --git a/build.gradle b/build.gradle index 8cd245cce..982385d81 100644 --- a/build.gradle +++ b/build.gradle @@ -24,6 +24,7 @@ targetCompatibility = 1.8 repositories { mavenCentral() + maven { url "http://repo.jenkins-ci.org/public/" } maven { url "https://jitpack.io" } } @@ -123,6 +124,11 @@ task pushDockerImage(type: DockerPushImage, dependsOn: buildDockerImage) { tag project.version } +configurations.all { + // Cache -SNAPSHOT for 60 seconds only + resolutionStrategy.cacheChangingModulesFor 60, 'seconds' +} + dependencyManagement { dependencies { dependency("org.hibernate:hibernate-core:${hibernateVersion}") @@ -140,11 +146,13 @@ dependencies { compile("org.springframework.boot:spring-boot-starter-security") compile("de.codecentric:spring-boot-admin-starter-client:${springBootAdminClientVersion}") + compile("com.github.FAForever:faf-java-commons:${fafCommonsVersion}") + compile("org.kohsuke:github-api:1.84") compile("org.jolokia:jolokia-core:${jolokiaVersion}") compile("org.springframework.security:spring-security-jwt:${springSecurityJwtVersion}") compile("org.springframework.security.oauth:spring-security-oauth2:${springSecurityOauth2Version}") compile("org.springframework:spring-context-support:${springContextSupportVersion}") - + compile("org.eclipse.jgit:org.eclipse.jgit:${jgitVersionn}") compile("org.jetbrains:annotations:${jetbrainsAnnotationsVersion}") compile("com.google.guava:guava:${guavaVersion}") compile("io.springfox:springfox-swagger-ui:${swaggerVersion}") diff --git a/gradle.properties b/gradle.properties index 1461f9ff9..8ff1c0510 100644 --- a/gradle.properties +++ b/gradle.properties @@ -21,5 +21,8 @@ luajVersion=3.0.1 nocatchVersion=1.1 junitAddonsVersion=1.4 zohhakVersion=1.1.1 +githubApiVersion=1.84 +jgitVersionn=4.5.0.201609210915-r +fafCommonsVersion=76fb583d146082f3db68c0bece38a3ee28ead8e6 h2Version=1.4.193 jacksonDatatypeJsr310Version=2.8.6 diff --git a/src/main/java/com/faforever/api/config/FafApiProperties.java b/src/main/java/com/faforever/api/config/FafApiProperties.java index f1b7c04d2..581c7d2a5 100644 --- a/src/main/java/com/faforever/api/config/FafApiProperties.java +++ b/src/main/java/com/faforever/api/config/FafApiProperties.java @@ -6,6 +6,8 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.time.Duration; +import java.util.ArrayList; +import java.util.List; @Data @ConfigurationProperties(prefix = "faf-api", ignoreUnknownFields = false) @@ -22,23 +24,22 @@ public class FafApiProperties { private Mod mod = new Mod(); private Clan clan = new Clan(); private FeaturedMods featuredMods = new FeaturedMods(); + private GitHub gitHub = new GitHub(); + private Deployment deployment = new Deployment(); @Data public static class OAuth2 { - private String resourceId = "faf-api"; } @Data public static class Jwt { - private int accessTokenValiditySeconds = 3600; private int refreshTokenValiditySeconds = 3600; } @Data public static class Async { - private int corePoolSize = Runtime.getRuntime().availableProcessors(); private int maxPoolSize = Runtime.getRuntime().availableProcessors() * 4; private int queueCapacity = Integer.MAX_VALUE; @@ -46,13 +47,12 @@ public static class Async { @Data public static class Map { - private String smallPreviewsUrlFormat = "http://content.faforever.com/faf/vault/map_previews/small/%s"; private String largePreviewsUrlFormat = "http://content.faforever.com/faf/vault/map_previews/large/%s"; private String downloadUrlFormat = "http://content.faforever.com/faf/vault/maps/%s"; private Path folderZipFiles = Paths.get("/content/faf/vault/maps"); - private Path folderPreviewPathSmall = Paths.get("/content/faf/vault/map_previews/small"); - private Path folderPreviewPathLarge = Paths.get("/content/faf/vault/map_previews/large"); + private Path folderPreviewPathSmall = Paths.get("static/map_previews/small"); + private Path folderPreviewPathLarge = Paths.get("static/map_previews/large"); private int previewSizeSmall = 128; private int previewSizeLarge = 512; private String[] allowedExtensions = {"zip"}; @@ -73,4 +73,30 @@ public static class FeaturedMods { public static class Clan { private long inviteLinkExpireDurationInMinutes = Duration.ofDays(3).toMinutes(); } + + @Data + public static class GitHub { + private String webhookSecret; + private String repositoriesDirectory; + private String accessToken; + private String deploymentEnvironment; + } + + @Data + public static class Deployment { + private String targetFolder; + private String repositoriesFolder; + private String filesFolderFormat = "updates_%s_files"; + private String forgedAllianceExePath; + private List configurations = new ArrayList<>(); + + @Data + public static class DeploymentConfiguration { + private String repositoryUrl; + private String branch; + private String modName; + private String modFilesExtension; + private boolean replaceExisting; + } + } } diff --git a/src/main/java/com/faforever/api/config/GitHubConfig.java b/src/main/java/com/faforever/api/config/GitHubConfig.java new file mode 100644 index 000000000..efbe686cb --- /dev/null +++ b/src/main/java/com/faforever/api/config/GitHubConfig.java @@ -0,0 +1,15 @@ +package com.faforever.api.config; + +import org.kohsuke.github.GitHub; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import java.io.IOException; + +@Configuration +public class GitHubConfig { + @Bean + public GitHub gitHub(FafApiProperties fafApiProperties) throws IOException { + return GitHub.connectUsingOAuth(fafApiProperties.getGitHub().getAccessToken()); + } +} diff --git a/src/main/java/com/faforever/api/config/security/WebSecurityConfig.java b/src/main/java/com/faforever/api/config/security/WebSecurityConfig.java index 5574f42e0..43f123c22 100644 --- a/src/main/java/com/faforever/api/config/security/WebSecurityConfig.java +++ b/src/main/java/com/faforever/api/config/security/WebSecurityConfig.java @@ -60,6 +60,7 @@ protected void configure(HttpSecurity http) throws Exception { .antMatchers("/leaderboards/**").permitAll() .antMatchers("/featuredMods/**").permitAll() .antMatchers("/oauth/**").permitAll() + .antMatchers("/gitHub/webhook").permitAll() // Redirects to Swagger UI .antMatchers("/").permitAll() // Swagger UI diff --git a/src/main/java/com/faforever/api/deployment/GitHubController.java b/src/main/java/com/faforever/api/deployment/GitHubController.java new file mode 100644 index 000000000..258302dc2 --- /dev/null +++ b/src/main/java/com/faforever/api/deployment/GitHubController.java @@ -0,0 +1,72 @@ +package com.faforever.api.deployment; + +import com.faforever.api.config.FafApiProperties; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.kohsuke.github.GHEventPayload; +import org.kohsuke.github.GHEventPayload.Deployment; +import org.kohsuke.github.GHEventPayload.Push; +import org.kohsuke.github.GitHub; +import org.springframework.scheduling.annotation.Async; +import org.springframework.security.jwt.crypto.sign.MacSigner; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestHeader; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +import javax.crypto.spec.SecretKeySpec; +import javax.xml.bind.DatatypeConverter; +import java.io.StringReader; +import java.nio.charset.StandardCharsets; + +@RestController +@RequestMapping(path = "/gitHub") +@Slf4j +public class GitHubController { + + private static final String HMAC_SHA1 = "HmacSHA1"; + private final GitHubDeploymentService gitHubDeploymentService; + private final GitHub gitHub; + private final FafApiProperties apiProperties; + + public GitHubController(GitHubDeploymentService gitHubDeploymentService, GitHub gitHub, FafApiProperties apiProperties) { + this.gitHubDeploymentService = gitHubDeploymentService; + this.gitHub = gitHub; + this.apiProperties = apiProperties; + } + + @Async + @RequestMapping(path = "/webhook", method = RequestMethod.POST) + @SneakyThrows + public void onPush(@RequestBody String body, + @RequestHeader("X-Hub-Signature") String signature, + @RequestHeader("X-GitHub-Event") String eventType) { + verifyRequest(body, signature); + switch (eventType) { + case "push": + gitHubDeploymentService.createDeploymentIfEligible(parseEvent(body, Push.class)); + break; + case "deployment": + gitHubDeploymentService.deploy(parseEvent(body, Deployment.class).getDeployment()); + break; + default: + log.warn("Unhandled event: " + eventType); + } + } + + @SneakyThrows + private T parseEvent(@RequestBody String body, Class type) { + return gitHub.parseEventPayload(new StringReader(body), type); + } + + @SneakyThrows + private void verifyRequest(String payload, String signature) { + String secret = apiProperties.getGitHub().getWebhookSecret(); + MacSigner macSigner = new MacSigner(HMAC_SHA1, new SecretKeySpec(secret.getBytes(StandardCharsets.US_ASCII), HMAC_SHA1)); + + byte[] content = payload.getBytes(StandardCharsets.US_ASCII); + // Signature starts with "sha1=" + macSigner.verify(content, DatatypeConverter.parseHexBinary(signature.substring(5))); + } +} diff --git a/src/main/java/com/faforever/api/deployment/GitHubDeploymentService.java b/src/main/java/com/faforever/api/deployment/GitHubDeploymentService.java new file mode 100644 index 000000000..863bc4b99 --- /dev/null +++ b/src/main/java/com/faforever/api/deployment/GitHubDeploymentService.java @@ -0,0 +1,81 @@ +package com.faforever.api.deployment; + +import com.faforever.api.config.FafApiProperties; +import com.faforever.api.config.FafApiProperties.Deployment.DeploymentConfiguration; +import com.faforever.api.error.ProgrammingError; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.kohsuke.github.GHDeployment; +import org.kohsuke.github.GHEventPayload.Push; +import org.kohsuke.github.GHEventPayload.Push.PushCommit; +import org.springframework.context.ApplicationContext; +import org.springframework.scheduling.annotation.Async; +import org.springframework.stereotype.Service; + +import java.util.List; +import java.util.Objects; +import java.util.Optional; + + +@Service +@Slf4j +public class GitHubDeploymentService { + + private final ApplicationContext applicationContext; + private final FafApiProperties fafApiProperties; + private final ObjectMapper objectMapper; + + public GitHubDeploymentService(ApplicationContext applicationContext, FafApiProperties fafApiProperties, ObjectMapper objectMapper) { + this.applicationContext = applicationContext; + this.fafApiProperties = fafApiProperties; + this.objectMapper = objectMapper; + } + + @SneakyThrows + public void createDeploymentIfEligible(Push push) { + List commits = push.getCommits(); + PushCommit headCommit = commits.get(0); + + if (!Objects.equals(headCommit.getSha(), push.getHead())) { + throw new ProgrammingError("Expected first commit to be the head commit, apparently that is not the case"); + } + + String ref = push.getRef(); + if (!headCommit.isDistinct()) { + log.debug("Ignoring non-distinct commit to ref: {}", ref); + return; + } + Optional optional = fafApiProperties.getDeployment().getConfigurations().stream() + .filter(configuration -> + push.getRepository().gitHttpTransportUrl().equals(configuration.getRepositoryUrl()) + && push.getRef().replace("refs/heads/", "").equals(configuration.getBranch())) + .findFirst(); + + if (!optional.isPresent()) { + log.warn("No configuration present for repository '{}' and ref '{}'", push.getRepository().gitHttpTransportUrl(), push.getRef()); + return; + } + + GHDeployment ghDeployment = push.getRepository().createDeployment(ref) + .environment(fafApiProperties.getGitHub().getDeploymentEnvironment()) + .payload(objectMapper.writeValueAsString(optional.get())) + .create(); + + log.info("Created deployment: {}", ghDeployment); + } + + @Async + @SneakyThrows + public void deploy(GHDeployment deployment) { + String environment = deployment.getEnvironment(); + if (!fafApiProperties.getGitHub().getDeploymentEnvironment().equals(environment)) { + log.warn("Ignoring deployment for environment: {}", environment); + return; + } + + applicationContext.getBean(LegacyFeaturedModDeploymentTask.class) + .setConfiguration(objectMapper.readValue(deployment.getPayload(), DeploymentConfiguration.class)) + .run(); + } +} diff --git a/src/main/java/com/faforever/api/deployment/LegacyFeaturedModDeploymentTask.java b/src/main/java/com/faforever/api/deployment/LegacyFeaturedModDeploymentTask.java new file mode 100644 index 000000000..e158b7acc --- /dev/null +++ b/src/main/java/com/faforever/api/deployment/LegacyFeaturedModDeploymentTask.java @@ -0,0 +1,243 @@ +package com.faforever.api.deployment; + +import com.faforever.api.config.FafApiProperties; +import com.faforever.api.config.FafApiProperties.Deployment; +import com.faforever.api.config.FafApiProperties.Deployment.DeploymentConfiguration; +import com.faforever.api.deployment.git.GitWrapper; +import com.faforever.api.featuredmods.FeaturedModFile; +import com.faforever.api.featuredmods.FeaturedModService; +import com.faforever.commons.fa.ForgedAllianceExePatcher; +import com.faforever.commons.mod.ModReader; +import com.faforever.commons.zip.Zipper; +import com.google.common.collect.Sets; +import lombok.Data; +import lombok.Setter; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Component; +import org.springframework.util.Assert; + +import javax.validation.ValidationException; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import java.util.zip.ZipOutputStream; + +import static com.github.nocatch.NoCatch.noCatch; +import static com.google.common.hash.Hashing.md5; +import static com.google.common.io.Files.hash; +import static java.nio.file.Files.createDirectories; + +/** + * Checks out a specific ref of a featured mod's Git repository and performs the required steps in order to deploy it. + * At this point, this mechanism is rather ridiculous but that's what legacy products of uneducated people often are. + * I hope that we'll be able to introduce a sane mechanism that doesn't require a database pretty soon. + */ +@Slf4j +@Component +@Lazy +public class LegacyFeaturedModDeploymentTask implements Runnable { + + private static final String NON_WORD_CHARACTER_PATTERN = "[^\\w]"; + private static final Set VALID_MOD_NAMES = Sets.newHashSet("faf", "fafbeta", "fafdevelop"); + + private final GitWrapper gitWrapper; + private final FeaturedModService featuredModService; + private final FafApiProperties apiProperties; + + @Setter + private DeploymentConfiguration configuration; + + public LegacyFeaturedModDeploymentTask(GitWrapper gitWrapper, FeaturedModService featuredModService, FafApiProperties apiProperties) { + this.gitWrapper = gitWrapper; + this.featuredModService = featuredModService; + this.apiProperties = apiProperties; + } + + @Override + @SneakyThrows + public void run() { + Assert.state(configuration != null, "Configuration must be set"); + String modName = configuration.getModName(); + Assert.state(VALID_MOD_NAMES.contains(modName), "Unsupported mod: " + modName); + + String repositoryUrl = configuration.getRepositoryUrl(); + String branch = configuration.getBranch(); + boolean replaceExisting = configuration.isReplaceExisting(); + String modFilesExtension = configuration.getModFilesExtension(); + Map fileIds = featuredModService.getFileIds(modName); + Deployment deployment = apiProperties.getDeployment(); + + log.info("Starting deployment of '{}' from '{}', branch '{}', replaceExisting '{}', modFilesExtension '{}'", + modName, repositoryUrl, branch, replaceExisting, modFilesExtension); + + Path repositoryDirectory = buildRepositoryDirectoryPath(repositoryUrl); + checkoutCode(repositoryDirectory, repositoryUrl, branch); + + short version = readModVersion(repositoryDirectory); + verifyVersion(version, replaceExisting, modName); + + Path targetFolder = Paths.get(deployment.getTargetFolder(), String.format(deployment.getFilesFolderFormat(), modName)); + List files = packageDirectories(repositoryDirectory, version, fileIds, targetFolder); + createPatchedExe(version, fileIds, targetFolder).ifPresent(files::add); + + files.forEach(this::renameToFinalFile); + + updateDatabase(files, version, modName); + + log.info("Deployment of '{}' version '{}' was successful", modName, version); + } + + /** + * Creates a ForgedAlliance.exe which contains the specified version number, if the file is specified for the current + * featured mod. + */ + @SneakyThrows + private Optional createPatchedExe(short version, Map fileIds, Path targetFolder) { + String clientFileName = "ForgedAlliance.exe"; + Short fileId = fileIds.get(clientFileName); + if (fileId == null) { + return Optional.empty(); + } + + Path targetFile = targetFolder.resolve(String.format("ForgedAlliance.%d.exe", version)); + Path tmpFile = toTmpFile(targetFile); + Files.copy(Paths.get(apiProperties.getDeployment().getForgedAllianceExePath()), tmpFile, StandardCopyOption.REPLACE_EXISTING); + + ForgedAllianceExePatcher.patchVersion(tmpFile, version); + + return Optional.of(new StagedFile(fileId, tmpFile, targetFile, clientFileName)); + } + + private short readModVersion(Path modPath) { + return (short) Integer.parseInt(new ModReader().extractModInfo(modPath).getVersion().toString()); + } + + private void verifyVersion(int version, boolean replaceExisting, String modName) { + if (!replaceExisting && !featuredModService.getFiles(modName, version).isEmpty()) { + throw new ValidationException(String.format("Version '%s' of mod '%s' already exists", version, modName)); + } + } + + private void updateDatabase(List files, short version, String modName) { + List featuredModFiles = files.stream() + .map(file -> new FeaturedModFile() + .setMd5(noCatch(() -> hash(file.getTargetFile().toFile(), md5())).toString()) + .setFileId(file.getFileId()) + .setName(file.getTargetFile().getFileName().toString()) + .setVersion(version) + ) + .collect(Collectors.toList()); + + featuredModService.save(modName, version, featuredModFiles); + } + + /** + * Reads all directories (except directories starting with {@code .}), zips their contents and moves the result to + * the target folder. + * + * @return the list of deployed files + */ + @SneakyThrows + private List packageDirectories(Path repositoryDirectory, short version, Map fileIds, Path targetFolder) { + try (Stream stream = Files.list(repositoryDirectory)) { + return stream + .filter((path) -> Files.isDirectory(path) && !path.getFileName().toString().startsWith(".")) + .map(path -> packFile(path, version, targetFolder, fileIds)) + .filter(Optional::isPresent) + .map(Optional::get) + .collect(Collectors.toList()); + } + } + + /** + * Renames the temporary file to the target file so the file is only available under its final name when it is + * complete. + */ + private StagedFile renameToFinalFile(StagedFile file) { + Path source = file.getTmpFile(); + Path target = file.getTargetFile(); + log.trace("Renaming '{}' to '{}", source, target); + noCatch(() -> Files.move(source, target, StandardCopyOption.ATOMIC_MOVE)); + return file; + } + + /** + * Creates a ZIP file with the file ending configured in {@link #configuration}. The content of the ZIP file is the + * content of the directory. If no file ID is available, an empty optional is returned. + */ + @SneakyThrows + private Optional packFile(Path folderToBeZipped, Short version, Path targetFolder, Map fileIds) { + String folderName = folderToBeZipped.getFileName().toString(); + Path targetNxtFile = targetFolder.resolve(String.format("%s.%d.nxt", folderName, version)); + Path tmpNxtFile = toTmpFile(targetNxtFile); + + // E.g. "effects.nx2" + String clientFileName = String.format("%s.%s", folderName, configuration.getModFilesExtension()); + Short fileId = fileIds.get(clientFileName); + if (fileId == null) { + log.debug("Skipping folder '{}' because there's no file ID available", folderName); + return Optional.empty(); + } + + log.trace("Packaging '{}' to '{}'", folderToBeZipped, targetFolder); + + createDirectories(targetFolder); + try (ZipOutputStream outputStream = new ZipOutputStream(Files.newOutputStream(tmpNxtFile))) { + Zipper.contentOf(folderToBeZipped).to(outputStream).zip(); + } + return Optional.of(new StagedFile(fileId, tmpNxtFile, targetNxtFile, clientFileName)); + } + + private void checkoutCode(Path repositoryDirectory, String repoUrl, String branch) throws IOException { + if (Files.notExists(repositoryDirectory)) { + createDirectories(repositoryDirectory.getParent()); + gitWrapper.clone(repoUrl, repositoryDirectory); + } else { + gitWrapper.fetch(repositoryDirectory); + } + gitWrapper.checkoutRef(repositoryDirectory, "refs/remotes/origin/" + branch); + } + + private Path buildRepositoryDirectoryPath(String repoUrl) { + String repoDirName = repoUrl.replaceAll(NON_WORD_CHARACTER_PATTERN, ""); + return Paths.get(apiProperties.getDeployment().getRepositoriesFolder(), repoDirName); + } + + private Path toTmpFile(Path targetFile) { + return targetFile.getParent().resolve(targetFile.getFileName().toString() + ".tmp"); + } + + /** + * Describes a file that is ready to be deployed. All files should be staged as temporary files first so they can be + * renamed to their target file name in one go, thus minimizing the time of inconsistent file system state. + */ + @Data + private class StagedFile { + /** + * ID of the file as stored in the database. + */ + private final int fileId; + /** + * The staged file, already in the correct location, that is ready to be renamed. + */ + private final Path tmpFile; + /** + * The final file name and location. + */ + private final Path targetFile; + /** + * Name of the file as the client will know it. + */ + private final String clientFileName; + } +} diff --git a/src/main/java/com/faforever/api/deployment/git/GitWrapper.java b/src/main/java/com/faforever/api/deployment/git/GitWrapper.java new file mode 100644 index 000000000..d3d147f8d --- /dev/null +++ b/src/main/java/com/faforever/api/deployment/git/GitWrapper.java @@ -0,0 +1,13 @@ +package com.faforever.api.deployment.git; + +import java.io.IOException; +import java.nio.file.Path; + +public interface GitWrapper { + + void clone(String repositoryUri, Path targetDirectory); + + void fetch(Path repoDirectory) throws IOException; + + void checkoutRef(Path repoDirectory, String ref); +} diff --git a/src/main/java/com/faforever/api/deployment/git/JGitWrapper.java b/src/main/java/com/faforever/api/deployment/git/JGitWrapper.java new file mode 100644 index 000000000..afb486e20 --- /dev/null +++ b/src/main/java/com/faforever/api/deployment/git/JGitWrapper.java @@ -0,0 +1,48 @@ +package com.faforever.api.deployment.git; + +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.eclipse.jgit.api.Git; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Component; + +import java.nio.file.Path; + +import static com.github.nocatch.NoCatch.noCatch; +import static org.eclipse.jgit.api.Git.cloneRepository; +import static org.eclipse.jgit.api.Git.open; + +@Lazy +@Component +@Slf4j +public class JGitWrapper implements GitWrapper { + + public void clone(String repositoryUri, Path targetDirectory) { + log.debug("Cloning '{}' to '{}'", repositoryUri, targetDirectory); + noCatch(() -> cloneRepository() + .setURI(repositoryUri) + .setDirectory(targetDirectory.toFile()) + .call()); + } + + @Override + @SneakyThrows + public void fetch(Path repoDirectory) { + log.debug("Fetching remote of '{}'", repoDirectory); + try (Git git = open(repoDirectory.toFile())) { + git.fetch().call(); + } + } + + @Override + @SneakyThrows + public void checkoutRef(Path repoDirectory, String ref) { + log.debug("Checking out '{}' in '{}'", ref, repoDirectory); + try (Git git = open(repoDirectory.toFile())) { + git.checkout() + .setForce(true) + .setName(ref) + .call(); + } + } +} diff --git a/src/main/java/com/faforever/api/featuredmods/FeaturedModFile.java b/src/main/java/com/faforever/api/featuredmods/FeaturedModFile.java index 0d626a273..dc1c4d98a 100644 --- a/src/main/java/com/faforever/api/featuredmods/FeaturedModFile.java +++ b/src/main/java/com/faforever/api/featuredmods/FeaturedModFile.java @@ -13,6 +13,8 @@ @Immutable @Entity @EntityListeners(FeaturedModFileEnricher.class) +// Thanks to the dynamic table nature of the legacy updater, this class is not mapped to an underlying database but +// a native query instead. This is why the columns here can't be found in any table. public class FeaturedModFile { private int id; @@ -22,6 +24,7 @@ public class FeaturedModFile { private short version; private String url; private String folderName; + private int fileId; @Id @Column(name = "id") @@ -49,6 +52,11 @@ public short getVersion() { return version; } + @Column(name = "fileId") + public int getFileId() { + return fileId; + } + @Transient // Enriched by FeaturedModFileEnricher public String getUrl() { @@ -56,8 +64,8 @@ public String getUrl() { } /** - * Returns the name of the folder in which the file resides (e.g. {@code updates_faf_files}). Used by the - * FeaturedModFileEnricher. + * Returns the name of the folder on the server in which the file resides (e.g. {@code updates_faf_files}). Used by + * the {@link FeaturedModFileEnricher}. */ @Column(name = "folderName") public String getFolderName() { diff --git a/src/main/java/com/faforever/api/featuredmods/FeaturedModFileRepository.java b/src/main/java/com/faforever/api/featuredmods/FeaturedModFileRepository.java deleted file mode 100644 index 830416b69..000000000 --- a/src/main/java/com/faforever/api/featuredmods/FeaturedModFileRepository.java +++ /dev/null @@ -1,55 +0,0 @@ -package com.faforever.api.featuredmods; - -import org.springframework.data.repository.Repository; - -import javax.persistence.EntityManager; -import javax.persistence.Query; -import java.util.List; -import java.util.regex.Pattern; - -@org.springframework.stereotype.Repository -public class FeaturedModFileRepository implements Repository { - private static final Pattern MOD_NAME_PATTERN = Pattern.compile("[a-z]+"); - private final EntityManager entityManager; - - public FeaturedModFileRepository(EntityManager entityManager) { - this.entityManager = entityManager; - } - - @SuppressWarnings("unchecked") - public List getFiles(String modName, Integer version) { - if (!MOD_NAME_PATTERN.matcher(modName).matches()) { - throw new IllegalArgumentException("Invalid mod name: " + modName); - } - - // The following joke is sponsored by FAF's patcher mechanism which shouldn't even require a DB. - Query query = entityManager.createNativeQuery(String.format( - "SELECT" + - " b.filename AS `name`," + - " file.version AS `version`," + - " b.path AS `group`," + - " file.md5 AS `md5`," + - " file.id AS `id`," + - " file.name AS `url`," + - " 'updates_%1$s_files' AS `folderName`" + - "FROM updates_%1$s_files file" + - " INNER JOIN" + - " (" + - " SELECT" + - " fileId," + - " MAX(version) AS version" + - " FROM updates_%1$s_files" + - " %2$s" + - " GROUP BY fileId" + - " ) latest" + - " ON file.fileId = latest.fileId" + - " AND file.version = latest.version" + - " LEFT JOIN updates_%1$s b" + - " ON b.id = file.fileId;", modName, (version == null ? "" : "WHERE version <= :version")), FeaturedModFile.class); - - if (version != null) { - query.setParameter("version", version); - } - return (List) query.getResultList(); - } -} diff --git a/src/main/java/com/faforever/api/featuredmods/FeaturedModService.java b/src/main/java/com/faforever/api/featuredmods/FeaturedModService.java index 26d37dbaf..a2d04e62c 100644 --- a/src/main/java/com/faforever/api/featuredmods/FeaturedModService.java +++ b/src/main/java/com/faforever/api/featuredmods/FeaturedModService.java @@ -4,8 +4,10 @@ import org.jetbrains.annotations.Nullable; import org.springframework.cache.annotation.Cacheable; import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; import java.util.List; +import java.util.Map; @Service public class FeaturedModService { @@ -13,20 +15,29 @@ public class FeaturedModService { public static final String FEATURED_MOD_FILES_CACHE_NAME = "featuredModFiles"; private final FeaturedModRepository featuredModRepository; - private final FeaturedModFileRepository featuredModFileRepository; + private final LegacyFeaturedModFileRepository legacyFeaturedModFileRepository; - public FeaturedModService(FeaturedModRepository featuredModRepository, FeaturedModFileRepository featuredModFileRepository) { + public FeaturedModService(FeaturedModRepository featuredModRepository, LegacyFeaturedModFileRepository legacyFeaturedModFileRepository) { this.featuredModRepository = featuredModRepository; - this.featuredModFileRepository = featuredModFileRepository; + this.legacyFeaturedModFileRepository = legacyFeaturedModFileRepository; } @Cacheable(FEATURED_MOD_FILES_CACHE_NAME) public List getFiles(String modName, @Nullable Integer version) { - return featuredModFileRepository.getFiles(modName, version); + return legacyFeaturedModFileRepository.getFiles(modName, version); } @Cacheable(FEATURED_MODS_CACHE_NAME) public List getFeaturedMods() { return featuredModRepository.findAll(); } + + @Transactional + public void save(String modName, short version, List featuredModFiles) { + legacyFeaturedModFileRepository.save(modName, version, featuredModFiles); + } + + public Map getFileIds(String modName) { + return legacyFeaturedModFileRepository.getFileIds(modName); + } } diff --git a/src/main/java/com/faforever/api/featuredmods/FeaturedModsController.java b/src/main/java/com/faforever/api/featuredmods/FeaturedModsController.java index d10f32107..63d368a15 100644 --- a/src/main/java/com/faforever/api/featuredmods/FeaturedModsController.java +++ b/src/main/java/com/faforever/api/featuredmods/FeaturedModsController.java @@ -12,11 +12,9 @@ import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; -import java.util.Collection; import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.function.Function; -import java.util.function.Supplier; import java.util.stream.Collectors; @RestController @@ -39,11 +37,7 @@ public CompletableFuture getFiles(@PathVariable("modId") int mo Integer innerVersion = "latest".equals(version) ? null : Integer.valueOf(version); - return getFiles(() -> featuredModService.getFiles(featuredMod.getTechnicalName(), innerVersion)); - } - - private CompletableFuture getFiles(Supplier> supplier) { - List values = supplier.get().stream() + List values = featuredModService.getFiles(featuredMod.getTechnicalName(), innerVersion).stream() .map(modFileMapper()) .collect(Collectors.toList()); diff --git a/src/main/java/com/faforever/api/featuredmods/LegacyFeaturedModFileRepository.java b/src/main/java/com/faforever/api/featuredmods/LegacyFeaturedModFileRepository.java new file mode 100644 index 000000000..2d47d0b67 --- /dev/null +++ b/src/main/java/com/faforever/api/featuredmods/LegacyFeaturedModFileRepository.java @@ -0,0 +1,93 @@ +package com.faforever.api.featuredmods; + +import org.springframework.data.repository.Repository; +import org.springframework.util.Assert; + +import javax.persistence.EntityManager; +import javax.persistence.Query; +import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +@org.springframework.stereotype.Repository +public class LegacyFeaturedModFileRepository implements Repository { + private static final Pattern MOD_NAME_PATTERN = Pattern.compile("[a-z]+"); + private final EntityManager entityManager; + + public LegacyFeaturedModFileRepository(EntityManager entityManager) { + this.entityManager = entityManager; + } + + @SuppressWarnings("unchecked") + public List getFiles(String modName, Integer version) { + verifyModName(modName); + + // The following joke is sponsored by FAF's patcher mechanism which shouldn't even require a DB. + Query query = entityManager.createNativeQuery(String.format( + "SELECT" + + " b.filename AS `name`," + + " file.version AS `version`," + + " b.path AS `group`," + + " file.md5 AS `md5`," + + " file.id AS `id`," + + " file.name AS `url`," + + " 'updates_%1$s_files' AS `folderName`" + + "FROM updates_%1$s_files file" + + " INNER JOIN" + + " (" + + " SELECT" + + " fileId," + + " MAX(version) AS version" + + " FROM updates_%1$s_files" + + " %2$s" + + " GROUP BY fileId" + + " ) latest" + + " ON file.fileId = latest.fileId" + + " AND file.version = latest.version" + + " LEFT JOIN updates_%1$s b" + + " ON b.id = file.fileId;", modName, (version == null ? "" : "WHERE version <= :version")), FeaturedModFile.class); + + if (version != null) { + query.setParameter("version", version); + } + return (List) query.getResultList(); + } + + public void save(String modName, short version, List featuredModFiles) { + verifyModName(modName); + + // Upsert would be preferred, but the tables have no unique constraints and it's not worth fixing them + Query deleteQuery = entityManager.createNativeQuery(String.format( + "DELETE FROM updates_%s_files WHERE version = :version", modName + )); + deleteQuery.setParameter("version", version); + deleteQuery.executeUpdate(); + + Query insertQuery = entityManager.createNativeQuery(String.format( + "INSERT INTO updates_%s_files (fileid, version, name, md5) VALUES (:fileId, :version, :name, :md5)", modName + )); + + featuredModFiles.forEach(featuredModFile -> { + insertQuery.setParameter("fileId", featuredModFile.getFileId()); + insertQuery.setParameter("version", featuredModFile.getVersion()); + insertQuery.setParameter("name", featuredModFile.getName()); + insertQuery.setParameter("md5", featuredModFile.getMd5()); + insertQuery.executeUpdate(); + }); + } + + @SuppressWarnings("unchecked") + public Map getFileIds(String modName) { + verifyModName(modName); + + Query query = entityManager.createNativeQuery(String.format("SELECT id, filename FROM updates_%s", modName)); + + return ((List) query.getResultList()).stream() + .collect(Collectors.toMap(row -> (String) row[1], row -> (short) row[0])); + } + + private void verifyModName(String modName) { + Assert.isTrue(MOD_NAME_PATTERN.matcher(modName).matches(), "Invalid mod name: " + modName); + } +} diff --git a/src/main/java/com/faforever/api/map/MapService.java b/src/main/java/com/faforever/api/map/MapService.java index 5a70f1e2d..70c52e017 100644 --- a/src/main/java/com/faforever/api/map/MapService.java +++ b/src/main/java/com/faforever/api/map/MapService.java @@ -10,9 +10,10 @@ import com.faforever.api.error.ErrorCode; import com.faforever.api.error.ProgrammingError; import com.faforever.api.utils.JavaFxUtil; -import com.faforever.api.utils.PreviewGenerator; -import com.faforever.api.utils.Unzipper; -import com.faforever.api.utils.Zipper; +import com.faforever.commons.lua.LuaLoader; +import com.faforever.commons.map.PreviewGenerator; +import com.faforever.commons.zip.Unzipper; +import com.faforever.commons.zip.Zipper; import javafx.scene.image.Image; import lombok.Data; import lombok.SneakyThrows; @@ -38,7 +39,6 @@ import java.util.zip.ZipInputStream; import java.util.zip.ZipOutputStream; -import static com.faforever.api.utils.LuaUtil.loadFile; import static com.github.nocatch.NoCatch.noCatch; @Service @@ -149,7 +149,7 @@ private void parseScenarioLua(MapUploadData progressData) { .filter(myFile -> myFile.toString().endsWith("_scenario.lua")) .findFirst() .orElseThrow(() -> new ApiException(new Error(ErrorCode.MAP_SCENARIO_LUA_MISSING))); - LuaValue root = noCatch(() -> loadFile(scenarioLuaPath), IllegalStateException.class); + LuaValue root = noCatch(() -> LuaLoader.loadFile(scenarioLuaPath), IllegalStateException.class); progressData.setLuaRoot(root); } } @@ -202,11 +202,8 @@ private boolean invalidTeam(LuaValue scenarioInfo) { return true; } LuaValue armies = firstTeam.get(ScenarioMapInfo.CONFIGURATION_STANDARD_TEAMS_ARMIES); - if (armies == LuaValue.NIL) { - return true; - } + return armies == LuaValue.NIL || !teamName.tojstring().equals("FFA"); - return !teamName.tojstring().equals("FFA"); } private void postProcessLuaFile(MapUploadData progressData) { diff --git a/src/main/java/com/faforever/api/utils/ByteCountListener.java b/src/main/java/com/faforever/api/utils/ByteCountListener.java deleted file mode 100644 index 04b2be408..000000000 --- a/src/main/java/com/faforever/api/utils/ByteCountListener.java +++ /dev/null @@ -1,7 +0,0 @@ -package com.faforever.api.utils; - -// TODO: move to shared faf code -public interface ByteCountListener { - - void updateBytesWritten(long written, long total); -} diff --git a/src/main/java/com/faforever/api/utils/LuaUtil.java b/src/main/java/com/faforever/api/utils/LuaUtil.java deleted file mode 100644 index 3a5a0af09..000000000 --- a/src/main/java/com/faforever/api/utils/LuaUtil.java +++ /dev/null @@ -1,37 +0,0 @@ -package com.faforever.api.utils; - -import com.google.common.io.CharStreams; -import org.luaj.vm2.Globals; -import org.luaj.vm2.LuaValue; -import org.luaj.vm2.lib.jse.JsePlatform; - -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.nio.file.Files; -import java.nio.file.Path; - -import static java.nio.charset.StandardCharsets.UTF_8; - -// TODO: move to shared faf code -public final class LuaUtil { - - private LuaUtil() { - throw new AssertionError("Not instantiatable"); - } - - public static LuaValue loadFile(Path file) throws IOException { - try (InputStream inputStream = Files.newInputStream(file)) { - return load(inputStream); - } - } - - public static LuaValue load(InputStream inputStream) throws IOException { - Globals globals = JsePlatform.standardGlobals(); - try (InputStreamReader inputStreamReader = new InputStreamReader(LuaUtil.class.getResourceAsStream("/lua/faf.lua"), UTF_8)) { - globals.baselib.load(globals.load(CharStreams.toString(inputStreamReader))); - globals.load(inputStream, "@" + inputStream.hashCode(), "bt", globals).invoke(); - return globals; - } - } -} diff --git a/src/main/java/com/faforever/api/utils/MapData.java b/src/main/java/com/faforever/api/utils/MapData.java deleted file mode 100644 index c9b6b3aed..000000000 --- a/src/main/java/com/faforever/api/utils/MapData.java +++ /dev/null @@ -1,14 +0,0 @@ -package com.faforever.api.utils; - -import lombok.Data; -import org.luaj.vm2.LuaTable; - -// TODO: move to shared faf code -@Data -public class MapData { - - private byte[] ddsData; - private LuaTable markers; - private float width; - private float height; -} diff --git a/src/main/java/com/faforever/api/utils/PreviewGenerator.java b/src/main/java/com/faforever/api/utils/PreviewGenerator.java deleted file mode 100644 index 540059b49..000000000 --- a/src/main/java/com/faforever/api/utils/PreviewGenerator.java +++ /dev/null @@ -1,190 +0,0 @@ -package com.faforever.api.utils; - -import com.google.common.io.LittleEndianDataInputStream; -import javafx.embed.swing.SwingFXUtils; -import javafx.scene.image.WritableImage; -import lombok.SneakyThrows; -import org.luaj.vm2.LuaTable; -import org.luaj.vm2.LuaValue; - -import javax.imageio.ImageIO; -import java.awt.Graphics; -import java.awt.Image; -import java.awt.Point; -import java.awt.image.BufferedImage; -import java.awt.image.DataBufferByte; -import java.awt.image.Raster; -import java.io.IOException; -import java.io.InputStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Optional; -import java.util.stream.Stream; - -import static com.github.nocatch.NoCatch.noCatch; -import static java.awt.Image.SCALE_SMOOTH; -import static java.awt.image.BufferedImage.TYPE_INT_ARGB; -import static java.nio.file.Files.list; - -// TODO: move to shared faf code -public final class PreviewGenerator { - - private static final double RESOURCE_ICON_RATIO = 20 / 1024d; - private static final String MASS_IMAGE = "/images/map_markers/mass.png"; - private static final String HYDRO_IMAGE = "/images/map_markers/hydro.png"; - private static final String ARMY_IMAGE = "/images/map_markers/army.png"; - private static final String ARMY_PREFIX = "ARMY_"; - - private PreviewGenerator() { - throw new AssertionError("Not instantiatable"); - } - - public static javafx.scene.image.Image generatePreview(Path mapFolder, int width, int height) throws IOException { - try (Stream mapFolderStream = list(mapFolder)) { - Optional mapPath = mapFolderStream - .filter(file -> file.getFileName().toString().endsWith(".scmap")) - .findFirst(); - if (!mapPath.isPresent()) { - throw new RuntimeException("No map file was found in: " + mapFolder.toAbsolutePath()); - } - - return noCatch(() -> { - MapData mapData = parseMap(mapPath.get()); - if (mapData == null) { - throw new RuntimeException("mapdata is null after parseMap from: " + mapPath.get()); - } - - BufferedImage previewImage = getDdsImage(mapData); - previewImage = scale(previewImage, width, height); - - addMarkers(previewImage, mapData); - - return SwingFXUtils.toFXImage(previewImage, new WritableImage(width, height)); - }); - - } - } - - @SneakyThrows - private static MapData parseMap(Path mapPath) { - MapData mapData = new MapData(); - try (LittleEndianDataInputStream mapInput = new LittleEndianDataInputStream(Files.newInputStream(mapPath))) { - mapInput.skip(16); - mapData.setWidth((int) mapInput.readFloat()); - mapData.setHeight((int) mapInput.readFloat()); - mapInput.skip(6); - - int ddsSize = mapInput.readInt(); - // Skip DDS header - mapInput.skipBytes(128); - - byte[] buffer = new byte[ddsSize - 128]; - mapInput.readFully(buffer); - - mapData.setDdsData(buffer); - - Path lua; - try (Stream fileStream = list(mapPath.getParent())) { - Optional saveLua = fileStream - .filter(filePath -> filePath.toString().toLowerCase().endsWith("_save.lua")) - .findFirst(); - lua = saveLua.get(); - } - if (!Files.isRegularFile(lua)) { - throw new RuntimeException("Path is no regular file: " + lua); - } - LuaTable markers = LuaUtil.loadFile(lua).get("Scenario").get("MasterChain").get("_MASTERCHAIN_").get("Markers").checktable(); - mapData.setMarkers(markers); - } - return mapData; - } - - private static BufferedImage getDdsImage(MapData mapData) throws IOException { - byte[] ddsData = mapData.getDdsData(); - int ddsDimension = (int) (Math.sqrt(ddsData.length) / 2); - - bgraToAbgr(ddsData); - BufferedImage previewImage = new BufferedImage(ddsDimension, ddsDimension, BufferedImage.TYPE_4BYTE_ABGR); - previewImage.setData(Raster.createRaster(previewImage.getSampleModel(), new DataBufferByte(ddsData, ddsData.length), new Point())); - return previewImage; - } - - private static BufferedImage scale(BufferedImage previewImage, double width, double height) { - int targetWidth = width < 1 ? 1 : (int) width; - int targetHeight = height < 1 ? 1 : (int) height; - - Image image = previewImage.getScaledInstance(targetWidth, targetHeight, SCALE_SMOOTH); - BufferedImage scaledImage = new BufferedImage(targetWidth, targetHeight, TYPE_INT_ARGB); - - Graphics graphics = scaledImage.createGraphics(); - graphics.drawImage(image, 0, 0, null); - graphics.dispose(); - - return scaledImage; - } - - private static void addMarkers(BufferedImage previewImage, MapData mapData) throws IOException { - float width = previewImage.getWidth(); - float height = previewImage.getHeight(); - - Image massImage = scale(readImage(MASS_IMAGE), RESOURCE_ICON_RATIO * width, RESOURCE_ICON_RATIO * height); - Image hydroImage = scale(readImage(HYDRO_IMAGE), RESOURCE_ICON_RATIO * width, RESOURCE_ICON_RATIO * height); - Image armyImage = scale(readImage(ARMY_IMAGE), RESOURCE_ICON_RATIO * width, RESOURCE_ICON_RATIO * height); - - LuaTable markers = mapData.getMarkers(); - for (LuaValue key : markers.keys()) { - LuaTable markerData = markers.get(key).checktable(); - - switch (markerData.get("type").toString()) { - case "Mass": - addMarker(massImage, mapData, markerData, previewImage); - break; - case "Hydrocarbon": - addMarker(hydroImage, mapData, markerData, previewImage); - break; - case "Blank Marker": - if (!key.tojstring().startsWith(ARMY_PREFIX)) { - continue; - } - addMarker(armyImage, mapData, markerData, previewImage); - break; - } - } - } - - private static void bgraToAbgr(byte[] buffer) { - for (int i = 0; i < buffer.length; i += 4) { - byte a = buffer[i + 3]; - buffer[i + 3] = buffer[i + 2]; - buffer[i + 2] = buffer[i + 1]; - buffer[i + 1] = buffer[i]; - buffer[i] = a; - } - } - - private static BufferedImage readImage(String resource) throws IOException { - try (InputStream inputStream = PreviewGenerator.class.getResourceAsStream(resource)) { - return ImageIO.read(inputStream); - } - } - - private static void addMarker(Image source, MapData mapData, LuaTable markerData, BufferedImage target) throws IOException { - LuaTable vector = markerData.get("position").checktable(); - float x = vector.get(1).tofloat() / mapData.getWidth(); - float y = vector.get(3).tofloat() / mapData.getHeight(); - - paintOnImage(source, x, y, target); - } - - private static void paintOnImage(Image overlay, float xPercent, float yPercent, BufferedImage baseImage) { - int overlayWidth = overlay.getWidth(null); - int overlayHeight = overlay.getHeight(null); - int x = (int) (xPercent * baseImage.getWidth() - overlayWidth / 2); - int y = (int) (yPercent * baseImage.getHeight() - overlayHeight / 2); - - x = Math.min(Math.max(0, x), baseImage.getWidth() - overlayWidth); - y = Math.min(Math.max(0, y), baseImage.getHeight() - overlayHeight); - - baseImage.getGraphics().drawImage(overlay, x, y, null); - } -} diff --git a/src/main/java/com/faforever/api/utils/Unzipper.java b/src/main/java/com/faforever/api/utils/Unzipper.java deleted file mode 100644 index 6369ec845..000000000 --- a/src/main/java/com/faforever/api/utils/Unzipper.java +++ /dev/null @@ -1,105 +0,0 @@ -package com.faforever.api.utils; - - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.io.OutputStream; -import java.lang.invoke.MethodHandles; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.zip.ZipEntry; -import java.util.zip.ZipInputStream; - -import static java.nio.file.StandardOpenOption.CREATE; - -// TODO: move to shared faf code -public final class Unzipper { - - private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - private final ZipInputStream zipInputStream; - private ByteCountListener byteCountListener; - private int byteCountInterval; - private int bufferSize; - private long totalBytes; - private Path targetDirectory; - private long lastCountUpdate; - - private Unzipper(ZipInputStream zipInputStream) { - this.zipInputStream = zipInputStream; - // 4K - bufferSize = 0x1000; - byteCountInterval = 40; - } - - public static Unzipper from(ZipInputStream zipInputStream) { - return new Unzipper(zipInputStream); - } - - public Unzipper to(Path targetDirectory) { - this.targetDirectory = targetDirectory; - return this; - } - - public Unzipper byteCountInterval(int byteCountInterval) { - this.byteCountInterval = byteCountInterval; - return this; - } - - public Unzipper listener(ByteCountListener byteCountListener) { - this.byteCountListener = byteCountListener; - return this; - } - - public Unzipper bufferSize(int bufferSize) { - this.bufferSize = bufferSize; - return this; - } - - public Unzipper totalBytes(long totalBytes) { - this.totalBytes = totalBytes; - return this; - } - - public void unzip() throws IOException { - byte[] buffer = new byte[bufferSize]; - - long bytesDone = 0; - - ZipEntry zipEntry; - while ((zipEntry = zipInputStream.getNextEntry()) != null) { - Path targetFile = targetDirectory.resolve(zipEntry.getName()); - if (zipEntry.isDirectory()) { - logger.trace("Creating directory {}", targetFile); - Files.createDirectories(targetFile); - continue; - } - - Path parentDirectory = targetFile.getParent(); - if (Files.notExists(parentDirectory)) { - logger.trace("Creating directory {}", parentDirectory); - Files.createDirectories(parentDirectory); - } - - long compressedSize = zipEntry.getCompressedSize(); - if (compressedSize != -1) { - bytesDone += compressedSize; - } - - logger.trace("Writing file {}", targetFile); - try (OutputStream outputStream = Files.newOutputStream(targetFile, CREATE)) { - int length; - while ((length = zipInputStream.read(buffer)) != -1) { - outputStream.write(buffer, 0, length); - - long now = System.currentTimeMillis(); - if (byteCountListener != null && lastCountUpdate < now - byteCountInterval) { - byteCountListener.updateBytesWritten(bytesDone, totalBytes); - lastCountUpdate = now; - } - } - } - } - } -} diff --git a/src/main/java/com/faforever/api/utils/Zipper.java b/src/main/java/com/faforever/api/utils/Zipper.java deleted file mode 100644 index 3254bff45..000000000 --- a/src/main/java/com/faforever/api/utils/Zipper.java +++ /dev/null @@ -1,133 +0,0 @@ -package com.faforever.api.utils; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.lang.invoke.MethodHandles; -import java.nio.file.FileVisitResult; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.SimpleFileVisitor; -import java.nio.file.attribute.BasicFileAttributes; -import java.util.Objects; -import java.util.zip.ZipEntry; -import java.util.zip.ZipOutputStream; - -// TODO move to shared FAF code -public final class Zipper { - - private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - private static final char PATH_SEPARATOR = File.separatorChar; - private final Path directoryToZip; - private boolean zipContent; - private ByteCountListener byteCountListener; - private int byteCountInterval; - private int bufferSize; - private long bytesTotal; - private long bytesDone; - private ZipOutputStream zipOutputStream; - private long lastCountUpdate; - private byte[] buffer; - - /** - * @param zipContent {@code true} if the contents of the directory should be zipped, {@code false} if the specified - * file (directory) should be zipped directly. - */ - private Zipper(Path directoryToZip, boolean zipContent) { - this.directoryToZip = directoryToZip; - this.zipContent = zipContent; - // 4K - bufferSize = 0x1000; - byteCountInterval = 40; - } - - public static Zipper of(Path path) { - return new Zipper(path, false); - } - - public static Zipper contentOf(Path path) { - return new Zipper(path, true); - } - - public Zipper to(ZipOutputStream zipOutputStream) { - this.zipOutputStream = zipOutputStream; - return this; - } - - public Zipper byteCountInterval(int byteCountInterval) { - this.byteCountInterval = byteCountInterval; - return this; - } - - public Zipper listener(ByteCountListener byteCountListener) { - this.byteCountListener = byteCountListener; - return this; - } - - public void zip() throws IOException { - Objects.requireNonNull(zipOutputStream, "zipOutputStream must not be null"); - Objects.requireNonNull(directoryToZip, "directoryToZip must not be null"); - - bytesTotal = calculateTotalBytes(); - bytesDone = 0; - buffer = new byte[bufferSize]; - - Files.walkFileTree(directoryToZip, new SimpleFileVisitor() { - public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { - if (zipContent) { - zipOutputStream.putNextEntry(new ZipEntry(directoryToZip.relativize(dir).toString() + "/")); - } else { - zipOutputStream.putNextEntry(new ZipEntry(directoryToZip.getParent().relativize(dir).toString() + "/")); - } - - zipOutputStream.closeEntry(); - return FileVisitResult.CONTINUE; - } - - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - logger.trace("Zipping file {}", file.toAbsolutePath()); - - if (zipContent) { - zipOutputStream.putNextEntry(new ZipEntry(directoryToZip.relativize(file).toString().replace(PATH_SEPARATOR, '/'))); - } else { - zipOutputStream.putNextEntry(new ZipEntry(directoryToZip.getParent().relativize(file).toString().replace(PATH_SEPARATOR, '/'))); - } - - try (InputStream inputStream = Files.newInputStream(file)) { - copy(inputStream, zipOutputStream); - } - zipOutputStream.closeEntry(); - return FileVisitResult.CONTINUE; - } - }); - } - - private long calculateTotalBytes() throws IOException { - final long[] bytesTotal = {0}; - Files.walkFileTree(directoryToZip, new SimpleFileVisitor() { - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - bytesTotal[0] += Files.size(file); - return FileVisitResult.CONTINUE; - } - }); - return bytesTotal[0]; - } - - private void copy(InputStream inputStream, OutputStream outputStream) throws IOException { - int length; - while ((length = inputStream.read(buffer)) > 0) { - outputStream.write(buffer, 0, length); - bytesDone += length; - - long now = System.currentTimeMillis(); - if (byteCountListener != null && lastCountUpdate < now - byteCountInterval) { - byteCountListener.updateBytesWritten(bytesDone, bytesTotal); - lastCountUpdate = now; - } - } - } -} diff --git a/src/main/resources/config/application-dev.yml b/src/main/resources/config/application-dev.yml index 7dd572577..7084d2727 100644 --- a/src/main/resources/config/application-dev.yml +++ b/src/main/resources/config/application-dev.yml @@ -1,9 +1,19 @@ faf-api: - jwt-secret: banana + jwt-secret: ${JWT_SECRET:banana} map: - folder-zip-files: ${MAP_UPLOAD_PATH:tmp/map/maps} - folder-preview-path-small: ${MAP_PREVIEW_PATH_SMALL:tmp/map/map_previews/small} - folder-preview-path-large: ${MAP_PREVIEW_PATH_LARGE:tmp/map/map_previews/large} + folder-zip-files: ${MAP_UPLOAD_PATH:build/cache/map/maps} + folder-preview-path-small: ${MAP_PREVIEW_PATH_SMALL:build/cache/map_previews/small} + folder-preview-path-large: ${MAP_PREVIEW_PATH_LARGE:build/cache/map_previews/large} + git-hub: + access-token: ${GITHUB_ACCESS_TOKEN:none} + deployment-environment: ${GITHUB_DEPLOYMENT_ENVIRONMENT:development} + repositories-directory: ${REPOSITORIES_DIRECTORY:build/cache/repos} + webhook-secret: ${GITHUB_WEBHOOK_SECRET:none} + deployment: + forged-alliance-exe-path: ${FORGED_ALLIANCE_EXE_PATH} + repositories-folder: ${REPOSITORIES_FOLDER:build/cache/repos} + target-folder: ${DEPLOYMENT_TARGET_FOLDER:build/cache/deployment} + spring: datasource: url: jdbc:mysql://${DATABASE_ADDRESS:localhost}/${DATABASE_NAME:faf}?useSSL=false @@ -12,3 +22,7 @@ spring: password: ${DATABASE_PASSWORD:banana} jpa: show-sql: true + +logging: + level: + com.faforever.api: trace diff --git a/src/main/resources/config/application-prod.yml b/src/main/resources/config/application-prod.yml index 6139e0b4e..debbaccd7 100644 --- a/src/main/resources/config/application-prod.yml +++ b/src/main/resources/config/application-prod.yml @@ -1,10 +1,39 @@ faf-api: jwt-secret: ${JWT_SECRET} + map: + folder-zip-files: ${MAP_UPLOAD_PATH} + folder-preview-path-small: ${MAP_PREVIEW_PATH_SMALL} + folder-preview-path-large: ${MAP_PREVIEW_PATH_LARGE} + git-hub: + access-token: ${GITHUB_ACCESS_TOKEN} + deployment-environment: ${GITHUB_DEPLOYMENT_ENVIRONMENT:production} + repositories-directory: ${REPOSITORY_DIRECTORY} + webhook-secret: ${GITHUB_WEBHOOK_SECRET} + deployment: + forged-alliance-exe-path: ${FORGED_ALLIANCE_EXE_PATH} + repositories-folder: ${REPOSITORIES_FOLDER} + target-folder: ${DEPLOYMENT_TARGET_FOLDER} + configurations: + - repositoryUrl: https://github.com/FAForever/fa.git + branch: deploy/faf + modName: faf + modFilesExtension: .nx2 + replaceExisting: false + - repositoryUrl: https://github.com/FAForever/fa.git + branch: deploy/fafbeta + modName: fafbeta + modFilesExtension: .nx4 + replaceExisting: true + - repositoryUrl: https://github.com/FAForever/fa.git + branch: deploy/fafdevelop + modName: fafdevelop + modFilesExtension: .nx5 + replaceExisting: true spring: datasource: url: jdbc:mysql://${DATABASE_ADDRESS}/${DATABASE_NAME}?useSSL=false - name: faf_lobby + name: faf username: ${DATABASE_USERNAME} password: ${DATABASE_PASSWORD} jpa: diff --git a/src/main/resources/config/application.yml b/src/main/resources/config/application.yml index e428e2b36..546c9099a 100644 --- a/src/main/resources/config/application.yml +++ b/src/main/resources/config/application.yml @@ -1,9 +1,5 @@ faf-api: version: #faf-api.version# - map: - folder-zip-files: ${MAP_UPLOAD_PATH:/content/faf/vault/maps} - folder-preview-path-small: ${MAP_PREVIEW_PATH_SMALL:/content/faf/vault/map_previews/small} - folder-preview-path-large: ${MAP_PREVIEW_PATH_LARGE:/content/faf/vault/map_previews/large} spring: application: diff --git a/src/test/java/com/faforever/api/deployment/GitHubDeploymentServiceTest.java b/src/test/java/com/faforever/api/deployment/GitHubDeploymentServiceTest.java new file mode 100644 index 000000000..df497b809 --- /dev/null +++ b/src/test/java/com/faforever/api/deployment/GitHubDeploymentServiceTest.java @@ -0,0 +1,149 @@ +package com.faforever.api.deployment; + +import com.faforever.api.config.FafApiProperties; +import com.faforever.api.config.FafApiProperties.Deployment.DeploymentConfiguration; +import com.faforever.api.error.ProgrammingError; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.kohsuke.github.GHDeployment; +import org.kohsuke.github.GHDeploymentBuilder; +import org.kohsuke.github.GHEventPayload.Push; +import org.kohsuke.github.GHEventPayload.Push.PushCommit; +import org.kohsuke.github.GHRepository; +import org.mockito.Mock; +import org.mockito.runners.MockitoJUnitRunner; +import org.springframework.context.ApplicationContext; + +import java.util.Collections; + +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class GitHubDeploymentServiceTest { + + private static final String EXAMPLE_REPO_URL = "https://example.com/repo.git"; + private static final String ENVIRONMENT = "junit"; + private GitHubDeploymentService instance; + + private FafApiProperties apiProperties; + + @Mock + private ApplicationContext applicationContext; + @Mock + private ObjectMapper objectMapper; + + @Before + public void setUp() throws Exception { + apiProperties = new FafApiProperties(); + instance = new GitHubDeploymentService(applicationContext, apiProperties, objectMapper); + } + + @Test(expected = ProgrammingError.class) + public void createDeploymentIfEligibleHeadCommitNotEqualToFirstCommit() throws Exception { + PushCommit pushCommit = mock(PushCommit.class); + when(pushCommit.getSha()).thenReturn("111"); + + Push push = mock(Push.class); + when(push.getHead()).thenReturn("222"); + when(push.getCommits()).thenReturn(Collections.singletonList(pushCommit)); + + instance.createDeploymentIfEligible(push); + } + + @Test + public void createDeploymentIfEligibleDistinctIgnored() throws Exception { + PushCommit pushCommit = mock(PushCommit.class); + when(pushCommit.isDistinct()).thenReturn(false); + + Push push = mock(Push.class); + when(push.getCommits()).thenReturn(Collections.singletonList(pushCommit)); + + instance.createDeploymentIfEligible(push); + + verify(push, never()).getRepository(); + } + + @Test + public void createDeploymentIfEligibleNoConfigurationAvailable() throws Exception { + PushCommit pushCommit = mock(PushCommit.class); + when(pushCommit.isDistinct()).thenReturn(true); + + Push push = mock(Push.class); + when(push.getCommits()).thenReturn(Collections.singletonList(pushCommit)); + + GHRepository repository = mock(GHRepository.class); + when(repository.gitHttpTransportUrl()).thenReturn(EXAMPLE_REPO_URL); + when(push.getRepository()).thenReturn(repository); + + instance.createDeploymentIfEligible(push); + + verify(repository, never()).createDeployment(any()); + } + + @Test + public void createDeploymentIfEligible() throws Exception { + PushCommit pushCommit = mock(PushCommit.class); + when(pushCommit.isDistinct()).thenReturn(true); + + Push push = mock(Push.class); + when(push.getRef()).thenReturn("refs/heads/master"); + when(push.getCommits()).thenReturn(Collections.singletonList(pushCommit)); + + GHRepository repository = mock(GHRepository.class); + when(repository.gitHttpTransportUrl()).thenReturn(EXAMPLE_REPO_URL); + + GHDeploymentBuilder deploymentBuilder = mock(GHDeploymentBuilder.class); + when(deploymentBuilder.environment(ENVIRONMENT)).thenReturn(deploymentBuilder); + when(deploymentBuilder.payload(anyString())).thenReturn(deploymentBuilder); + when(repository.createDeployment("refs/heads/master")).thenReturn(deploymentBuilder); + + when(push.getRepository()).thenReturn(repository); + + apiProperties.getGitHub().setDeploymentEnvironment(ENVIRONMENT); + apiProperties.getDeployment().setConfigurations(Collections.singletonList( + new DeploymentConfiguration() + .setBranch("master") + .setModFilesExtension(".nx2") + .setModName("faf") + .setRepositoryUrl(EXAMPLE_REPO_URL) + )); + + instance.createDeploymentIfEligible(push); + + verify(repository).createDeployment(any()); + verify(deploymentBuilder).environment(ENVIRONMENT); + verify(deploymentBuilder).create(); + } + + @Test + public void deployEnvironmentMismatch() throws Exception { + apiProperties.getGitHub().setDeploymentEnvironment(ENVIRONMENT); + + instance.deploy(new GHDeployment()); + + verify(applicationContext, never()).getBean(LegacyFeaturedModDeploymentTask.class); + } + + @Test + public void deployEnvironmentMatch() throws Exception { + apiProperties.getGitHub().setDeploymentEnvironment(ENVIRONMENT); + + GHDeployment deployment = mock(GHDeployment.class); + when(deployment.getEnvironment()).thenReturn(ENVIRONMENT); + + LegacyFeaturedModDeploymentTask task = mock(LegacyFeaturedModDeploymentTask.class); + when(task.setConfiguration(any())).thenReturn(task); + when(applicationContext.getBean(LegacyFeaturedModDeploymentTask.class)).thenReturn(task); + + instance.deploy(deployment); + + verify(task).run(); + } +} diff --git a/src/test/java/com/faforever/api/deployment/LegacyFeaturedModDeploymentTaskTest.java b/src/test/java/com/faforever/api/deployment/LegacyFeaturedModDeploymentTaskTest.java new file mode 100644 index 000000000..94b5293f2 --- /dev/null +++ b/src/test/java/com/faforever/api/deployment/LegacyFeaturedModDeploymentTaskTest.java @@ -0,0 +1,123 @@ +package com.faforever.api.deployment; + +import com.faforever.api.config.FafApiProperties; +import com.faforever.api.config.FafApiProperties.Deployment; +import com.faforever.api.config.FafApiProperties.Deployment.DeploymentConfiguration; +import com.faforever.api.deployment.git.GitWrapper; +import com.faforever.api.featuredmods.FeaturedModFile; +import com.faforever.api.featuredmods.FeaturedModService; +import com.google.common.collect.ImmutableMap; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.runners.MockitoJUnitRunner; + +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Comparator; +import java.util.List; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.junit.Assert.assertThat; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class LegacyFeaturedModDeploymentTaskTest { + + @Rule + public TemporaryFolder repositoriesFolder = new TemporaryFolder(); + @Rule + public TemporaryFolder targetFolder = new TemporaryFolder(); + private LegacyFeaturedModDeploymentTask instance; + @Mock + private GitWrapper gitWrapper; + @Mock + private FeaturedModService featuredModService; + private FafApiProperties properties; + + @Before + public void setUp() throws Exception { + properties = new FafApiProperties(); + Deployment deployment = properties.getDeployment(); + deployment.setRepositoriesFolder(repositoriesFolder.getRoot().getAbsolutePath()); + deployment.setTargetFolder(targetFolder.getRoot().getAbsolutePath()); + + instance = new LegacyFeaturedModDeploymentTask(gitWrapper, featuredModService, properties); + } + + @Test(expected = IllegalStateException.class) + public void testRunWithoutConfigurationThrowsException() throws Exception { + instance.run(); + } + + @Test + @SuppressWarnings("unchecked") + public void testRun() throws Exception { + instance.setConfiguration(new DeploymentConfiguration() + .setBranch("branch") + .setModFilesExtension("nx3") + .setModName("faf") + .setReplaceExisting(true) + .setRepositoryUrl("git@example.com/FAForever/faf")); + + Mockito.doAnswer(invocation -> { + Path repoFolder = invocation.getArgumentAt(0, Path.class); + Files.createDirectories(repoFolder.resolve("someDir")); + Files.copy( + LegacyFeaturedModDeploymentTaskTest.class.getResourceAsStream("/featured_mod/mod_info.lua"), + repoFolder.resolve("mod_info.lua") + ); + Files.copy(LegacyFeaturedModDeploymentTaskTest.class.getResourceAsStream("/featured_mod/someDir/someFile"), + repoFolder.resolve("someDir/someFile") + ); + return null; + }).when(gitWrapper).checkoutRef(any(), any()); + + when(featuredModService.getFileIds("faf")).thenReturn(ImmutableMap.of( + "ForgedAlliance.exe", (short) 1, + "someDir.nx3", (short) 2 + )); + + Path dummyExe = repositoriesFolder.getRoot().toPath().resolve("TemplateForgedAlliance.exe"); + createDummyExe(dummyExe); + properties.getDeployment().setForgedAllianceExePath(dummyExe.toAbsolutePath().toString()); + + instance.run(); + + ArgumentCaptor> filesCaptor = ArgumentCaptor.forClass((Class) List.class); + verify(featuredModService).save(eq("faf"), eq((short) 1337), filesCaptor.capture()); + + List files = filesCaptor.getValue(); + files.sort(Comparator.comparing(FeaturedModFile::getFileId)); + + assertThat(files.get(0).getFileId(), is(1)); + assertThat(files.get(0).getMd5(), is("47df959058cb52fe966ea5936dbd8f4c")); + assertThat(files.get(0).getName(), is("ForgedAlliance.1337.exe")); + assertThat(files.get(0).getVersion(), is((short) 1337)); + + assertThat(files.get(1).getFileId(), is(2)); + assertThat(files.get(1).getMd5(), is(notNullValue())); + assertThat(files.get(1).getName(), is("someDir.1337.nxt")); + assertThat(files.get(1).getVersion(), is((short) 1337)); + + assertThat(Files.exists(targetFolder.getRoot().toPath().resolve("updates_faf_files/someDir.1337.nxt")), is(true)); + assertThat(Files.exists(targetFolder.getRoot().toPath().resolve("updates_faf_files/ForgedAlliance.1337.exe")), is(true)); + } + + private void createDummyExe(Path file) throws IOException { + try (RandomAccessFile randomAccessFile = new RandomAccessFile(file.toFile(), "rw")) { + randomAccessFile.setLength(12_444_928); + } + } +} diff --git a/src/test/java/com/faforever/api/map/MapServiceTest.java b/src/test/java/com/faforever/api/map/MapServiceTest.java index d2c6d2a08..93bfa9c20 100644 --- a/src/test/java/com/faforever/api/map/MapServiceTest.java +++ b/src/test/java/com/faforever/api/map/MapServiceTest.java @@ -8,7 +8,7 @@ import com.faforever.api.error.ApiException; import com.faforever.api.error.ApiExceptionWithMultipleCodes; import com.faforever.api.error.ErrorCode; -import com.faforever.api.utils.Unzipper; +import com.faforever.commons.zip.Unzipper; import com.google.common.io.ByteStreams; import com.googlecode.zohhak.api.TestWith; import com.googlecode.zohhak.api.runners.ZohhakRunner; @@ -49,20 +49,18 @@ @RunWith(ZohhakRunner.class) public class MapServiceTest { - private MapService instance; - private Map mapProperties; - @Rule public final TemporaryFolder temporaryDirectory = new TemporaryFolder(); @Rule public final TemporaryFolder finalDirectory = new TemporaryFolder(); @Rule public final ExpectedException expectedException = ExpectedException.none(); - private final MapRepository mapRepository = mock(MapRepository.class); private final FafApiProperties fafApiProperties = mock(FafApiProperties.class); private final ContentService contentService = mock(ContentService.class); private final Player author = mock(Player.class); + private MapService instance; + private Map mapProperties; @Before public void setUp() { @@ -276,7 +274,6 @@ public void positiveUploadTest() throws IOException { FileAssert.assertEquals("Difference in " + expectedFile.getFileName().toString(), expectedFile.toFile(), finalGeneratedFile.resolve(expectedFile.getFileName().toString()).toFile()) - ); assertTrue(Files.exists(mapProperties.getFolderPreviewPathLarge().resolve("sludge_test.v0001.png"))); @@ -285,7 +282,6 @@ public void positiveUploadTest() throws IOException { } } - private InputStream loadMapResourceAsStream(String filename) { return MapServiceTest.class.getResourceAsStream("/maps/" + filename); } diff --git a/src/test/resources/featured_mod/mod_info.lua b/src/test/resources/featured_mod/mod_info.lua new file mode 100644 index 000000000..0640a2c79 --- /dev/null +++ b/src/test/resources/featured_mod/mod_info.lua @@ -0,0 +1,28 @@ +-- Forged Alliance Forever mod_info.lua file +-- +-- Documentation for the extended FAF mod_info.lua format can be found here: +-- https://github.com/FAForever/fa/wiki/mod_info.lua-documentation +name = "Forged Alliance Forever" +version = 1337 +_faf_modname = 'balancetesting' +copyright = "Forged Alliance Forever Community" +description = "Forged Alliance Forever extends Forged Alliance, bringing new patches, game modes, units, ladder, and much more!" +author = "Forged Alliance Forever Community" +url = "http://www.faforever.com" +uid = "dcd9a5e5-5444-4266-a016-ccbbff528268" +selectable = false +exclusive = false +ui_only = false +conflicts = {} +mountpoints = { + env = "/env", + loc = '/loc', + schook = '/schook', + effects = '/effects', + lua = '/lua', + meshes = '/meshes', + modules = '/modules', + projectiles = '/projectiles', + textures = '/textures', + units = '/units' +} diff --git a/src/test/resources/featured_mod/someDir/someFile b/src/test/resources/featured_mod/someDir/someFile new file mode 100644 index 000000000..e69de29bb From 1bf883db501a31c841ee3493da9808cd1868e3f8 Mon Sep 17 00:00:00 2001 From: Michel Jung Date: Thu, 9 Mar 2017 00:16:08 +0100 Subject: [PATCH 6/9] Fix #22 exception on invalid oauth client --- .../security/OAuthClientDetailsService.java | 15 +++++-- .../OAuthClientDetailsServiceTest.java | 44 +++++++++++++++++++ 2 files changed, 56 insertions(+), 3 deletions(-) create mode 100644 src/test/java/com/faforever/api/security/OAuthClientDetailsServiceTest.java diff --git a/src/main/java/com/faforever/api/security/OAuthClientDetailsService.java b/src/main/java/com/faforever/api/security/OAuthClientDetailsService.java index 8bf003944..a06c82522 100644 --- a/src/main/java/com/faforever/api/security/OAuthClientDetailsService.java +++ b/src/main/java/com/faforever/api/security/OAuthClientDetailsService.java @@ -1,13 +1,16 @@ package com.faforever.api.security; +import com.faforever.api.client.OAuthClient; import com.faforever.api.client.OAuthClientRepository; import com.faforever.api.config.FafApiProperties; +import com.faforever.api.config.FafApiProperties.Jwt; import org.springframework.cache.annotation.Cacheable; import org.springframework.security.oauth2.provider.ClientDetails; import org.springframework.security.oauth2.provider.ClientDetailsService; import org.springframework.security.oauth2.provider.ClientRegistrationException; import javax.inject.Inject; +import java.util.Optional; public class OAuthClientDetailsService implements ClientDetailsService { @@ -24,9 +27,15 @@ public OAuthClientDetailsService(OAuthClientRepository oAuthClientRepository, Fa @Override @Cacheable(CLIENTS_CACHE_NAME) public ClientDetails loadClientByClientId(String clientId) throws ClientRegistrationException { - OAuthClientDetails clientDetails = new OAuthClientDetails(oAuthClientRepository.findOne(clientId)); - clientDetails.setAccessTokenValiditySeconds(fafApiProperties.getJwt().getAccessTokenValiditySeconds()); - clientDetails.setRefreshTokenValiditySeconds(fafApiProperties.getJwt().getRefreshTokenValiditySeconds()); + OAuthClient oAuthClient = Optional.ofNullable(oAuthClientRepository.findOne(clientId)) + .orElseThrow(() -> new ClientRegistrationException("Unknown client: " + clientId)); + + OAuthClientDetails clientDetails = new OAuthClientDetails(oAuthClient); + + Jwt jwt = fafApiProperties.getJwt(); + clientDetails.setAccessTokenValiditySeconds(jwt.getAccessTokenValiditySeconds()); + clientDetails.setRefreshTokenValiditySeconds(jwt.getRefreshTokenValiditySeconds()); + return clientDetails; } } diff --git a/src/test/java/com/faforever/api/security/OAuthClientDetailsServiceTest.java b/src/test/java/com/faforever/api/security/OAuthClientDetailsServiceTest.java new file mode 100644 index 000000000..ea34f6bcf --- /dev/null +++ b/src/test/java/com/faforever/api/security/OAuthClientDetailsServiceTest.java @@ -0,0 +1,44 @@ +package com.faforever.api.security; + +import com.faforever.api.client.OAuthClient; +import com.faforever.api.client.OAuthClientRepository; +import com.faforever.api.config.FafApiProperties; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.runners.MockitoJUnitRunner; +import org.springframework.security.oauth2.provider.ClientDetails; +import org.springframework.security.oauth2.provider.ClientRegistrationException; + +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.junit.Assert.assertThat; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class OAuthClientDetailsServiceTest { + + private OAuthClientDetailsService instance; + + @Mock + private OAuthClientRepository oAuthClientRepository; + + @Before + public void setUp() throws Exception { + instance = new OAuthClientDetailsService(oAuthClientRepository, new FafApiProperties()); + } + + @Test + public void loadClientByClientId() throws Exception { + when(oAuthClientRepository.findOne("123")).thenReturn(new OAuthClient().setDefaultScope("")); + + ClientDetails result = instance.loadClientByClientId("123"); + + assertThat(result, notNullValue()); + } + + @Test(expected = ClientRegistrationException.class) + public void loadClientByClientIdThrowsClientRegistrationExceptionIfNotExists() throws Exception { + instance.loadClientByClientId("123"); + } +} From 49087e45b3df076fc9348d42a24a5ebf6f6a7adc Mon Sep 17 00:00:00 2001 From: Michel Jung Date: Thu, 9 Mar 2017 00:16:08 +0100 Subject: [PATCH 7/9] Offline git and Instants --- .../java/com/faforever/api/config/GitHubConfig.java | 8 ++++++++ src/main/java/com/faforever/api/data/domain/Game.java | 10 +++++----- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/faforever/api/config/GitHubConfig.java b/src/main/java/com/faforever/api/config/GitHubConfig.java index efbe686cb..7b73c60fe 100644 --- a/src/main/java/com/faforever/api/config/GitHubConfig.java +++ b/src/main/java/com/faforever/api/config/GitHubConfig.java @@ -3,13 +3,21 @@ import org.kohsuke.github.GitHub; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Profile; import java.io.IOException; @Configuration public class GitHubConfig { @Bean + @Profile("!dev") public GitHub gitHub(FafApiProperties fafApiProperties) throws IOException { return GitHub.connectUsingOAuth(fafApiProperties.getGitHub().getAccessToken()); } + + @Bean + @Profile("dev") + public GitHub offlineGitHub() { + return GitHub.offline(); + } } diff --git a/src/main/java/com/faforever/api/data/domain/Game.java b/src/main/java/com/faforever/api/data/domain/Game.java index 6c423cb8f..b2be57a3b 100644 --- a/src/main/java/com/faforever/api/data/domain/Game.java +++ b/src/main/java/com/faforever/api/data/domain/Game.java @@ -15,7 +15,7 @@ import javax.persistence.ManyToOne; import javax.persistence.OneToMany; import javax.persistence.Table; -import java.time.OffsetDateTime; +import java.time.Instant; import java.util.List; @Entity @@ -26,8 +26,8 @@ public class Game { private int id; - private OffsetDateTime startTime; - private OffsetDateTime endTime; + private Instant startTime; + private Instant endTime; private VictoryCondition victoryCondition; private FeaturedMod featuredMod; private Player host; @@ -43,7 +43,7 @@ public int getId() { } @Column(name = "startTime") - public OffsetDateTime getStartTime() { + public Instant getStartTime() { return startTime; } @@ -88,7 +88,7 @@ public List getPlayerStats() { @Formula(value = "(SELECT game_player_stats.scoreTime FROM game_player_stats WHERE game_player_stats.gameId = id ORDER BY game_player_stats.scoreTime DESC LIMIT 1)") @Nullable - public OffsetDateTime getEndTime() { + public Instant getEndTime() { return endTime; } } From 50117489644aef0f5a7c2f6c0627da207d4359de Mon Sep 17 00:00:00 2001 From: Michel Jung Date: Tue, 14 Mar 2017 23:18:30 +0100 Subject: [PATCH 8/9] Clean up as much config as I'm capable right now (I'm pretty tired and it's very tedious) --- build.gradle | 2 +- .../api/config/FafApiProperties.java | 67 ++++++++++++++----- .../security/oauth2/OAuthJwtConfig.java | 2 +- .../com/faforever/api/data/domain/Game.java | 12 ++++ .../api/data/listeners/GameEnricher.java | 24 +++++++ .../LegacyFeaturedModDeploymentTask.java | 4 +- .../com/faforever/api/map/MapService.java | 6 +- .../faforever/api/security/JwtService.java | 2 +- src/main/resources/config/application-dev.yml | 23 +++++-- .../resources/config/application-prod.yml | 24 +++++-- src/main/resources/config/application.yml | 1 + .../LegacyFeaturedModDeploymentTaskTest.java | 4 +- .../com/faforever/api/map/MapServiceTest.java | 10 +-- 13 files changed, 136 insertions(+), 45 deletions(-) create mode 100644 src/main/java/com/faforever/api/data/listeners/GameEnricher.java diff --git a/build.gradle b/build.gradle index 982385d81..58f0852b0 100644 --- a/build.gradle +++ b/build.gradle @@ -106,7 +106,7 @@ task createDockerfile(type: Dockerfile, dependsOn: dockerCopyDistResources) { runCommand 'chmod +x default-cmd.sh' - exposePort 8080 + exposePort 8010 defaultCommand './default-cmd.sh', "/${project.name}-${project.version}/bin/${project.name}" } diff --git a/src/main/java/com/faforever/api/config/FafApiProperties.java b/src/main/java/com/faforever/api/config/FafApiProperties.java index 581c7d2a5..c7ca7f6be 100644 --- a/src/main/java/com/faforever/api/config/FafApiProperties.java +++ b/src/main/java/com/faforever/api/config/FafApiProperties.java @@ -13,15 +13,15 @@ @ConfigurationProperties(prefix = "faf-api", ignoreUnknownFields = false) public class FafApiProperties { /** - * The secret used for JWT token generation. + * The API version. */ - private String jwtSecret = "banana"; - private String version = "dev"; + private String version; private Jwt jwt = new Jwt(); private OAuth2 oAuth2 = new OAuth2(); private Async async = new Async(); private Map map = new Map(); private Mod mod = new Mod(); + private Replay replay = new Replay(); private Clan clan = new Clan(); private FeaturedMods featuredMods = new FeaturedMods(); private GitHub gitHub = new GitHub(); @@ -34,6 +34,10 @@ public static class OAuth2 { @Data public static class Jwt { + /** + * The secret used for JWT token generation. + */ + private String secret = "secret"; private int accessTokenValiditySeconds = 3600; private int refreshTokenValiditySeconds = 3600; } @@ -47,26 +51,58 @@ public static class Async { @Data public static class Map { - private String smallPreviewsUrlFormat = "http://content.faforever.com/faf/vault/map_previews/small/%s"; - private String largePreviewsUrlFormat = "http://content.faforever.com/faf/vault/map_previews/large/%s"; - private String downloadUrlFormat = "http://content.faforever.com/faf/vault/maps/%s"; - private Path folderZipFiles = Paths.get("/content/faf/vault/maps"); - private Path folderPreviewPathSmall = Paths.get("static/map_previews/small"); - private Path folderPreviewPathLarge = Paths.get("static/map_previews/large"); + /** + * For instance {@code http://content.faforever.com/faf/vault/map_previews/small/%s} + */ + private String smallPreviewsUrlFormat; + /** + * For instance {@code http://content.faforever.com/faf/vault/map_previews/large/%s} + */ + private String largePreviewsUrlFormat; + /** + * For instance {@code http://content.faforever.com/faf/vault/maps/%s} + */ + private String downloadUrlFormat; + /** + * The directory in which map files are stored. + */ + private Path targetDirectory = Paths.get("static/maps"); + /** + * The directory in which small map previews are stored. + */ + private Path directoryPreviewPathSmall = Paths.get("static/map_previews/small"); + /** + * The directory in which large map previews are stored. + */ + private Path directoryPreviewPathLarge = Paths.get("static/map_previews/large"); + /** + * The size (in pixels) of small map previews. + */ private int previewSizeSmall = 128; + /** + * The size (in pixels) of large map previews. + */ private int previewSizeLarge = 512; + /** + * Allowed file extensions of uploaded maps. + */ private String[] allowedExtensions = {"zip"}; } @Data public static class Mod { - private String previewUrlFormat = "http://content.faforever.com/faf/vault/mods_thumbs/%s"; - private String downloadUrlFormat = "http://content.faforever.com/faf/vault/mods/%s"; + private String previewUrlFormat; + private String downloadUrlFormat; + } + + @Data + public static class Replay { + private String downloadUrlFormat; } @Data public static class FeaturedMods { - private String fileUrlFormat = "http://content.faforever.com/faf/updaterNew/%s/%s"; + private String fileUrlFormat; } @Data @@ -77,16 +113,15 @@ public static class Clan { @Data public static class GitHub { private String webhookSecret; - private String repositoriesDirectory; private String accessToken; private String deploymentEnvironment; } @Data public static class Deployment { - private String targetFolder; - private String repositoriesFolder; - private String filesFolderFormat = "updates_%s_files"; + private String featuredModsTargetDirectory; + private String repositoriesDirectory; + private String filesDirectoryFormat = "updates_%s_files"; private String forgedAllianceExePath; private List configurations = new ArrayList<>(); diff --git a/src/main/java/com/faforever/api/config/security/oauth2/OAuthJwtConfig.java b/src/main/java/com/faforever/api/config/security/oauth2/OAuthJwtConfig.java index 504c5005f..9b83d85d6 100644 --- a/src/main/java/com/faforever/api/config/security/oauth2/OAuthJwtConfig.java +++ b/src/main/java/com/faforever/api/config/security/oauth2/OAuthJwtConfig.java @@ -37,7 +37,7 @@ public TokenStore tokenStore(JwtAccessTokenConverter jwtAccessTokenConverter) { @Bean protected JwtAccessTokenConverter jwtAccessTokenConverter() { JwtAccessTokenConverter jwtAccessTokenConverter = new JwtAccessTokenConverter(); - jwtAccessTokenConverter.setSigningKey(fafApiProperties.getJwtSecret()); + jwtAccessTokenConverter.setSigningKey(fafApiProperties.getJwt().getSecret()); ((DefaultAccessTokenConverter) jwtAccessTokenConverter.getAccessTokenConverter()).setUserTokenConverter(new FafUserAuthenticationConverter()); return jwtAccessTokenConverter; } diff --git a/src/main/java/com/faforever/api/data/domain/Game.java b/src/main/java/com/faforever/api/data/domain/Game.java index b2be57a3b..495bed7b9 100644 --- a/src/main/java/com/faforever/api/data/domain/Game.java +++ b/src/main/java/com/faforever/api/data/domain/Game.java @@ -1,5 +1,7 @@ package com.faforever.api.data.domain; +import com.faforever.api.data.listeners.GameEnricher; +import com.yahoo.elide.annotation.ComputedAttribute; import com.yahoo.elide.annotation.Include; import lombok.Setter; import org.hibernate.annotations.Formula; @@ -8,6 +10,7 @@ import javax.persistence.Column; import javax.persistence.Entity; +import javax.persistence.EntityListeners; import javax.persistence.EnumType; import javax.persistence.Enumerated; import javax.persistence.Id; @@ -15,6 +18,7 @@ import javax.persistence.ManyToOne; import javax.persistence.OneToMany; import javax.persistence.Table; +import javax.persistence.Transient; import java.time.Instant; import java.util.List; @@ -23,6 +27,7 @@ @Include(rootLevel = true, type = "game") @Immutable @Setter +@EntityListeners(GameEnricher.class) public class Game { private int id; @@ -35,6 +40,7 @@ public class Game { private String name; private Validity validity; private List playerStats; + private String replayUrl; @Id @Column(name = "id") @@ -91,4 +97,10 @@ public List getPlayerStats() { public Instant getEndTime() { return endTime; } + + @Transient + @ComputedAttribute + public String getReplayUrl() { + return replayUrl; + } } diff --git a/src/main/java/com/faforever/api/data/listeners/GameEnricher.java b/src/main/java/com/faforever/api/data/listeners/GameEnricher.java new file mode 100644 index 000000000..0b06db0c3 --- /dev/null +++ b/src/main/java/com/faforever/api/data/listeners/GameEnricher.java @@ -0,0 +1,24 @@ +package com.faforever.api.data.listeners; + +import com.faforever.api.config.FafApiProperties; +import com.faforever.api.data.domain.Game; +import org.springframework.stereotype.Component; + +import javax.inject.Inject; +import javax.persistence.PostLoad; + +@Component +public class GameEnricher { + + private static FafApiProperties fafApiProperties; + + @Inject + public void init(FafApiProperties fafApiProperties) { + GameEnricher.fafApiProperties = fafApiProperties; + } + + @PostLoad + public void enhance(Game game) { + game.setReplayUrl(String.format(fafApiProperties.getReplay().getDownloadUrlFormat(), game.getId())); + } +} diff --git a/src/main/java/com/faforever/api/deployment/LegacyFeaturedModDeploymentTask.java b/src/main/java/com/faforever/api/deployment/LegacyFeaturedModDeploymentTask.java index e158b7acc..dd280594a 100644 --- a/src/main/java/com/faforever/api/deployment/LegacyFeaturedModDeploymentTask.java +++ b/src/main/java/com/faforever/api/deployment/LegacyFeaturedModDeploymentTask.java @@ -86,7 +86,7 @@ public void run() { short version = readModVersion(repositoryDirectory); verifyVersion(version, replaceExisting, modName); - Path targetFolder = Paths.get(deployment.getTargetFolder(), String.format(deployment.getFilesFolderFormat(), modName)); + Path targetFolder = Paths.get(deployment.getFeaturedModsTargetDirectory(), String.format(deployment.getFilesDirectoryFormat(), modName)); List files = packageDirectories(repositoryDirectory, version, fileIds, targetFolder); createPatchedExe(version, fileIds, targetFolder).ifPresent(files::add); @@ -210,7 +210,7 @@ private void checkoutCode(Path repositoryDirectory, String repoUrl, String branc private Path buildRepositoryDirectoryPath(String repoUrl) { String repoDirName = repoUrl.replaceAll(NON_WORD_CHARACTER_PATTERN, ""); - return Paths.get(apiProperties.getDeployment().getRepositoriesFolder(), repoDirName); + return Paths.get(apiProperties.getDeployment().getRepositoriesDirectory(), repoDirName); } private Path toTmpFile(Path targetFile) { diff --git a/src/main/java/com/faforever/api/map/MapService.java b/src/main/java/com/faforever/api/map/MapService.java index 70c52e017..341985f49 100644 --- a/src/main/java/com/faforever/api/map/MapService.java +++ b/src/main/java/com/faforever/api/map/MapService.java @@ -254,7 +254,7 @@ private void updateMapEntities(MapUploadData progressData) { version.setFilename(progressData.getFinalZipName()); progressData.setFinalZipFile( - this.fafApiProperties.getMap().getFolderZipFiles() + this.fafApiProperties.getMap().getTargetDirectory() .resolve(progressData.getFinalZipName())); if (Files.exists(progressData.getFinalZipFile())) { @@ -293,12 +293,12 @@ private void updateLuaFiles(MapUploadData mapData) { private void generatePreview(MapUploadData mapData) { String previewFilename = mapData.getNewFolderName() + ".png"; generateImage( - fafApiProperties.getMap().getFolderPreviewPathSmall().resolve(previewFilename), + fafApiProperties.getMap().getDirectoryPreviewPathSmall().resolve(previewFilename), mapData.getNewMapFolder(), fafApiProperties.getMap().getPreviewSizeSmall()); generateImage( - fafApiProperties.getMap().getFolderPreviewPathLarge().resolve(previewFilename), + fafApiProperties.getMap().getDirectoryPreviewPathLarge().resolve(previewFilename), mapData.getNewMapFolder(), fafApiProperties.getMap().getPreviewSizeLarge()); } diff --git a/src/main/java/com/faforever/api/security/JwtService.java b/src/main/java/com/faforever/api/security/JwtService.java index c8c797e85..c95abe974 100644 --- a/src/main/java/com/faforever/api/security/JwtService.java +++ b/src/main/java/com/faforever/api/security/JwtService.java @@ -19,7 +19,7 @@ public class JwtService { @Inject public JwtService(FafApiProperties fafApiProperties, ObjectMapper objectMapper) { - this.macSigner = new MacSigner(fafApiProperties.getJwtSecret()); + this.macSigner = new MacSigner(fafApiProperties.getJwt().getSecret()); this.objectMapper = objectMapper; } diff --git a/src/main/resources/config/application-dev.yml b/src/main/resources/config/application-dev.yml index 7084d2727..aab932526 100644 --- a/src/main/resources/config/application-dev.yml +++ b/src/main/resources/config/application-dev.yml @@ -1,18 +1,27 @@ faf-api: - jwt-secret: ${JWT_SECRET:banana} + version: dev map: - folder-zip-files: ${MAP_UPLOAD_PATH:build/cache/map/maps} - folder-preview-path-small: ${MAP_PREVIEW_PATH_SMALL:build/cache/map_previews/small} - folder-preview-path-large: ${MAP_PREVIEW_PATH_LARGE:build/cache/map_previews/large} + target-directory: ${MAP_UPLOAD_PATH:build/cache/map/maps} + directory-preview-path-small: ${MAP_PREVIEW_PATH_SMALL:build/cache/map_previews/small} + directory-preview-path-large: ${MAP_PREVIEW_PATH_LARGE:build/cache/map_previews/large} + small-previews-url-format: ${MAP_SMALL_PREVIEWS_URL_FORMAT:http://test.content.faforever.com/faf/vault/map_previews/small/%s.png} + large-previews-url-format: ${MAP_LARGE_PREVIEWS_URL_FORMAT:http://test.content.faforever.com/faf/vault/map_previews/large/%s.png} + download-url-format: ${MAP_DOWNLOAD_URL_FORMAT:http://test.content.faforever.com/faf/vault/maps/%s.zip} + mod: + download-url-format: ${MOD_DOWNLOAD_URL_FORMAT:http://test.content.faforever.com/faf/vault/maps/%s.zip} + preview-url-format: ${MOD_PREVIEW_URL_FORMAT:http://test.content.faforever.com/faf/vault/mods/%s.zip} + replay: + download-url-format: ${REPLAY_DOWNLOAD_URL_FORMAT:http://content.test.faforever.com/faf/vault/replay_vault/replay.php?id=%s} + featured-mods: + file-url-format: ${FEATURED_MOD_URL_FORMAT:http://content.test.faforever.com/faf/updaterNew/%s/%s} git-hub: access-token: ${GITHUB_ACCESS_TOKEN:none} deployment-environment: ${GITHUB_DEPLOYMENT_ENVIRONMENT:development} - repositories-directory: ${REPOSITORIES_DIRECTORY:build/cache/repos} webhook-secret: ${GITHUB_WEBHOOK_SECRET:none} deployment: forged-alliance-exe-path: ${FORGED_ALLIANCE_EXE_PATH} - repositories-folder: ${REPOSITORIES_FOLDER:build/cache/repos} - target-folder: ${DEPLOYMENT_TARGET_FOLDER:build/cache/deployment} + repositories-directory: ${REPOSITORIES_DIRECTORY:build/cache/repos} + featured-mods-target-directory: ${FEATURED_MODS_TARGET_DIRECTORY:build/cache/deployment} spring: datasource: diff --git a/src/main/resources/config/application-prod.yml b/src/main/resources/config/application-prod.yml index debbaccd7..540fd1247 100644 --- a/src/main/resources/config/application-prod.yml +++ b/src/main/resources/config/application-prod.yml @@ -1,18 +1,28 @@ faf-api: - jwt-secret: ${JWT_SECRET} + jwt: + secret: ${JWT_SECRET} map: - folder-zip-files: ${MAP_UPLOAD_PATH} - folder-preview-path-small: ${MAP_PREVIEW_PATH_SMALL} - folder-preview-path-large: ${MAP_PREVIEW_PATH_LARGE} + target-directory: ${MAP_UPLOAD_PATH} + directory-preview-path-small: ${MAP_PREVIEW_PATH_SMALL} + directory-preview-path-large: ${MAP_PREVIEW_PATH_LARGE} + small-previews-url-format: ${MAP_SMALL_PREVIEWS_URL_FORMAT} + large-previews-url-format: ${MAP_LARGE_PREVIEWS_URL_FORMAT} + download-url-format: ${MAP_DOWNLOAD_URL_FORMAT} + mod: + download-url-format: ${MOD_DOWNLOAD_URL_FORMAT} + preview-url-format: ${MOD_PREVIEW_URL_FORMAT} + replay: + download-url-format: ${REPLAY_DOWNLOAD_URL_FORMAT} + featured-mods: + file-url-format: ${FEATURED_MOD_URL_FORMAT} git-hub: access-token: ${GITHUB_ACCESS_TOKEN} deployment-environment: ${GITHUB_DEPLOYMENT_ENVIRONMENT:production} - repositories-directory: ${REPOSITORY_DIRECTORY} webhook-secret: ${GITHUB_WEBHOOK_SECRET} deployment: forged-alliance-exe-path: ${FORGED_ALLIANCE_EXE_PATH} - repositories-folder: ${REPOSITORIES_FOLDER} - target-folder: ${DEPLOYMENT_TARGET_FOLDER} + repositories-directory: ${REPOSITORIES_DIRECTORY} + featured-mods-target-directory: ${FEATURED_MODS_TARGET_DIRECTORY} configurations: - repositoryUrl: https://github.com/FAForever/fa.git branch: deploy/faf diff --git a/src/main/resources/config/application.yml b/src/main/resources/config/application.yml index 546c9099a..3670855ac 100644 --- a/src/main/resources/config/application.yml +++ b/src/main/resources/config/application.yml @@ -28,6 +28,7 @@ spring: active: ${API_PROFILE:dev} server: + # Mind that this is configured in the docker compose file as well (that is, in the gradle script that generates it) port: ${API_PORT:8010} security: diff --git a/src/test/java/com/faforever/api/deployment/LegacyFeaturedModDeploymentTaskTest.java b/src/test/java/com/faforever/api/deployment/LegacyFeaturedModDeploymentTaskTest.java index 94b5293f2..f4fd3045e 100644 --- a/src/test/java/com/faforever/api/deployment/LegacyFeaturedModDeploymentTaskTest.java +++ b/src/test/java/com/faforever/api/deployment/LegacyFeaturedModDeploymentTaskTest.java @@ -50,8 +50,8 @@ public class LegacyFeaturedModDeploymentTaskTest { public void setUp() throws Exception { properties = new FafApiProperties(); Deployment deployment = properties.getDeployment(); - deployment.setRepositoriesFolder(repositoriesFolder.getRoot().getAbsolutePath()); - deployment.setTargetFolder(targetFolder.getRoot().getAbsolutePath()); + deployment.setRepositoriesDirectory(repositoriesFolder.getRoot().getAbsolutePath()); + deployment.setFeaturedModsTargetDirectory(targetFolder.getRoot().getAbsolutePath()); instance = new LegacyFeaturedModDeploymentTask(gitWrapper, featuredModService, properties); } diff --git a/src/test/java/com/faforever/api/map/MapServiceTest.java b/src/test/java/com/faforever/api/map/MapServiceTest.java index 93bfa9c20..9025f3ffe 100644 --- a/src/test/java/com/faforever/api/map/MapServiceTest.java +++ b/src/test/java/com/faforever/api/map/MapServiceTest.java @@ -66,9 +66,9 @@ public class MapServiceTest { public void setUp() { instance = new MapService(fafApiProperties, mapRepository, contentService); mapProperties = new Map() - .setFolderZipFiles(finalDirectory.getRoot().toPath()) - .setFolderPreviewPathLarge(finalDirectory.getRoot().toPath().resolve("large")) - .setFolderPreviewPathSmall(finalDirectory.getRoot().toPath().resolve("small")); + .setTargetDirectory(finalDirectory.getRoot().toPath()) + .setDirectoryPreviewPathLarge(finalDirectory.getRoot().toPath().resolve("large")) + .setDirectoryPreviewPathSmall(finalDirectory.getRoot().toPath().resolve("small")); when(fafApiProperties.getMap()).thenReturn(mapProperties); when(contentService.createTempDir()).thenReturn(temporaryDirectory.getRoot().toPath()); } @@ -276,8 +276,8 @@ public void positiveUploadTest() throws IOException { finalGeneratedFile.resolve(expectedFile.getFileName().toString()).toFile()) ); - assertTrue(Files.exists(mapProperties.getFolderPreviewPathLarge().resolve("sludge_test.v0001.png"))); - assertTrue(Files.exists(mapProperties.getFolderPreviewPathSmall().resolve("sludge_test.v0001.png"))); + assertTrue(Files.exists(mapProperties.getDirectoryPreviewPathLarge().resolve("sludge_test.v0001.png"))); + assertTrue(Files.exists(mapProperties.getDirectoryPreviewPathSmall().resolve("sludge_test.v0001.png"))); } } } From 9eea6afda35d9331f7e622bfb975d09338554223 Mon Sep 17 00:00:00 2001 From: Michel Jung Date: Tue, 14 Mar 2017 23:45:40 +0100 Subject: [PATCH 9/9] Bump version to v0.3.4 --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 58f0852b0..2dc282568 100644 --- a/build.gradle +++ b/build.gradle @@ -17,7 +17,7 @@ apply plugin: 'org.springframework.boot' apply plugin: 'propdeps' group = 'micheljung' -version = '0.3.3' +version = '0.3.4' sourceCompatibility = 1.8 targetCompatibility = 1.8