Skip to content

Commit aa920ea

Browse files
liron-shalomliron-shalom
authored
BI-516 - Add Go extractor to support for Go run and publish. (#291)
1 parent 011a1f6 commit aa920ea

File tree

22 files changed

+986
-1
lines changed

22 files changed

+986
-1
lines changed

Diff for: ‎appveyor.yml

+2-1
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,10 @@
1-
stack: jdk 8, node 8
1+
stack: jdk 8, node 8, go 1.13
22
skip_tags: true
33
environment:
44
matrix:
55
- APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
66
JAVA_HOME: C:\Program Files\Java\jdk1.8.0
7+
GOPATH: c:\gopath
78
- APPVEYOR_BUILD_WORKER_IMAGE: Ubuntu
89

910
BITESTS_ARTIFACTORY_URL:

Diff for: ‎build-info-extractor-go/src/main/java/org/jfrog/build/extractor/go/extractor/GoCommand.java

+107
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
package org.jfrog.build.extractor.go.extractor;
2+
3+
import org.apache.commons.lang.StringUtils;
4+
import org.jfrog.build.api.Artifact;
5+
import org.jfrog.build.api.Build;
6+
import org.jfrog.build.api.Dependency;
7+
import org.jfrog.build.api.Module;
8+
import org.jfrog.build.api.builder.ModuleBuilder;
9+
import org.jfrog.build.api.util.Log;
10+
import org.jfrog.build.client.ArtifactoryVersion;
11+
import org.jfrog.build.extractor.clientConfiguration.ArtifactoryBuildInfoClientBuilder;
12+
import org.jfrog.build.extractor.clientConfiguration.ArtifactoryClientBuilderBase;
13+
import org.jfrog.build.extractor.clientConfiguration.client.ArtifactoryBaseClient;
14+
import org.jfrog.build.util.VersionCompatibilityType;
15+
import org.jfrog.build.util.VersionException;
16+
17+
import java.io.File;
18+
import java.io.IOException;
19+
import java.io.Serializable;
20+
import java.nio.charset.StandardCharsets;
21+
import java.nio.file.Files;
22+
import java.nio.file.Path;
23+
import java.nio.file.Paths;
24+
import java.util.Arrays;
25+
import java.util.List;
26+
import java.util.regex.Pattern;
27+
import java.util.stream.Stream;
28+
29+
/**
30+
* Base class for go build and go publish commands.
31+
*/
32+
abstract class GoCommand implements Serializable {
33+
34+
protected static final String SHA1 = "SHA1";
35+
protected static final String MD5 = "MD5";
36+
protected static final String LOCAL_GO_MOD_FILENAME = "go.mod";
37+
private static final long serialVersionUID = 1L;
38+
private static final ArtifactoryVersion MIN_SUPPORTED_ARTIFACTORY_VERSION = new ArtifactoryVersion("6.10.0");
39+
static final Pattern MODULE_NAME_REGEX_PATTERN = Pattern.compile("module \"?([\\w\\.@:%_\\+-.~#?&]+/?.+\\w)");
40+
41+
ArtifactoryClientBuilderBase clientBuilder;
42+
Path path;
43+
String moduleName;
44+
Log logger;
45+
46+
GoCommand(ArtifactoryBuildInfoClientBuilder clientBuilder, Path path, Log logger) throws IOException {
47+
this.clientBuilder = clientBuilder;
48+
this.logger = logger;
49+
this.path = path;
50+
parseModuleName();
51+
}
52+
53+
private void parseModuleName() throws IOException {
54+
Path modPath = Paths.get(path.toString(), LOCAL_GO_MOD_FILENAME);
55+
Stream<String> stream = Files.lines(modPath, StandardCharsets.UTF_8);
56+
stream.forEach(line -> {
57+
if (MODULE_NAME_REGEX_PATTERN.matcher(line).matches()) {
58+
moduleName = line;
59+
}
60+
});
61+
moduleName = StringUtils.split(moduleName, " ")[1];
62+
}
63+
64+
protected void preparePrerequisites(String repo, ArtifactoryBaseClient client) throws VersionException, IOException {
65+
validateArtifactoryVersion(client);
66+
validateRepoExists(repo, client, "The provided repo must be specified");
67+
}
68+
69+
private void validateArtifactoryVersion(ArtifactoryBaseClient client) throws VersionException {
70+
ArtifactoryVersion version = client.getArtifactoryVersion();
71+
if (version.isNotFound()) {
72+
String message = "Couldn't execute go task. Check connection with Artifactory.";
73+
throw new VersionException(message, VersionCompatibilityType.NOT_FOUND);
74+
}
75+
if (!version.isAtLeast(MIN_SUPPORTED_ARTIFACTORY_VERSION)) {
76+
String message = String.format("Couldn't execute Go task. Artifactory version is %s but must be at least %s.", version.toString(), MIN_SUPPORTED_ARTIFACTORY_VERSION.toString());
77+
throw new VersionException(message, VersionCompatibilityType.INCOMPATIBLE);
78+
}
79+
}
80+
81+
private void validateRepoExists(String repo, ArtifactoryBaseClient client, String repoNotSpecifiedMsg) throws IOException {
82+
if (StringUtils.isBlank(repo)) {
83+
throw new IllegalArgumentException(repoNotSpecifiedMsg);
84+
}
85+
if (!client.isRepoExist(repo)) {
86+
throw new IOException("Repo " + repo + " doesn't exist");
87+
}
88+
}
89+
90+
protected Build createBuild(List<Artifact> artifacts, List<Dependency> dependencies) {
91+
ModuleBuilder moduleBuilder = new ModuleBuilder().id(moduleName);
92+
if (artifacts != null) {
93+
moduleBuilder.artifacts(artifacts);
94+
}
95+
if (dependencies != null) {
96+
moduleBuilder.dependencies(dependencies);
97+
}
98+
List<Module> modules = Arrays.asList(moduleBuilder.build());
99+
Build build = new Build();
100+
build.setModules(modules);
101+
return build;
102+
}
103+
104+
protected String getModFilePath() {
105+
return path.toString() + File.separator + LOCAL_GO_MOD_FILENAME;
106+
}
107+
}

Diff for: ‎build-info-extractor-go/src/main/java/org/jfrog/build/extractor/go/extractor/GoPublish.java

+176
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,176 @@
1+
package org.jfrog.build.extractor.go.extractor;
2+
3+
import com.google.common.collect.ArrayListMultimap;
4+
import org.apache.commons.compress.archivers.zip.ZipFile;
5+
import org.apache.commons.io.FileUtils;
6+
import org.jfrog.build.api.Artifact;
7+
import org.jfrog.build.api.Build;
8+
import org.jfrog.build.api.builder.ArtifactBuilder;
9+
import org.jfrog.build.api.util.FileChecksumCalculator;
10+
import org.jfrog.build.api.util.Log;
11+
import org.jfrog.build.client.ArtifactoryUploadResponse;
12+
import org.jfrog.build.extractor.clientConfiguration.ArtifactoryBuildInfoClientBuilder;
13+
import org.jfrog.build.extractor.clientConfiguration.client.ArtifactoryBuildInfoClient;
14+
import org.jfrog.build.extractor.clientConfiguration.deploy.DeployDetails;
15+
16+
import java.io.File;
17+
import java.io.IOException;
18+
import java.nio.file.Files;
19+
import java.nio.file.Path;
20+
import java.util.*;
21+
import java.util.stream.Collectors;
22+
import java.util.zip.ZipEntry;
23+
import java.util.zip.ZipOutputStream;
24+
import java.io.FileOutputStream;
25+
import java.time.Instant;
26+
27+
import com.fasterxml.jackson.databind.ObjectMapper;
28+
29+
@SuppressWarnings({"unused", "WeakerAccess"})
30+
public class GoPublish extends GoCommand {
31+
32+
private static final String LOCAL_PKG_FILENAME = "package";
33+
private static final String LOCAL_TMP_PKG_PREFIX = "tmp.";
34+
private static final String LOCAL_INFO_FILENAME = "package.info";
35+
private static final String GO_VERSION_PREFIX = "v";
36+
private static final String PKG_ZIP_FILE_EXTENSION = "zip";
37+
private static final String PKG_MOD_FILE_EXTENSION = "mod";
38+
private static final String PKG_INFO_FILE_EXTENSION = "info";
39+
40+
private ArrayListMultimap<String, String> properties;
41+
private List<Artifact> artifactList = new ArrayList<>();
42+
private String deploymentRepo;
43+
private String version;
44+
45+
/**
46+
* Publish go package.
47+
*
48+
* @param clientBuilder - Client builder for deployment.
49+
* @param properties - Properties to set on each deployed artifact (Build name, Build number, etc...).
50+
* @param repo - Artifactory's repository for deployment.
51+
* @param path - Path to directory contains go.mod.
52+
* @param version - The package's version.
53+
* @param logger - The logger.
54+
*/
55+
public GoPublish(ArtifactoryBuildInfoClientBuilder clientBuilder, ArrayListMultimap<String, String> properties, String repo, Path path, String version, Log logger) throws IOException {
56+
super(clientBuilder, path, logger);
57+
this.properties = properties;
58+
this.deploymentRepo = repo;
59+
this.version = GO_VERSION_PREFIX + version;
60+
}
61+
62+
public Build execute() {
63+
try (ArtifactoryBuildInfoClient artifactoryClient = (ArtifactoryBuildInfoClient) clientBuilder.build()) {
64+
preparePrerequisites(deploymentRepo, artifactoryClient);
65+
publishPkg(artifactoryClient);
66+
return createBuild(artifactList, null);
67+
} catch (Exception e) {
68+
logger.error(e.getMessage(), e);
69+
}
70+
return null;
71+
}
72+
73+
/**
74+
* The deployment of a Go package requires 3 files:
75+
* 1. zip file of source files.
76+
* 2. go.mod file.
77+
* 3. go.info file.
78+
*/
79+
private void publishPkg(ArtifactoryBuildInfoClient client) throws Exception {
80+
createAndDeployZip(client);
81+
deployGoMod(client);
82+
createAndDeployInfo(client);
83+
}
84+
85+
private void createAndDeployZip(ArtifactoryBuildInfoClient client) throws Exception {
86+
// First, we create a temporary zip file of all project files.
87+
File tmpZipFile = archiveProjectDir();
88+
89+
// Second, filter the raw zip file according to Go rules and create deployable zip can be later resolved.
90+
// We use the same code as Artifactory when he resolve a Go module directly from Github.
91+
File deployableZipFile = File.createTempFile(LOCAL_PKG_FILENAME, PKG_ZIP_FILE_EXTENSION, path.toFile());
92+
try (GoZipBallStreamer pkgArchiver = new GoZipBallStreamer(new ZipFile(tmpZipFile), moduleName, version, logger)) {
93+
pkgArchiver.writeDeployableZip(deployableZipFile);
94+
Artifact deployedPackage = deploy(client, deployableZipFile, PKG_ZIP_FILE_EXTENSION);
95+
artifactList.add(deployedPackage);
96+
} finally {
97+
Files.deleteIfExists(tmpZipFile.toPath());
98+
Files.deleteIfExists(deployableZipFile.toPath());
99+
}
100+
}
101+
102+
private void deployGoMod(ArtifactoryBuildInfoClient client) throws Exception {
103+
String modLocalPath = getModFilePath();
104+
Artifact deployedMod = deploy(client, new File(modLocalPath), PKG_MOD_FILE_EXTENSION);
105+
artifactList.add(deployedMod);
106+
}
107+
108+
private void createAndDeployInfo(ArtifactoryBuildInfoClient client) throws Exception {
109+
String infoLocalPath = path.toString() + File.separator + LOCAL_INFO_FILENAME;
110+
File infoFile = writeInfoFile(infoLocalPath);
111+
Artifact deployedInfo = deploy(client, infoFile, PKG_INFO_FILE_EXTENSION);
112+
artifactList.add(deployedInfo);
113+
infoFile.delete();
114+
}
115+
116+
private File archiveProjectDir() throws IOException {
117+
File zipFile = File.createTempFile(LOCAL_TMP_PKG_PREFIX + LOCAL_PKG_FILENAME, PKG_ZIP_FILE_EXTENSION, path.toFile());
118+
119+
try (ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(zipFile))) {
120+
List<Path> pathsList = Files.walk(path)
121+
.filter(p -> !Files.isDirectory(p) && !p.toFile().getName().equals(zipFile.getName()))
122+
.collect(Collectors.toList());
123+
for (Path filePath : pathsList) {
124+
ZipEntry zipEntry = new ZipEntry(path.relativize(filePath).toString());
125+
zos.putNextEntry(zipEntry);
126+
Files.copy(filePath, zos);
127+
zos.closeEntry();
128+
}
129+
}
130+
131+
return zipFile;
132+
}
133+
134+
/**
135+
* pkg info is a json file containing:
136+
* 1. The package's version.
137+
* 2. The package creation timestamp.
138+
*/
139+
private File writeInfoFile(String localInfoPath) throws IOException{
140+
File infoFile = new File(localInfoPath);
141+
ObjectMapper mapper = new ObjectMapper();
142+
Map<String, String> infoMap = new HashMap();
143+
Date date = new Date();
144+
Instant instant = date.toInstant();
145+
146+
infoMap.put("Version", version);
147+
infoMap.put("Time", instant.toString());
148+
149+
mapper.writeValue(infoFile, infoMap);
150+
return infoFile;
151+
}
152+
153+
/**
154+
* Deploy pkg file and add it as an buildInfo's artifact
155+
*/
156+
private Artifact deploy(ArtifactoryBuildInfoClient client, File deployedFile, String extension) throws Exception {
157+
String artifactName = version + "." + extension;
158+
Map<String, String> checksums = FileChecksumCalculator.calculateChecksums(deployedFile, MD5, SHA1);
159+
DeployDetails deployDetails = new DeployDetails.Builder()
160+
.file(deployedFile)
161+
.targetRepository(deploymentRepo)
162+
.addProperties(properties)
163+
.artifactPath(moduleName + "/@v/" + artifactName)
164+
.md5(checksums.get(MD5)).sha1(checksums.get(SHA1))
165+
.build();
166+
167+
ArtifactoryUploadResponse response = client.deployArtifact(deployDetails);
168+
169+
Artifact deployedArtifact = new ArtifactBuilder(moduleName + ":" + artifactName)
170+
.md5(response.getChecksums().getMd5())
171+
.sha1(response.getChecksums().getSha1())
172+
.build();
173+
174+
return deployedArtifact;
175+
}
176+
}

0 commit comments

Comments
 (0)