Skip to content

Commit

Permalink
Merge pull request #910 from amvanbaren/observe-publish-and-downloads
Browse files Browse the repository at this point in the history 
Observe publish endpoint
  • Loading branch information
@amvanbaren
amvanbaren committed May 13, 2024
2 parents 876f0df + d4537ae commit 0b4ec92
Show file tree
Hide file tree
Showing 45 changed files with 1,167 additions and 867 deletions.
405 changes: 229 additions & 176 deletions server/src/main/java/org/eclipse/openvsx/ExtensionProcessor.java
View file

Large diffs are not rendered by default.

75 changes: 44 additions & 31 deletions server/src/main/java/org/eclipse/openvsx/ExtensionService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@
********************************************************************************/
package org.eclipse.openvsx;

import io.micrometer.observation.Observation;
import io.micrometer.observation.ObservationRegistry;
import jakarta.transaction.Transactional;
import jakarta.transaction.Transactional.TxType;
import org.apache.commons.lang3.StringUtils;
Expand All @@ -28,8 +30,10 @@
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.StandardCopyOption;
import java.time.LocalDateTime;
import java.util.LinkedHashSet;
import java.util.concurrent.atomic.AtomicLong;

@Component
public class ExtensionService {
Expand All @@ -40,6 +44,7 @@ public class ExtensionService {
private final SearchUtilService search;
private final CacheService cache;
private final PublishExtensionVersionHandler publishHandler;
private final ObservationRegistry observations;

@Value("${ovsx.publishing.require-license:false}")
boolean requireLicense;
Expand All @@ -48,12 +53,14 @@ public ExtensionService(
RepositoryService repositories,
SearchUtilService search,
CacheService cache,
PublishExtensionVersionHandler publishHandler
PublishExtensionVersionHandler publishHandler,
ObservationRegistry observations
) {
this.repositories = repositories;
this.search = search;
this.cache = cache;
this.publishHandler = publishHandler;
this.observations = observations;
}

@Transactional
Expand All @@ -64,44 +71,50 @@ public ExtensionVersion mirrorVersion(TempFile extensionFile, String signatureNa
}

public ExtensionVersion publishVersion(InputStream content, PersonalAccessToken token) {
var extensionFile = createExtensionFile(content);
var download = doPublish(extensionFile, null, token, TimeUtil.getCurrentUTC(), true);
publishHandler.publishAsync(download, extensionFile, this);
publishHandler.schedulePublicIdJob(download);
return download.getExtension();
return Observation.createNotStarted("ExtensionService#publishVersion", observations).observe(() -> {
var extensionFile = createExtensionFile(content);
var download = doPublish(extensionFile, null, token, TimeUtil.getCurrentUTC(), true);
publishHandler.publishAsync(download, extensionFile, this);
publishHandler.schedulePublicIdJob(download);
return download.getExtension();
});
}

private FileResource doPublish(TempFile extensionFile, String binaryName, PersonalAccessToken token, LocalDateTime timestamp, boolean checkDependencies) {
try (var processor = new ExtensionProcessor(extensionFile)) {
var extVersion = publishHandler.createExtensionVersion(processor, token, timestamp, checkDependencies);
if (requireLicense) {
// Check the extension's license
var license = processor.getLicense(extVersion);
checkLicense(extVersion, license);
return Observation.createNotStarted("ExtensionService#doPublish", observations).observe(() -> {
try (var processor = new ExtensionProcessor(extensionFile, observations)) {
var extVersion = publishHandler.createExtensionVersion(processor, token, timestamp, checkDependencies);
if (requireLicense) {
// Check the extension's license
var license = processor.getLicense(extVersion);
Observation.createNotStarted("ExtensionService#checkLicense", observations).observe(() -> checkLicense(extVersion, license));
}

return processor.getBinary(extVersion, binaryName);
}

return processor.getBinary(extVersion, binaryName);
}
});
}

private TempFile createExtensionFile(InputStream content) {
try (var input = new BufferedInputStream(content)) {
input.mark(0);
var skipped = input.skip(MAX_CONTENT_SIZE + 1);
if (skipped > MAX_CONTENT_SIZE) {
throw new ErrorResultException("The extension package exceeds the size limit of 512 MB.", HttpStatus.PAYLOAD_TOO_LARGE);
return Observation.createNotStarted("ExtensionService#createExtensionFile", observations).observe(() -> {
try (var input = new BufferedInputStream(content)) {
input.mark(0);
var skipped = input.skip(MAX_CONTENT_SIZE + 1);
if (skipped > MAX_CONTENT_SIZE) {
throw new ErrorResultException("The extension package exceeds the size limit of 512 MB.", HttpStatus.PAYLOAD_TOO_LARGE);
}

var extensionFile = new TempFile("extension_", ".vsix");
try(var out = Files.newOutputStream(extensionFile.getPath())) {
input.reset();
input.transferTo(out);
}

return extensionFile;
} catch (IOException e) {
throw new ErrorResultException("Failed to read extension file", e);
}

var extensionFile = new TempFile("extension_", ".vsix");
try(var out = Files.newOutputStream(extensionFile.getPath())) {
input.reset();
input.transferTo(out);
}

return extensionFile;
} catch (IOException e) {
throw new ErrorResultException("Failed to read extension file", e);
}
});
}

private void checkLicense(ExtensionVersion extVersion, FileResource license) {
Expand Down
116 changes: 66 additions & 50 deletions server/src/main/java/org/eclipse/openvsx/ExtensionValidator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@
********************************************************************************/
package org.eclipse.openvsx;

import io.micrometer.observation.Observation;
import io.micrometer.observation.ObservationRegistry;
import org.apache.commons.lang3.StringUtils;
import org.apache.tika.Tika;
import org.apache.tika.mime.MediaType;
Expand Down Expand Up @@ -46,6 +48,11 @@ public class ExtensionValidator {
private final static int GALLERY_COLOR_SIZE = 16;

private final Pattern namePattern = Pattern.compile("[\\w\\-\\+\\$~]+");
private final ObservationRegistry observations;

public ExtensionValidator(ObservationRegistry observations) {
this.observations = observations;
}

public Optional<Issue> validateNamespace(String namespace) {
if (StringUtils.isEmpty(namespace) || namespace.equals("-")) {
Expand Down Expand Up @@ -102,57 +109,63 @@ public List<Issue> validateNamespaceDetails(NamespaceDetailsJson json) {
}

public Optional<Issue> validateExtensionName(String name) {
if (StringUtils.isEmpty(name)) {
return Optional.of(new Issue("Name must not be empty."));
}
if (!namePattern.matcher(name).matches()) {
return Optional.of(new Issue("Invalid extension name: " + name));
}
if (name.length() > DEFAULT_STRING_SIZE) {
return Optional.of(new Issue("The extension name exceeds the current limit of " + DEFAULT_STRING_SIZE + " characters."));
}
return Optional.empty();
return Observation.createNotStarted("ExtensionValidator#validateExtensionName", observations).observe(() -> {
if (StringUtils.isEmpty(name)) {
return Optional.of(new Issue("Name must not be empty."));
}
if (!namePattern.matcher(name).matches()) {
return Optional.of(new Issue("Invalid extension name: " + name));
}
if (name.length() > DEFAULT_STRING_SIZE) {
return Optional.of(new Issue("The extension name exceeds the current limit of " + DEFAULT_STRING_SIZE + " characters."));
}
return Optional.empty();
});
}

public Optional<Issue> validateExtensionVersion(String version) {
var issues = new ArrayList<Issue>();
checkVersion(version, issues);
return issues.isEmpty()
? Optional.empty()
: Optional.of(issues.get(0));
return Observation.createNotStarted("ExtensionValidator#validateExtensionVersion", observations).observe(() -> {
var issues = new ArrayList<Issue>();
checkVersion(version, issues);
return issues.isEmpty()
? Optional.empty()
: Optional.of(issues.get(0));
});
}

public List<Issue> validateMetadata(ExtensionVersion extVersion) {
var issues = new ArrayList<Issue>();
checkVersion(extVersion.getVersion(), issues);
checkTargetPlatform(extVersion.getTargetPlatform(), issues);
checkCharacters(extVersion.getDisplayName(), "displayName", issues);
checkFieldSize(extVersion.getDisplayName(), DEFAULT_STRING_SIZE, "displayName", issues);
checkCharacters(extVersion.getDescription(), "description", issues);
checkFieldSize(extVersion.getDescription(), DESCRIPTION_SIZE, "description", issues);
checkCharacters(extVersion.getCategories(), "categories", issues);
checkFieldSize(extVersion.getCategories(), DEFAULT_STRING_SIZE, "categories", issues);
checkCharacters(extVersion.getTags(), "keywords", issues);
checkFieldSize(extVersion.getTags(), DEFAULT_STRING_SIZE, "keywords", issues);
checkCharacters(extVersion.getLicense(), "license", issues);
checkFieldSize(extVersion.getLicense(), DEFAULT_STRING_SIZE, "license", issues);
checkURL(extVersion.getHomepage(), "homepage", issues);
checkFieldSize(extVersion.getHomepage(), DEFAULT_STRING_SIZE, "homepage", issues);
checkURL(extVersion.getRepository(), "repository", issues);
checkFieldSize(extVersion.getRepository(), DEFAULT_STRING_SIZE, "repository", issues);
checkURL(extVersion.getBugs(), "bugs", issues);
checkFieldSize(extVersion.getBugs(), DEFAULT_STRING_SIZE, "bugs", issues);
checkInvalid(extVersion.getMarkdown(), s -> !MARKDOWN_VALUES.contains(s), "markdown", issues,
MARKDOWN_VALUES.toString());
checkCharacters(extVersion.getGalleryColor(), "galleryBanner.color", issues);
checkFieldSize(extVersion.getGalleryColor(), GALLERY_COLOR_SIZE, "galleryBanner.color", issues);
checkInvalid(extVersion.getGalleryTheme(), s -> !GALLERY_THEME_VALUES.contains(s), "galleryBanner.theme", issues,
GALLERY_THEME_VALUES.toString());
checkFieldSize(extVersion.getLocalizedLanguages(), DEFAULT_STRING_SIZE, "localizedLanguages", issues);
checkInvalid(extVersion.getQna(), s -> !QNA_VALUES.contains(s) && isInvalidURL(s), "qna", issues,
QNA_VALUES.toString() + " or a URL");
checkFieldSize(extVersion.getQna(), DEFAULT_STRING_SIZE, "qna", issues);
return issues;
return Observation.createNotStarted("ExtensionValidator#validateMetadata", observations).observe(() -> {
var issues = new ArrayList<Issue>();
checkVersion(extVersion.getVersion(), issues);
checkTargetPlatform(extVersion.getTargetPlatform(), issues);
checkCharacters(extVersion.getDisplayName(), "displayName", issues);
checkFieldSize(extVersion.getDisplayName(), DEFAULT_STRING_SIZE, "displayName", issues);
checkCharacters(extVersion.getDescription(), "description", issues);
checkFieldSize(extVersion.getDescription(), DESCRIPTION_SIZE, "description", issues);
checkCharacters(extVersion.getCategories(), "categories", issues);
checkFieldSize(extVersion.getCategories(), DEFAULT_STRING_SIZE, "categories", issues);
checkCharacters(extVersion.getTags(), "keywords", issues);
checkFieldSize(extVersion.getTags(), DEFAULT_STRING_SIZE, "keywords", issues);
checkCharacters(extVersion.getLicense(), "license", issues);
checkFieldSize(extVersion.getLicense(), DEFAULT_STRING_SIZE, "license", issues);
checkURL(extVersion.getHomepage(), "homepage", issues);
checkFieldSize(extVersion.getHomepage(), DEFAULT_STRING_SIZE, "homepage", issues);
checkURL(extVersion.getRepository(), "repository", issues);
checkFieldSize(extVersion.getRepository(), DEFAULT_STRING_SIZE, "repository", issues);
checkURL(extVersion.getBugs(), "bugs", issues);
checkFieldSize(extVersion.getBugs(), DEFAULT_STRING_SIZE, "bugs", issues);
checkInvalid(extVersion.getMarkdown(), s -> !MARKDOWN_VALUES.contains(s), "markdown", issues,
MARKDOWN_VALUES.toString());
checkCharacters(extVersion.getGalleryColor(), "galleryBanner.color", issues);
checkFieldSize(extVersion.getGalleryColor(), GALLERY_COLOR_SIZE, "galleryBanner.color", issues);
checkInvalid(extVersion.getGalleryTheme(), s -> !GALLERY_THEME_VALUES.contains(s), "galleryBanner.theme", issues,
GALLERY_THEME_VALUES.toString());
checkFieldSize(extVersion.getLocalizedLanguages(), DEFAULT_STRING_SIZE, "localizedLanguages", issues);
checkInvalid(extVersion.getQna(), s -> !QNA_VALUES.contains(s) && isInvalidURL(s), "qna", issues,
QNA_VALUES.toString() + " or a URL");
checkFieldSize(extVersion.getQna(), DEFAULT_STRING_SIZE, "qna", issues);
return issues;
});
}

private void checkVersion(String version, List<Issue> issues) {
Expand All @@ -163,11 +176,14 @@ private void checkVersion(String version, List<Issue> issues) {
if (version.equals(VersionAlias.LATEST) || version.equals(VersionAlias.PRE_RELEASE) || version.equals("reviews")) {
issues.add(new Issue("The version string '" + version + "' is reserved."));
}
try {
SemanticVersion.parse(version);
} catch (RuntimeException e) {
issues.add(new Issue(e.getMessage()));
}

Observation.createNotStarted("SemanticVersion#parse", observations).observe(() -> {
try {
SemanticVersion.parse(version);
} catch (RuntimeException e) {
issues.add(new Issue(e.getMessage()));
}
});
}

private void checkTargetPlatform(String targetPlatform, List<Issue> issues) {
Expand Down
2 changes: 0 additions & 2 deletions server/src/main/java/org/eclipse/openvsx/IExtensionRegistry.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@
********************************************************************************/
package org.eclipse.openvsx;

import io.micrometer.observation.annotation.Observed;
import org.eclipse.openvsx.json.*;
import org.eclipse.openvsx.search.ISearchService;
import org.springframework.http.ResponseEntity;
Expand Down Expand Up @@ -39,7 +38,6 @@ public interface IExtensionRegistry {

QueryResultJson queryV2(QueryRequestV2 request);

@Observed
NamespaceDetailsJson getNamespaceDetails(String namespace);

ResponseEntity<byte[]> getNamespaceLogo(String namespaceName, String fileName);
Expand Down
Loading

0 comments on commit 0b4ec92

Please sign in to comment.