diff --git a/Dockerfile b/Dockerfile
deleted file mode 100644
index e0da9ba2..00000000
--- a/Dockerfile
+++ /dev/null
@@ -1,11 +0,0 @@
-FROM sourcegraph/lsif-java
-COPY bin/packagehub.sh /packagehub.sh
-RUN chmod +x /packagehub.sh
-RUN git config --global user.email "you@example.com"
-RUN git config --global user.name "Your Name"
-RUN git config --global http.postBuffer 1048576000
-RUN curl -L https://sourcegraph.com/.api/src-cli/src_linux_amd64 -o /src
-RUN chmod +x /src
-RUN /coursier bootstrap -r sonatype:snapshots com.sourcegraph:packagehub_2.13:0.5.2-1-c8910a01-SNAPSHOT -o /packagehub
-ENV COURSIER_REPOSITORIES=central|https://maven.google.com/|jitpack
-CMD ["/packagehub.sh"]
diff --git a/Dockerfile.template b/Dockerfile.template
deleted file mode 100644
index 17b23ec3..00000000
--- a/Dockerfile.template
+++ /dev/null
@@ -1,11 +0,0 @@
-FROM sourcegraph/lsif-java
-COPY bin/packagehub.sh /packagehub.sh
-RUN chmod +x /packagehub.sh
-RUN git config --global user.email "you@example.com"
-RUN git config --global user.name "Your Name"
-RUN git config --global http.postBuffer 1048576000
-RUN curl -L https://sourcegraph.com/.api/src-cli/src_linux_amd64 -o /src
-RUN chmod +x /src
-RUN /coursier bootstrap -r sonatype:snapshots com.sourcegraph:packagehub_2.13:VERSION -o /packagehub
-ENV COURSIER_REPOSITORIES=central|https://maven.google.com/|jitpack
-CMD ["/packagehub.sh"]
diff --git a/build.sbt b/build.sbt
index 17f1449a..42ac052d 100644
--- a/build.sbt
+++ b/build.sbt
@@ -219,10 +219,6 @@ lazy val cli = project
)
.enablePlugins(NativeImagePlugin, BuildInfoPlugin)
.dependsOn(lsif)
-commands +=
- Command.command("dockerfile") { s =>
- "commitall" :: "reload" :: "packagehub/dockerfileUpdate" :: "publish" :: s
- }
def commitAll(): Unit = {
import scala.sys.process._
@@ -236,30 +232,6 @@ commands +=
s
}
-lazy val packagehub = project
- .in(file("packagehub"))
- .settings(
- moduleName := "packagehub",
- (Compile / mainClass) := Some("com.sourcegraph.packagehub.PackageHub"),
- TaskKey[Unit]("dockerfileUpdate") := {
- val template = IO.read(file("Dockerfile.template"))
- IO.write(file("Dockerfile"), template.replace("VERSION", version.value))
- commitAll()
- },
- libraryDependencies ++=
- List(
- "com.google.cloud.sql" % "postgres-socket-factory" % "1.3.4",
- "com.zaxxer" % "HikariCP" % "5.0.0",
- "org.flywaydb" % "flyway-core" % "8.0.1",
- "org.postgresql" % "postgresql" % "42.2.23",
- "org.rauschig" % "jarchivelib" % "1.2.0",
- "org.scalameta" %% "scalameta" % V.scalameta,
- "com.lihaoyi" %% "cask" % "0.7.8"
- )
- )
- .enablePlugins(AssemblyPlugin)
- .dependsOn(cli)
-
commands +=
Command.command("nativeImageProfiled") { s =>
val targetroot =
@@ -400,7 +372,7 @@ lazy val unit = project
),
buildInfoPackage := "tests"
)
- .dependsOn(plugin, cli, packagehub)
+ .dependsOn(plugin, cli)
.enablePlugins(BuildInfoPlugin)
lazy val buildTools = project
diff --git a/packagehub/src/main/resources/db/migration/V1__Create_tables.sql b/packagehub/src/main/resources/db/migration/V1__Create_tables.sql
deleted file mode 100644
index 4f6c4f0a..00000000
--- a/packagehub/src/main/resources/db/migration/V1__Create_tables.sql
+++ /dev/null
@@ -1,7 +0,0 @@
-CREATE TABLE packages (
- id VARCHAR(1000) NOT NULL PRIMARY KEY
-);
-
-CREATE TABLE indexed_packages (
- id VARCHAR(1000) NOT NULL PRIMARY KEY
-);
diff --git a/packagehub/src/main/scala/com/sourcegraph/packagehub/EmptyJsonCodec.scala b/packagehub/src/main/scala/com/sourcegraph/packagehub/EmptyJsonCodec.scala
deleted file mode 100644
index fad85ded..00000000
--- a/packagehub/src/main/scala/com/sourcegraph/packagehub/EmptyJsonCodec.scala
+++ /dev/null
@@ -1,22 +0,0 @@
-package com.sourcegraph.packagehub
-
-import moped.json.DecodingContext
-import moped.json.ErrorResult
-import moped.json.JsonCodec
-import moped.json.JsonElement
-import moped.json.JsonString
-import moped.json.Result
-import moped.macros.ClassShape
-import moped.reporters.Diagnostic
-
-/**
- * Codec that always fails the decoding step.
- *
- * Useful for types that cannot be configured from the command-line.
- */
-class EmptyJsonCodec[T] extends JsonCodec[T] {
- def decode(context: DecodingContext): Result[T] =
- ErrorResult(Diagnostic.error(s"not supported: $context"))
- def encode(value: T): JsonElement = JsonString(value.toString())
- def shape: ClassShape = ClassShape.empty
-}
diff --git a/packagehub/src/main/scala/com/sourcegraph/packagehub/Package.scala b/packagehub/src/main/scala/com/sourcegraph/packagehub/Package.scala
deleted file mode 100644
index 29279049..00000000
--- a/packagehub/src/main/scala/com/sourcegraph/packagehub/Package.scala
+++ /dev/null
@@ -1,178 +0,0 @@
-package com.sourcegraph.packagehub
-
-import java.nio.file.Path
-import java.nio.file.Paths
-
-import scala.util.control.NonFatal
-
-import com.sourcegraph.lsif_java.Dependencies
-import coursier.core.Dependency
-import coursier.core.Module
-import coursier.core.ModuleName
-import coursier.core.Organization
-import ujson.Obj
-
-/**
- * Package represents a published library such as a Java artifact, or the JDK.
- *
- * @param id
- * unique representation for this package that does not include the forward
- * slash character. Can be used as the primary key in a relational database.
- * Should ideally be human-readable and be easy to parse.
- * @param path
- * relative URL of this package.
- */
-sealed abstract class Package(
- val id: String,
- val path: String,
- val version: String
-) {
- def toJsonRepo: Obj = Obj("Name" -> path, "URI" -> s"/repos/$path")
- def relativePath: Path = Paths.get(path)
-}
-object Package {
- def npm(name: String, version: String): NpmPackage = {
- NpmPackage(name, version)
- }
- def jdk(version: String): JdkPackage = {
- JdkPackage(version)
- }
- def maven(org: String, name: String, version: String): MavenPackage = {
- MavenPackage(
- Dependency(
- Module(Organization(org), ModuleName(name), Map.empty),
- version
- )
- )
- }
- def parse(value: String): Package = {
- value match {
- case s"jdk:$version" =>
- JdkPackage(version)
- case s"maven:$library" =>
- val Right(dep) = Dependencies.parseDependencyEither(library)
- MavenPackage(dep)
- case s"npm:$name:$version" =>
- NpmPackage(name, version)
- }
- }
- def fromPath(path: List[String]): Option[(Package, List[String])] =
- path match {
- case "maven" :: org :: name :: version :: requestPath =>
- Some(Package.maven(org, name, version) -> requestPath)
- case "jdk" :: version :: requestPath =>
- Some(Package.jdk(version) -> requestPath)
- case "npm" :: GitRequestPrefix(parts, requestPath) =>
- val name = parts.init.mkString("/")
- val version = parts.last
- val actualName =
- if (name.startsWith("-"))
- "@" + name.stripPrefix("-")
- else
- name
- Some(Package.npm(actualName, version) -> requestPath)
- case _ =>
- None
- }
-
- object GitRequestPrefix {
- private val suffixes = List(
- ".git" :: "info" :: "refs" :: Nil,
- "info" :: "refs" :: Nil,
- ".git" :: "git-upload-pack" :: Nil,
- "git-upload-pack" :: Nil
- )
- def unapply(path: List[String]): Option[(List[String], List[String])] = {
- suffixes.find(path.endsWith) match {
- case Some(suffix) =>
- Some(path.dropRight(suffix.length) -> suffix)
- case None =>
- None
- }
- }
- }
- def fromString(value: String, coursier: String): Either[String, Package] = {
- value match {
- case s"maven:$library" =>
- try {
- val deps = Dependencies
- .resolveDependencies(List(library), transitive = false)
- if (deps.sources.isEmpty) {
- Left(s"no sources for package '$library'")
- } else if (deps.dependencies.length != 1) {
- Left(s"expected a single dependency, got ${deps.dependencies}")
- } else {
- Right(MavenPackage(deps.dependencies.head))
- }
- } catch {
- case NonFatal(e) =>
- Left(e.getMessage())
- }
- case s"jdk:$version" =>
- val exit = os
- .proc(coursier, "java-home", "--jvm", version)
- .call(check = false)
- if (exit.exitCode == 0)
- Right(JdkPackage(version))
- else
- Left(exit.out.trim())
- case s"npm:$name:$version" =>
- try {
- val out = os
- .proc("npm", "info", s"$name@$version")
- .call(check = false)
- .out
- .trim()
- if (out.nonEmpty)
- Right(NpmPackage(name, version))
- else
- Left(s"no such npm package: $name@$version")
- } catch {
- case NonFatal(e) =>
- Left(e.getMessage)
- }
- case other =>
- Left(
- s"unsupported package '$other'. To fix this problem, use a valid syntax " +
- s"such as 'maven:ORGANIZATION:ARTIFACT_NAME_VERSION' for Java libraries."
- )
- }
- }
-}
-
-/**
- * A Java library that is published "Maven style".
- *
- * The most widely used Maven package host is "Maven Central"
- * https://search.maven.org/. Most companies self-host an Artifactory instance
- * to publish internal libraries and to proxy Maven Central.
- */
-case class MavenPackage(dep: Dependency)
- extends Package(
- s"maven:${dep.module.repr}:${dep.version}",
- s"maven/${dep.module.organization.value}/${dep.module.name.value}/${dep.version}",
- dep.version
- ) {
- def repr = id.stripPrefix("maven:")
-}
-
-/**
- * The Java standard library.
- *
- * The sources of the Java standard library are typically available under
- * JAVA_HOME.
- */
-case class JdkPackage(override val version: String)
- extends Package(s"jdk:${version}", s"jdk/${version}", version) {
- def repr = id.stripPrefix("jdk:")
-}
-
-case class NpmPackage(packageName: String, override val version: String)
- extends Package(
- s"npm:$packageName:$version",
- s"npm/$packageName/$version".replace("@", "-"),
- version
- ) {
- def npmName = s"$packageName@$version"
- def tarballFilename = s"$packageName-$version.tgz".replace('/', '-')
-}
diff --git a/packagehub/src/main/scala/com/sourcegraph/packagehub/PackageActor.scala b/packagehub/src/main/scala/com/sourcegraph/packagehub/PackageActor.scala
deleted file mode 100644
index 9bcd29e2..00000000
--- a/packagehub/src/main/scala/com/sourcegraph/packagehub/PackageActor.scala
+++ /dev/null
@@ -1,440 +0,0 @@
-package com.sourcegraph.packagehub
-
-import java.io.File
-import java.io.IOException
-import java.net.URL
-import java.nio.file.FileSystems
-import java.nio.file.FileVisitResult
-import java.nio.file.Files
-import java.nio.file.Path
-import java.nio.file.Paths
-import java.nio.file.SimpleFileVisitor
-import java.nio.file.StandardCopyOption
-import java.nio.file.StandardOpenOption
-import java.nio.file.attribute.BasicFileAttributes
-
-import scala.collection.mutable.ArrayBuffer
-import scala.collection.mutable.ListBuffer
-import scala.jdk.CollectionConverters._
-
-import scala.meta.internal.io.FileIO
-import scala.meta.io.AbsolutePath
-
-import castor.Context
-import castor.SimpleActor
-import com.sourcegraph.io.DeleteVisitor
-import com.sourcegraph.lsif_java.Dependencies
-import com.sourcegraph.lsif_java.buildtools.LsifBuildTool
-import com.sourcegraph.lsif_semanticdb.JavaVersion
-import coursier.core.Dependency
-import org.rauschig.jarchivelib.ArchiverFactory
-import os.CommandResult
-import os.ProcessOutput.Readlines
-import os.Shellable
-import os.SubProcess
-import ujson.Arr
-import ujson.Obj
-import ujson.Str
-
-/**
- * Actor that creates git repos from package sources and (optionally LSIF
- * indexes the sources.
- *
- * This class is implemented as an actor in order to support the use-case:
- * "schedule an LSIF index job for this package after 1 minute". We don't care
- * about the return value of that scheduled LSIF index job so the "fire and
- * forget" nature of actors is OK.
- */
-class PackageActor(
- src: String,
- coursier: String,
- store: PackageStore,
- lsifJavaVersion: String,
- val dir: Path,
- val addr: Int = 3434
-)(implicit ctx: Context, log: cask.Logger)
- extends SimpleActor[Package] {
- createGitConfig()
-
- def run(msg: Package): Unit = {
- lsifIndex(msg, lsifUpload = true)
- }
- private def tmpDir = dir.resolve("lsif-java-tmp")
- private var serveGit = Option.empty[SubProcess]
- private def createGitConfig(): Unit = {
- def setConfig(key: String, value: String): Unit = {
- val result = os.proc("git", "config", "--get", key).call(check = false)
- if (result.exitCode != 0) {
- os.proc("git", "config", "--global", key, value).call()
- }
- }
- setConfig("user.name", "lsif-java")
- setConfig("user.email", "lsif-java@sourcegraph.com")
- }
- def lsifIndex(msg: Package, lsifUpload: Boolean): Path = {
- commitSources(msg)
- val dump: Path = lsifIndexPackage(msg)
- if (lsifUpload && Files.isRegularFile(dump)) {
- os.proc(src, "login").call(stdout = os.Pipe, stderr = os.Pipe)
- os.proc(src, "lsif", "upload", "--no-progress", "--repo", msg.path)
- .call(
- stdout = os.Pipe,
- stderr = os.Pipe,
- cwd = os.Path(packageDir(msg))
- )
- store.addIndexedPackage(msg)
- }
- dump
- }
-
- def commitSources(pkg: Package): Unit = {
- if (isCached(pkg))
- return
- pkg match {
- case mvn @ MavenPackage(_) =>
- val deps = Dependencies
- .resolveDependencies(List(mvn.repr), transitive = false)
- indexDeps(mvn, deps)
- case JdkPackage(version) =>
- val home = os
- .proc(coursier, "java-home", "--jvm", version)
- .call()
- .out
- .trim()
- val srcs = List(
- Paths.get(home, "src.zip"),
- Paths.get(home, "lib", "src.zip")
- )
- srcs.find(Files.isRegularFile(_)) match {
- case Some(src) =>
- commitSourcesArtifact(pkg, src, Dependencies.empty)
- case None =>
- throw new IllegalArgumentException(s"no such files: $srcs")
- }
- case npm: NpmPackage =>
- commitNpmPackage(npm)
- }
- }
-
- private def commitNpmPackage(npm: NpmPackage): Unit = {
- val repo = packageDir(npm)
- if (!isCached(npm)) {
- val tmp = Files.createTempDirectory("packagehub")
- val filename = tmp.resolve(npm.tarballFilename)
- val url = os
- .proc("npm", "info", npm.npmName, "dist.tarball")
- .call(env = Map("NO_UPDATE_NOTIFIER" -> "true"))
- .out
- .trim()
- val in = new URL(url).openStream()
- try Files.copy(in, filename, StandardCopyOption.REPLACE_EXISTING)
- finally in.close()
- ArchiverFactory
- .createArchiver(new File("archive.tgz"))
- .extract(filename.toFile, tmp.toFile)
- val allFiles = FileIO.listAllFilesRecursively(AbsolutePath(tmp))
- val packageJson = allFiles
- .filter(_.toNIO.endsWith("package.json"))
- .sortBy(_.toNIO.iterator().asScala.length)
- .headOption
- .getOrElse(
- sys.error(s"no such file: package.json (${allFiles.mkString(", ")})")
- )
- Files.createDirectories(repo.getParent)
- val parent = Paths.get(
- packageJson
- .toNIO
- .getParent
- .toRealPath()
- .toString
- .stripPrefix("/private")
- )
- moveDirectory(parent, repo)
- Files.write(
- repo.resolve(".gitignore"),
- List("/tsconfig.json", "node_modules/").asJava
- )
- cacheDirectory(npm)
- Files.walkFileTree(tmp, new DeleteVisitor())
- gitInit(repo)
- gitCommitAll(npm.version, repo)
- }
- }
- private def moveDirectory(from: Path, to: Path): Unit = {
- Files.walkFileTree(
- from,
- new SimpleFileVisitor[Path] {
- override def visitFile(
- file: Path,
- attrs: BasicFileAttributes
- ): FileVisitResult = {
- val relpath = from.relativize(file)
- Files.copy(
- file,
- to.resolve(relpath),
- StandardCopyOption.REPLACE_EXISTING
- )
- FileVisitResult.CONTINUE
-
- }
- override def preVisitDirectory(
- dir: Path,
- attrs: BasicFileAttributes
- ): FileVisitResult = {
- val relpath = from.relativize(dir)
- val out = to.resolve(relpath)
- if (!Files.isDirectory(out))
- Files.createDirectory(out)
- FileVisitResult.CONTINUE
- }
- }
- )
- }
- private def indexDeps(dep: MavenPackage, deps: Dependencies): Unit = {
- deps
- .sourcesResult
- .fullDetailedArtifacts
- .foreach {
- case (dep, _, _, Some(file)) =>
- commitSourcesArtifact(MavenPackage(dep), file.toPath, deps)
- case _ =>
- }
- }
-
- private def packageId(dep: Dependency): String = {
- s"${dep.module.organization.value}:${dep.module.name.value}:${dep.version}"
- }
- private def packageRelDir(dep: Dependency): Path = {
- Paths.get(dep.module.organization.value, dep.module.name.value, dep.version)
- }
- def packageTmpDir(dep: Dependency): Path = {
- tmpDir.resolve(packageRelDir(dep))
- }
- def packageDir(dep: Dependency): Path = {
- Paths.get(dep.module.organization.value, dep.module.name.value, dep.version)
- }
- def packageDir(pkg: Package): Path = {
- dir.resolve(pkg.relativePath)
- }
- private def collectAllJavaFiles(dir: Path): List[Path] = {
- val javaPattern = FileSystems.getDefault.getPathMatcher("glob:**.java")
- val buf = ListBuffer.empty[Path]
- Files.walkFileTree(
- dir,
- new SimpleFileVisitor[Path] {
- override def visitFile(
- file: Path,
- attrs: BasicFileAttributes
- ): FileVisitResult = {
- if (javaPattern.matches(file)) {
- buf += file
- }
- FileVisitResult.CONTINUE
- }
- override def visitFileFailed(
- file: Path,
- exc: IOException
- ): FileVisitResult = FileVisitResult.CONTINUE
- }
- )
- buf.toList
- }
-
- def lsifIndexPackage(pkg: Package): Path = {
- val sourceroot = packageDir(pkg)
- val dump = sourceroot.resolve("dump.lsif")
- if (!Files.isDirectory(sourceroot))
- return dump
- pkg match {
- case _: NpmPackage =>
- val toolVersions = sourceroot.resolve(".tool-versions")
- if (!Files.isRegularFile(toolVersions)) {
- Files.write(toolVersions, List("yarn 1.22.4").asJava)
- }
- val tsconfig = sourceroot.resolve("tsconfig.json")
- if (!Files.isRegularFile(tsconfig)) {
- val config = Obj(
- "include" -> Arr("**/*"),
- "compilerOptions" -> Obj("allowJs" -> true)
- )
- Files.write(tsconfig, List(ujson.write(config, indent = 2)).asJava)
- }
- exec(sourceroot, List("yarn", "install"))
- exec(
- sourceroot,
- List("npx", "@olafurpg/lsif-tsc", "-p", sourceroot.toString)
- )
- case _ =>
- val jvm: String =
- pkg match {
- case JdkPackage(version) =>
- version
- case _ =>
- val configVersion: Option[String] =
- for {
- config <- Option(
- sourceroot.resolve(LsifBuildTool.ConfigFileName)
- )
- if Files.isRegularFile(config)
- obj <- ujson.read(os.read(os.Path(config))).objOpt
- version <- obj.get("jvm")
- versionStr <- version.strOpt
- } yield versionStr
- configVersion.getOrElse(JavaVersion.DEFAULT_JAVA_VERSION.toString)
- }
- exec(
- sourceroot,
- List(
- coursier,
- "launch",
- "--jvm",
- jvm,
- s"com.sourcegraph:lsif-java_2.13:${lsifJavaVersion}",
- "-r",
- "sonatype:snapshots",
- "--",
- "index",
- "--output",
- dump.toString(),
- "--build-tool",
- "lsif"
- )
- )
- }
- dump
- }
-
- private def commitSourcesArtifact(
- dep: Package,
- file: Path,
- deps: Dependencies
- ): Unit = {
- val repo = packageDir(dep)
- Files.createDirectories(repo)
- gitInit(repo)
- deleteAllNonGitFiles(repo)
- FileIO.withJarFileSystem(AbsolutePath(file), create = false, close = true) {
- root =>
- FileIO
- .listAllFilesRecursively(root)
- .foreach { file =>
- val bytes = Files.readAllBytes(file.toNIO)
- val rel = file.toRelative(root).toURI(false).toString()
- val out = repo.resolve(rel)
- Files.createDirectories(out.getParent())
- Files.write(
- out,
- bytes,
- StandardOpenOption.CREATE,
- StandardOpenOption.TRUNCATE_EXISTING
- )
- }
- Files.walkFileTree(
- root.toNIO,
- new SimpleFileVisitor[Path] {
- override def visitFile(
- file: Path,
- attrs: BasicFileAttributes
- ): FileVisitResult = {
- FileVisitResult.CONTINUE
- }
- }
- )
- }
- Files.write(
- repo.resolve(".gitignore"),
- List("dump.lsif", packagehubCached).asJava
- )
- val build = Obj()
- dep match {
- case MavenPackage(dep) =>
- val inferredJvmVersion =
- for {
- jar <- deps.classpath.headOption
- version <- Option(JavaVersion.classfileJvmVersion(jar).orElse(null))
- // Some libraries like JUnit target Java 3 but we can still compile them with Java 8.
- } yield JavaVersion.roundToNearestStableRelease(version).toString
- val jvmVersion = inferredJvmVersion
- .getOrElse(JavaVersion.DEFAULT_JAVA_VERSION.toString)
-
- build("kind") = Str("maven")
- build("jvm") = Str(jvmVersion)
- build("dependencies") = Arr(packageId(dep))
- case JdkPackage(version) =>
- build("kind") = Str("jdk")
- build("jvm") = Str(version)
- build("dependencies") = Arr()
- case _ =>
- }
- Files.write(
- repo.resolve(LsifBuildTool.ConfigFileName),
- List(ujson.write(build, indent = 2)).asJava
- )
- cacheDirectory(dep)
- gitCommitAll(dep.version, repo)
- }
-
- def exec(cwd: Path, command: List[String]): CommandResult = {
- val lines = ArrayBuffer.empty[String]
- val out = Readlines(line => lines += line)
- val result = os
- .proc(Shellable(command))
- .call(stdout = out, stderr = out, cwd = os.Path(cwd), check = false)
- if (result.exitCode == 0)
- result
- else {
- throw CommandFailed(command, cwd, lines, result)
- }
- }
-
- case class CommandFailed(
- command: List[String],
- cwd: Path,
- stdout: collection.Seq[String],
- result: CommandResult
- ) extends RuntimeException(
- ujson.write(
- Obj(
- "command" -> Arr.from(command),
- "cwd" -> cwd.toString,
- "exit" -> result.exitCode,
- "stdout" -> Arr.from(stdout)
- ),
- indent = 2
- )
- )
-
- val date = "Thu Apr 8 14:24:52 2021 +0200"
- def proc(repo: Path, command: String*): CommandResult = {
- os.proc(Shellable(command))
- .call(cwd = os.Path(repo), env = Map("GIT_COMMITTER_DATE" -> date))
- }
-
- private def gitCommitAll(version: String, repo: Path): Unit = {
- val message = s"Version ${version}"
- proc(repo, "git", "add", ".")
- proc(repo, "git", "commit", "--allow-empty", "--date", date, "-m", message)
- proc(repo, "git", "tag", "-f", "-m", message, s"v${version}")
- }
-
- private def gitInit(repo: Path): Unit = {
- proc(repo, "git", "init")
- }
- private def deleteAllNonGitFiles(repo: Path): Unit = {
- val gitDir = repo.resolve(".git")
- val deleteNonGitFiles =
- new DeleteVisitor(deleteFile = file => !file.startsWith(gitDir))
- Files.walkFileTree(repo, deleteNonGitFiles)
- }
-
- private val packagehubCached = "packagehub_cached"
-
- private def cacheDirectory(pkg: Package): Unit = {
- Files
- .write(packageDir(pkg).resolve(packagehubCached), List[String]().asJava)
- }
- private def isCached(pkg: Package): Boolean = {
- Files.exists(packageDir(pkg).resolve(packagehubCached))
- }
-
-}
diff --git a/packagehub/src/main/scala/com/sourcegraph/packagehub/PackageHub.scala b/packagehub/src/main/scala/com/sourcegraph/packagehub/PackageHub.scala
deleted file mode 100644
index 619e81a2..00000000
--- a/packagehub/src/main/scala/com/sourcegraph/packagehub/PackageHub.scala
+++ /dev/null
@@ -1,116 +0,0 @@
-package com.sourcegraph.packagehub
-
-import java.nio.file.Path
-import java.time
-import java.time.Duration
-
-import scala.concurrent.Await
-import scala.concurrent.Promise
-import scala.concurrent.duration
-
-import cask.main.Main
-import cask.util.Logger
-import castor.Context
-import com.sourcegraph.lsif_java.BuildInfo
-import io.undertow.Undertow
-import moped.annotations.Description
-import moped.annotations.ExampleValue
-import moped.annotations.Hidden
-import moped.cli.Application
-import moped.cli.Command
-import moped.cli.CommandParser
-import moped.commands.HelpCommand
-import moped.commands.VersionCommand
-import moped.json.DecodingContext
-import moped.json.JsonString
-import moped.json.Result
-
-/**
- * Entrypoint to start the PackageHub server.
- */
-case class PackageHub(
- port: String = "8080",
- host: String = "localhost",
- verbose: Boolean = false,
- dir: Option[Path] = None,
- app: Application = Application.default,
- @Hidden() cancelToken: Promise[Unit] = Promise(),
- ctx: Context = castor.Context.Simple.global,
- postgres: PostgresOptions = PostgresOptions(),
- @Description("URL of the PackageHub server") packagehubUrl: String =
- "https://packagehub-ohcltxh6aq-uc.a.run.app",
- @Description(
- "The version of lsif-java to use for indexing packages"
- ) lsifJavaVersion: String = BuildInfo.version,
- @Description("Path to the src-cli binary") src: String = "src",
- @Description("Path to the coursier binary") coursier: String = "coursier",
- @Description("If enabled, schedule an LSIF index after the given delay")
- @ExampleValue("--auto-index-delay=PT1M") autoIndexDelay: Option[Duration] =
- None
-) extends Command {
- val log = new Logger.Console()
- def run(): Int = {
- val store: PackageStore =
- postgres.toDataSource(app.reporter) match {
- case Some(source) =>
- app.info("Connected to PostgreSQL database")
- new PostgresPackageStore(source)
- case None =>
- new InMemoryPackageStore
- }
- if (!verbose)
- Main.silenceJboss()
- val actor =
- new PackageActor(
- src,
- coursier,
- store,
- lsifJavaVersion,
- dir.getOrElse(app.env.cacheDirectory)
- )(ctx, log)
- val routes = new PackageRoutes(this, actor, store, log)(ctx)
- val server = Undertow
- .builder
- .addHttpListener(routes.port, host)
- .setHandler(routes.defaultHandler)
- .build()
- server.start()
- app.info(s"Listening on http://$host:$port")
- try Await.result(cancelToken.future, duration.Duration.Inf)
- finally server.stop()
- 0
- }
-
- def isEnv(name: String): Boolean = app.env.environmentVariables.contains(name)
-
-}
-
-object PackageHub {
- implicit val promiseCodec = new EmptyJsonCodec[Promise[Unit]]
- implicit val contextCodec = new EmptyJsonCodec[Context]
- implicit val durationCodec =
- new EmptyJsonCodec[Duration] {
- override def decode(context: DecodingContext): Result[Duration] =
- context.json match {
- case JsonString(value) =>
- Result.fromUnsafe(() => Duration.parse(value))
- case _ =>
- super.decode(context)
- }
- }
- implicit val parser = CommandParser.derive(PackageHub())
- val app = Application
- .fromName(
- "packagehub",
- BuildInfo.version,
- commands = List(
- CommandParser[HelpCommand],
- CommandParser[VersionCommand],
- CommandParser[PackageHub]
- )
- )
- .withIsSingleCommand(true)
- def main(args: Array[String]): Unit = {
- app.runAndExitIfNonZero(args.toList)
- }
-}
diff --git a/packagehub/src/main/scala/com/sourcegraph/packagehub/PackageRoutes.scala b/packagehub/src/main/scala/com/sourcegraph/packagehub/PackageRoutes.scala
deleted file mode 100644
index 769d0610..00000000
--- a/packagehub/src/main/scala/com/sourcegraph/packagehub/PackageRoutes.scala
+++ /dev/null
@@ -1,252 +0,0 @@
-package com.sourcegraph.packagehub
-
-import java.io.ByteArrayOutputStream
-import java.io.OutputStream
-import java.nio.file.Files
-
-import scala.collection.mutable
-import scala.collection.mutable.ListBuffer
-import scala.util.control.NoStackTrace
-
-import cask.Response.Data
-import cask.model.Request
-import cask.model.Response
-import cask.model.StaticFile
-import cask.model.Status
-import cask.util.Logger
-import castor.Context
-import os.Internals
-import os.Shellable
-import os.SubProcess
-import ujson.Arr
-import ujson.Obj
-import ujson.Value
-
-/**
- * The HTTP routes that are served by PackageHub.
- */
-class PackageRoutes(
- server: PackageHub,
- actor: PackageActor,
- store: PackageStore,
- override implicit val log: Logger
-)(implicit ctx: Context)
- extends cask.MainRoutes {
- override def host: String = server.host
- override def port: Int = server.port.toInt
- override def verbose: Boolean = server.verbose
-
- // ===========================================================
- // The following endpoints are required by src-cli "serve-git"
- // ===========================================================
-
- @cask.get("/v1/list-repos")
- def listRepos(): ujson.Value = {
- val items = Arr.from(store.allPackages().map(_.toJsonRepo))
- Obj("Items" -> items)
- }
-
- @cask.route("/repos", methods = Seq("get", "post"), subpath = true)
- def repoSubpath(
- request: cask.Request,
- service: Seq[String] = Nil
- ): cask.Response[Data] = {
- Package.fromPath(request.remainingPathSegments.toList) match {
- case Some((pkg, requestPath)) =>
- repoSubpath(request, pkg, requestPath)
- case _ =>
- badRequest(
- "invalid repo name (want /repos/maven/ORGANIZATION/NAME/VERSION/...)"
- )
- }
- }
-
- // =====================================================================
- // The following endpoints are not required by src-cli "serve-git". They
- // monstly exist for debugging purposes and to manually trigger stuff.
- // =====================================================================
-
- @cask.get("/packagehub/packages")
- def packages(): ujson.Value = Arr.from(store.allPackages().map(_.id))
-
- @cask.postJson("/packagehub/packages")
- def addPackage(packages: Seq[String]): cask.Response[Value] = {
- val parsed = packages.toList.map(parsePackage)
- val ok = parsed.collect { case Right(value) =>
- value
- }
- store.addPackages(ok)
- val errors = parsed.collect { case Left(value) =>
- value
- }
- cask.Response(Obj("errors" -> Arr.from(errors)))
- }
-
- @cask.post("/packagehub/package/:pkg")
- def addPackage(pkg: String): cask.Response[Value] = {
- withPackage(pkg) { p =>
- store.addPackage(p)
- }
- }
-
- @cask.get("/packagehub/indexed-packages")
- def indexedPackages(): ujson.Value =
- Arr.from(store.allIndexedPackages().map(_.id))
-
- @cask.post("/packagehub/index-package", subpath = true)
- def indexPackage(
- request: cask.Request,
- upload: Boolean = false
- ): cask.Response[Data] = {
- val pkg = request.remainingPathSegments.mkString("/")
- parsePackage(pkg) match {
- case Left(error) =>
- badRequest(error)
- case Right(pkg) =>
- store.addPackage(pkg)
- if (!store.isIndexedPackage(pkg)) {
- val dump = actor.lsifIndex(pkg, lsifUpload = upload)
- val verb =
- if (upload)
- "uploaded"
- else
- "generated"
- okResponse(s"$verb index for package '${pkg.id}'")
- } else {
- okResponse(s"package '${pkg.id}' is already indexed")
- }
- }
- }
-
- @cask.post("/packagehub/indexed-package/:pkg")
- def addIndexedPackage(pkg: String): cask.Response[Value] = {
- withPackage(pkg) { p =>
- store.addIndexedPackage(p)
- }
- }
-
- private def repoSubpath(
- request: cask.Request,
- pkg: Package,
- requestPath: List[String]
- ): cask.Response[Data] = {
- val path = actor.packageDir(pkg)
- actor.commitSources(pkg)
- val args = ListBuffer[String](
- "git",
- // Partial clones/fetches
- "-c",
- "uploadpack.allowFilter=true",
- // Can fetch any object. Used in case of race between a resolve ref and a
- // fetch of a commit. Safe to do, since this is only used internally.
- "-c",
- "uploadpack.allowAnySHA1InWant=true",
- "upload-pack",
- "--stateless-rpc"
- )
- val headers = mutable.Map.empty[String, String]
- val bytes = new ByteArrayOutputStream
- val isStaticFile: Boolean =
- if (requestPath.endsWith(List("info", "refs"))) {
- server
- .autoIndexDelay
- .foreach { delay =>
- ctx.scheduleMsg(actor, pkg, delay)
- }
- headers("Content-Type") = "application/x-git-upload-pack-advertisement"
- bytes.write(packetWrite("# service=git-upload-pack\n"))
- bytes.write("0000".getBytes())
- args += "--advertise-refs"
- false
- } else if (requestPath.endsWith(List("git-upload-pack"))) {
- headers("Content-Type") = "application/x-git-upload-pack-result"
- false
- } else {
- true
- }
- if (isStaticFile) {
- val file = path.resolve(requestPath.mkString("/"))
- if (Files.isRegularFile(file)) {
- val text = new String(Files.readAllBytes(file))
- StaticFile(file.toString(), Nil)
- } else if (Files.isDirectory(file)) {
- val ls = file.toFile().list().map(f => s"$f")
- ls.mkString("")
- } else {
- notFound(s"no such file: $file")
- }
- } else {
- val env = mutable.Map.empty[String, String]
- Option(request.exchange.getRequestHeaders().getLast("Git-Protocol"))
- .foreach { protocol =>
- env("GIT_PROTOCOL") = protocol
- }
- args += path.toString()
- val command = args.toSeq.mkString(" ")
- val result = os
- .proc(Shellable(args.toSeq))
- .spawn(
- env = env.toMap,
- stdout = os.Pipe,
- stderr = os.Pipe,
- stdin = request.bytes
- )
- Response(new ProcessData(result, request, command), 200, headers.toSeq)
- }
- }
-
- private class ProcessData(proc: SubProcess, request: Request, command: String)
- extends Data {
- def write(out: OutputStream): Unit = {
- Internals.transfer(proc.stdout, out)
- proc.waitFor()
- val exit = proc.exitCode()
- if (exit != 0) {
- val path = Obj(
- "exit" -> exit,
- "path" -> request.exchange.getRequestPath(),
- "method" -> request.exchange.getRequestMethod().toString(),
- "command" -> command
- )
- throw new RuntimeException(ujson.write(path)) with NoStackTrace
- }
- }
- def headers: Seq[(String, String)] = Nil
- }
-
- private def packetWrite(str: String): Array[Byte] = {
- var s = Integer.toString(str.length() + 4, 16)
- val modulo = s.length() % 4
- if (modulo != 0) {
- val padding = "0" * (4 - modulo)
- s = padding + s
- }
- (s + str).getBytes()
- }
-
- private def parsePackage(pkg: String) =
- Package.fromString(pkg, server.coursier)
-
- private def withPackage(
- pkg: String
- )(fn: Package => Unit): cask.Response[Value] = {
- parsePackage(pkg) match {
- case Left(error) =>
- badRequest(error)
- case Right(p) =>
- fn(p)
- cask.Response(Obj())
- }
- }
-
- private def badRequest(error: Value): Response[Value] =
- errorResponse(error, Status.BadRequest.code)
- private def notFound(error: Value): Response[Value] =
- errorResponse(error, Status.NotFound.code)
- private def okResponse(result: String*): Response[Value] =
- Response(Obj("message" -> Arr.from(result)))
- private def errorResponse(error: Value, code: Int): Response[Value] =
- Response(Obj("error" -> error), code, Nil, Nil)
-
- initialize()
-}
diff --git a/packagehub/src/main/scala/com/sourcegraph/packagehub/PackageStore.scala b/packagehub/src/main/scala/com/sourcegraph/packagehub/PackageStore.scala
deleted file mode 100644
index 40f3d96b..00000000
--- a/packagehub/src/main/scala/com/sourcegraph/packagehub/PackageStore.scala
+++ /dev/null
@@ -1,110 +0,0 @@
-package com.sourcegraph.packagehub
-
-import java.sql.PreparedStatement
-import java.util.Collections
-import java.util.concurrent.ConcurrentHashMap
-import java.{util => ju}
-import javax.sql.DataSource
-
-import scala.collection.mutable.ListBuffer
-import scala.jdk.CollectionConverters._
-import scala.util.Using
-
-import org.flywaydb.core.Flyway
-
-/**
- * Data storage for what packages to host.
- */
-trait PackageStore {
- def addPackages(pkg: List[Package]): Unit
- final def addPackage(pkg: Package): Unit = addPackages(List(pkg))
- def allPackages(): List[Package]
-
- def addIndexedPackages(pkg: List[Package]): Unit
- final def addIndexedPackage(pkg: Package): Unit =
- addIndexedPackages(List(pkg))
- def isIndexedPackage(pkg: Package): Boolean
- def allIndexedPackages(): List[Package]
-}
-
-class InMemoryPackageStore extends PackageStore {
- private def concurrentHashSet[T]: ju.Set[T] =
- Collections.newSetFromMap(new ConcurrentHashMap[T, java.lang.Boolean])
- val packages = new ConcurrentHashMap[String, Package]
- val jdks = concurrentHashSet[String]
- val indexedPackages = concurrentHashSet[String]
-
- override def addPackages(pkgs: List[Package]): Unit = {
- pkgs.foreach { pkg =>
- packages.put(pkg.id, pkg)
- }
- }
- override def allPackages(): List[Package] = packages.values.asScala.toList
- override def addIndexedPackages(pkgs: List[Package]): Unit = {
- pkgs.foreach { pkg =>
- indexedPackages.add(pkg.id)
- }
- }
- override def isIndexedPackage(pkg: Package): Boolean = {
- indexedPackages.contains(pkg.id)
- }
- override def allIndexedPackages(): List[Package] =
- indexedPackages.asScala.map(Package.parse).toList
-
-}
-
-class PostgresPackageStore(source: DataSource) extends PackageStore {
- Flyway.configure().dataSource(source).load().migrate()
-
- override def addPackages(pkgs: List[Package]): Unit = {
- add("packages", pkgs)
- }
- override def allPackages(): List[Package] = {
- all("packages")
- }
- override def addIndexedPackages(pkgs: List[Package]): Unit = {
- add("indexed_packages", pkgs)
- }
- override def isIndexedPackage(pkg: Package): Boolean = {
- withStatement("select id from indexed_packages where id = ?") { stat =>
- stat.setString(1, pkg.id)
- stat.executeQuery().next()
- }
- }
- override def allIndexedPackages(): List[Package] = {
- all("indexed_packages")
- }
-
- private def withStatement[T](query: String)(fn: PreparedStatement => T): T = {
- Using
- .Manager { use =>
- val conn = use(source.getConnection())
- val stat = use(conn.prepareStatement(query))
- fn(stat)
- }
- .get
- }
-
- private def add(table: String, pkgs: List[Package]): Unit = {
- withStatement(
- s"INSERT into $table (id) VALUES (?) ON CONFLICT DO NOTHING"
- ) { stat =>
- pkgs.foreach { pkg =>
- stat.setString(1, pkg.id)
- stat.addBatch()
- }
- stat.executeBatch()
- }
- }
-
- private def all(table: String): List[Package] = {
- withStatement(s"SELECT id FROM $table") { stat =>
- val rs = stat.executeQuery()
- val buf = ListBuffer.empty[Package]
- while (rs.next()) {
- buf += Package.parse(rs.getString("id"))
- }
- buf.toList
- }
- }
-}
diff --git a/packagehub/src/main/scala/com/sourcegraph/packagehub/PostgresOptions.scala b/packagehub/src/main/scala/com/sourcegraph/packagehub/PostgresOptions.scala
deleted file mode 100644
index c6ef7a37..00000000
--- a/packagehub/src/main/scala/com/sourcegraph/packagehub/PostgresOptions.scala
+++ /dev/null
@@ -1,50 +0,0 @@
-package com.sourcegraph.packagehub
-
-import javax.sql.DataSource
-
-import com.zaxxer.hikari.HikariConfig
-import com.zaxxer.hikari.HikariDataSource
-import moped.reporters.Reporter
-
-final case class PostgresOptions(
- url: String = "",
- username: String = "",
- password: String = ""
-) {
- def toDataSource(reporter: Reporter): Option[DataSource] = {
- if (url.isEmpty() || username.isEmpty() || password.isEmpty()) {
- if (url.nonEmpty) {
- reporter.warning(
- "ignoring unused PostgresSQL URL. To fix this problem, ensure the username, password and URL are all configured."
- )
- }
- if (username.nonEmpty) {
- reporter.warning(
- "ignoring unused PostgresSQL username. To fix this problem, ensure the username, password and URL are all configured."
- )
- }
- if (password.nonEmpty) {
- reporter.warning(
- "ignoring unused PostgresSQL password. To fix this problem, ensure the username, password and URL are all configured."
- )
- }
- None
- } else {
- Class.forName("org.postgresql.Driver")
- val config = new HikariConfig()
- config.setUsername(username)
- config.setPassword(password)
- config.setJdbcUrl(url)
- config.setMaximumPoolSize(5)
- config.setMinimumIdle(5)
- config.setConnectionTimeout(10000) // 10 seconds
- config.setIdleTimeout(600000) // 10 minutes
- config.setMaxLifetime(1800000) // 30 minutes
- Some(new HikariDataSource(config))
- }
- }
-}
-
-object PostgresOptions {
- implicit val codec = moped.macros.deriveCodec(PostgresOptions())
-}