build.gradle

import org.elasticsearch.gradle.Architecture
import org.elasticsearch.gradle.LoggedExec
import org.elasticsearch.gradle.VersionProperties
import org.elasticsearch.gradle.internal.DockerBase
import org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes
import org.elasticsearch.gradle.internal.docker.DockerBuildTask
import org.elasticsearch.gradle.internal.docker.DockerSupportPlugin
import org.elasticsearch.gradle.internal.docker.DockerSupportService
import org.elasticsearch.gradle.internal.docker.ShellRetry
import org.elasticsearch.gradle.internal.docker.TransformLog4jConfigFilter
import org.elasticsearch.gradle.internal.info.BuildParams
import org.elasticsearch.gradle.util.GradleUtils

import java.nio.file.Path
import java.time.temporal.ChronoUnit

apply plugin: 'elasticsearch.internal-yaml-rest-test'
apply plugin: 'elasticsearch.test.fixtures'
apply plugin: 'elasticsearch.internal-distribution-download'

String buildId = providers.systemProperty('build.id').getOrNull()
boolean useLocalArtifacts = buildId != null && buildId.isBlank() == false

repositories {
  // Define a repository that allows Gradle to fetch a resource from GitHub. This
  // is only used to fetch the `tini` binary, when building the Iron Bank docker image
  // for testing purposes. While in theory we could download `tini` this way for the
  // other Docker variants, the need for the main image to be rebuildable by Docker Hub
  // means that the Dockerfile itself has to fetch the binary.
  ivy {
    url 'https://github.com/'
    patternLayout {
      artifact '/[organisation]/[module]/releases/download/v[revision]/[module]-[classifier]'
    }
    metadataSources { artifact() }
    content { includeGroup 'krallin' }
  }

  // Cloud builds bundle some beats
  ivy {
    name = 'beats'
    if (useLocalArtifacts) {
      url "file://${buildDir}/artifacts/"
      patternLayout {
        artifact '/[organisation]/[module]-[revision]-[classifier].[ext]'
      }
    } else {
      url "https://artifacts-snapshot.elastic.co/"
      patternLayout {
        if (VersionProperties.isElasticsearchSnapshot()) {
          artifact '/[organization]/[revision]/downloads/[organization]/[module]/[module]-[revision]-[classifier].[ext]'
        } else {
          // When building locally we always use snapshot artifacts even if passing `-Dbuild.snapshot=false`.
          // Release builds are always done with a local repo.
          artifact '/[organization]/[revision]-SNAPSHOT/downloads/[organization]/[module]/[module]-[revision]-SNAPSHOT-[classifier].[ext]'
        }
      }
    }
    metadataSources { artifact() }
    content { includeGroup 'beats' }
  }
}

testFixtures.useFixture()

configurations {
  aarch64DockerSource
  dockerSource
  log4jConfig
  tini
  allPlugins
  filebeat
  metricbeat
}

String beatsArch = Architecture.current() == Architecture.AARCH64 ? 'arm64' : 'x86_64'
String tiniArch = Architecture.current() == Architecture.AARCH64 ? 'arm64' : 'amd64'

dependencies {
  aarch64DockerSource project(path: ":distribution:archives:linux-aarch64-tar", configuration: 'default')
  dockerSource project(path: ":distribution:archives:linux-tar", configuration: 'default')
  log4jConfig project(path: ":distribution", configuration: 'log4jConfig')
  tini "krallin:tini:0.19.0:${tiniArch}"
  allPlugins project(path: ':plugins', configuration: 'allPlugins')
  filebeat "beats:filebeat:${VersionProperties.elasticsearch}:linux-${beatsArch}@tar.gz"
  metricbeat "beats:metricbeat:${VersionProperties.elasticsearch}:linux-${beatsArch}@tar.gz"
}

ext.expansions = { Architecture architecture, DockerBase base ->
  def (major,minor) = VersionProperties.elasticsearch.split("\\.")

  // We tag our Docker images with various pieces of information, including a timestamp
  // for when the image was built. However, this makes it impossible completely cache
  // the image. When developing the Docker images, it's very tedious to completely rebuild
  // an image for every single change. Therefore, outside of CI, we fix the
  // build time to midnight so that the Docker build cache is usable.
  def buildDate = BuildParams.isCi() ? BuildParams.buildDate : BuildParams.buildDate.truncatedTo(ChronoUnit.DAYS)

  return [
    'arch'               : architecture.classifier,
    'base_image'         : base.image,
    'bin_dir'            : base == DockerBase.IRON_BANK ? 'scripts' : 'bin',
    'build_date'         : buildDate,
    'config_dir'         : base == DockerBase.IRON_BANK ? 'scripts' : 'config',
    'git_revision'       : BuildParams.gitRevision,
    'license'            : base == DockerBase.IRON_BANK ? 'Elastic License 2.0' : 'Elastic-License-2.0',
    'package_manager'    : base == DockerBase.IRON_BANK ? 'yum' : (base == DockerBase.UBI ? 'microdnf' : 'apt-get'),
    'docker_base'        : base.name().toLowerCase(),
    'version'            : VersionProperties.elasticsearch,
    'major_minor_version': "${major}.${minor}",
    'retry'              : ShellRetry
  ]
}

/**
 * This filter squashes long runs of newlines so that the output
 * is a little more aesthetically pleasing.
 */
class SquashNewlinesFilter extends FilterReader {
  SquashNewlinesFilter(Reader input) {
    super(new StringReader(input.text.replaceAll("\n{2,}", "\n\n")))
  }
}

private static String toCamel(String input) {
  return input.split("[^a-zA-Z0-9]").collect({ it.substring(0, 1) + it.substring(1).toLowerCase(Locale.ROOT) }).join("")
}

private static String taskName(String prefix, Architecture architecture, DockerBase base, String suffix) {
  return prefix +
    (architecture == Architecture.AARCH64 ? 'Aarch64' : '') +
    (base == DockerBase.DEFAULT ? "" : toCamel(base.name())) +
    suffix
}

ext.dockerBuildContext = { Architecture architecture, DockerBase base ->
  copySpec {
    final Map<String, String> varExpansions = expansions(architecture, base)
    final Path projectDir = project.projectDir.toPath()

    if (base == DockerBase.IRON_BANK) {
      into('scripts') {
        from projectDir.resolve("src/docker/bin")
        from projectDir.resolve("src/docker/config")
      }
      from(projectDir.resolve("src/docker/iron_bank")) {
        expand(varExpansions)
      }
    } else {
      into('bin') {
        from projectDir.resolve("src/docker/bin")
      }

      into('config') {
        from projectDir.resolve("src/docker/config")
      }
    }

    from(projectDir.resolve("src/docker/Dockerfile")) {
      expand(varExpansions)
      filter SquashNewlinesFilter
    }
  }
}

def createAndSetWritable(Object... locations) {
  locations.each { location ->
    File file = file(location)
    file.mkdirs()
    file.setWritable(true, false)
  }
}

tasks.register("copyNodeKeyMaterial", Sync) {
  from project(':x-pack:plugin:core')
    .files(
      'src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.pem',
      'src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt'
    )
  into "${buildDir}/certs"
  doLast {
    file("${buildDir}/certs").setReadable(true, false)
    file("${buildDir}/certs/testnode.pem").setReadable(true, false)
    file("${buildDir}/certs/testnode.crt").setReadable(true, false)
  }
}

elasticsearch_distributions {
  Architecture.values().each { eachArchitecture ->
    "docker_${eachArchitecture == Architecture.AARCH64 ? '_aarch64' : ''}" {
      architecture = eachArchitecture
      type = InternalElasticsearchDistributionTypes.DOCKER
      version = VersionProperties.getElasticsearch()
      failIfUnavailable = false // This ensures we don't attempt to build images if docker is unavailable
    }
  }
}

tasks.named("preProcessFixture").configure {
  dependsOn elasticsearch_distributions.matching { it.architecture == Architecture.current() }
  dependsOn "copyNodeKeyMaterial"
  doLast {
    // tests expect to have an empty repo
    project.delete(
      "${testFixturesDir}/repo",
    )
    createAndSetWritable(
      "${testFixturesDir}/repo",
      "${testFixturesDir}/logs/default-1",
      "${testFixturesDir}/logs/default-2",
    )
  }
}

tasks.named("processYamlRestTestResources").configure {
  from project(':x-pack:plugin:core')
    .files(
      'src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.pem',
      'src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt'
    )
}

tasks.named("yamlRestTest").configure {
  outputs.doNotCacheIf('Build cache is disabled for Docker tests') { true }
  maxParallelForks = '1'
  include '**/*IT.class'
}

tasks.named("check").configure {
  dependsOn "yamlRestTest"
}

// We build the images used in compose locally, but the pull command insists on using a repository
// thus we must disable it to prevent it from doing so.
// Everything will still be pulled since we will build the local images on a pull
tasks.named("composePull").configure {
  enabled = false
}

tasks.named("composeUp").configure {
  dependsOn tasks.named("preProcessFixture")
}

void addBuildDockerContextTask(Architecture architecture, DockerBase base) {
  String configDirectory = base == DockerBase.IRON_BANK ? 'scripts' : 'config'
  String arch = architecture == Architecture.AARCH64 ? '-aarch64' : ''

  final TaskProvider<Tar> buildDockerContextTask =
    tasks.register(taskName('build', architecture, base, 'DockerContext'), Tar) {
      archiveExtension = 'tar.gz'
      compression = Compression.GZIP
      archiveClassifier = "docker-build-context${arch}"
      archiveBaseName = "elasticsearch${base.suffix}"
      with dockerBuildContext(architecture, base)

      into(configDirectory) {
        from(configurations.log4jConfig) {
          filter TransformLog4jConfigFilter
        }
      }

      if (base == DockerBase.CLOUD) {
        // If we're performing a release build, but `build.id` hasn't been set, we can
        // infer that we're not at the Docker building stage of the build, and therefore
        // we should skip the beats part of the build.
        String buildId = providers.systemProperty('build.id').forUseAtConfigurationTime().getOrNull()
        boolean includeBeats = VersionProperties.isElasticsearchSnapshot() == true || buildId != null

        if (includeBeats) {
          from configurations.filebeat
          from configurations.metricbeat
        }
        // For some reason, the artifact name can differ depending on what repository we used.
        rename ~/((?:file|metric)beat)-.*\.tar\.gz$/, "\$1-${VersionProperties.elasticsearch}.tar.gz"
      }

      onlyIf { isArchitectureSupported(architecture) }
    }

  if (base == DockerBase.IRON_BANK) {
    tasks.named("assemble").configure {
      dependsOn(buildDockerContextTask)
    }
  }
}

void addTransformDockerContextTask(Architecture architecture, DockerBase base) {
  tasks.register(taskName("transform", architecture, base, "DockerContext"), Sync) {
    TaskProvider<Tar> buildContextTask = tasks.named(taskName("build", architecture, base, "DockerContext"))
    dependsOn(buildContextTask)

    String arch = architecture == Architecture.AARCH64 ? '-aarch64' : ''
    String archiveName = "elasticsearch${base.suffix}-${VersionProperties.elasticsearch}-docker-build-context${arch}"
    String distributionName = "elasticsearch-${VersionProperties.elasticsearch}-linux-${architecture.classifier}.tar.gz"

    from(tarTree("${project.buildDir}/distributions/${archiveName}.tar.gz")) {
      eachFile { FileCopyDetails details ->
        if (details.name.equals("Dockerfile")) {
          filter { String contents ->
            return contents.replaceAll('^RUN curl.*artifacts-no-kpi.*$', "COPY ${distributionName} /tmp/elasticsearch.tar.gz")
          }
        }
      }
    }
    into "${project.buildDir}/docker-context/${archiveName}"

    // Since we replaced the remote URL in the Dockerfile, copy in the required file
    if (architecture == Architecture.AARCH64) {
      from configurations.aarch64DockerSource
    } else {
      from configurations.dockerSource
    }

    if (base == DockerBase.IRON_BANK) {
      from (configurations.tini) {
        rename { _ -> 'tini' }
      }
    }

    expansions(architecture, base).findAll { it.key != 'build_date' }.each { k, v ->
      inputs.property(k, { v.toString() })
    }

    onlyIf { isArchitectureSupported(architecture) }
  }
}


private static List<String> generateTags(DockerBase base, Architecture architecture) {
  final String version = VersionProperties.elasticsearch

  String image = "elasticsearch${base.suffix}"

  String namespace = 'elasticsearch'
  if (base == DockerBase.CLOUD || base == DockerBase.CLOUD_ESS) {
    namespace += '-ci'
  }

  def tags = ["${image}:${architecture.classifier}"]

  if (architecture == Architecture.current()) {
    tags.addAll(["${image}:test", "${image}:${version}", "docker.elastic.co/${namespace}/${image}:${version}"])
  }

  return tags
}

void addBuildDockerImageTask(Architecture architecture, DockerBase base) {
  final TaskProvider<DockerBuildTask> buildDockerImageTask =
    tasks.register(taskName("build", architecture, base, "DockerImage"), DockerBuildTask) {

      TaskProvider<Sync> transformTask = tasks.named(taskName("transform", architecture, base, "DockerContext"))
      dependsOn(transformTask)

      dockerContext.fileProvider(transformTask.map { Sync task -> task.getDestinationDir() })

      noCache = BuildParams.isCi
      tags = generateTags(base, architecture)
      platform = architecture.dockerPlatform

      // We don't build the Iron Bank image when we release Elasticsearch, as there's
      // separate process for submitting new releases. However, for testing we do a
      // "test build" so that we can check that the Docker build at least works and
      // is functional.
      if (base == DockerBase.IRON_BANK) {
        Map<String, String> buildArgsMap = [
          'BASE_REGISTRY': 'docker.elastic.co',
          'BASE_IMAGE'   : 'ubi8/ubi',
          'BASE_TAG'     : 'latest'
        ]

        // Iron Bank has a single, parameterized base image
        String baseImage = base.image
        for (String key : buildArgsMap.keySet()) {
          baseImage = baseImage.replace('${' + key + '}', buildArgsMap.get(key))
        }

        baseImages = [baseImage]
        buildArgs = buildArgsMap
      } else {
        baseImages = [base.image]
      }

      onlyIf { isArchitectureSupported(architecture) }
    }

  if (base != DockerBase.IRON_BANK && base != DockerBase.CLOUD && base != DockerBase.CLOUD_ESS) {
    tasks.named("assemble").configure {
      dependsOn(buildDockerImageTask)
    }
  }
}

void addBuildEssDockerImageTask(Architecture architecture) {
  DockerBase base = DockerBase.CLOUD_ESS
  String arch = architecture == Architecture.AARCH64 ? '-aarch64' : ''
  String contextDir = "${project.buildDir}/docker-context/elasticsearch${base.suffix}-${VersionProperties.elasticsearch}-docker-build-context${arch}"

  final TaskProvider<Sync> buildContextTask =
    tasks.register(taskName('build', architecture, base, 'DockerContext'), Sync) {
      into contextDir

      final Path projectDir = project.projectDir.toPath()

      into("plugins") {
        from configurations.allPlugins
      }

      from(projectDir.resolve("src/docker/Dockerfile.cloud-ess")) {
        expand([
          base_image: "elasticsearch${DockerBase.CLOUD.suffix}:${architecture.classifier}"
        ])
        filter SquashNewlinesFilter
        rename ~/Dockerfile\.cloud-ess$/, 'Dockerfile'
      }
    }

  final TaskProvider<DockerBuildTask> buildDockerImageTask =
    tasks.register(taskName("build", architecture, base, "DockerImage"), DockerBuildTask) {

      TaskProvider<DockerBuildTask> buildCloudTask = tasks.named(taskName("build", architecture, DockerBase.CLOUD, "DockerImage"))
      inputs.files(buildCloudTask)

      dockerContext.fileProvider(buildContextTask.map { it.getDestinationDir() })

      noCache = BuildParams.isCi
      baseImages = []
      tags = generateTags(base, architecture)
      platform = architecture.dockerPlatform

      onlyIf { isArchitectureSupported(architecture) }
    }

  tasks.named("assemble").configure {
    dependsOn(buildDockerImageTask)
  }
}

for (final Architecture architecture : Architecture.values()) {
  for (final DockerBase base : DockerBase.values()) {
    if (base == DockerBase.CLOUD_ESS) {
      continue
    }
    addBuildDockerContextTask(architecture, base)
    addTransformDockerContextTask(architecture, base)
    addBuildDockerImageTask(architecture, base)
  }

  addBuildEssDockerImageTask(architecture)
}

boolean isArchitectureSupported(Architecture architecture) {
  Provider<DockerSupportService> serviceProvider = GradleUtils.getBuildService(project.gradle.sharedServices, DockerSupportPlugin.DOCKER_SUPPORT_SERVICE_NAME)
  return serviceProvider.get().dockerAvailability.supportedArchitectures().contains(architecture)
}

/*
 * The export subprojects write out the generated Docker images to disk, so
 * that they can be easily reloaded, for example into a VM for distribution testing
 */
subprojects { Project subProject ->
  if (subProject.name.endsWith('-export')) {
    apply plugin: 'distribution'

    final Architecture architecture = subProject.name.contains('aarch64-') ? Architecture.AARCH64 : Architecture.X64
    DockerBase base = DockerBase.DEFAULT
    if (subProject.name.contains('ubi-')) {
      base = DockerBase.UBI
    } else if (subProject.name.contains('ironbank-')) {
      base = DockerBase.IRON_BANK
    } else if (subProject.name.contains('cloud-ess-')) {
      base = DockerBase.CLOUD_ESS
    } else if (subProject.name.contains('cloud-')) {
      base = DockerBase.CLOUD
    }

    final String arch = architecture == Architecture.AARCH64 ? '-aarch64' : ''
    final String extension = base == DockerBase.UBI ? 'ubi.tar' :
      (base == DockerBase.IRON_BANK ? 'ironbank.tar' :
        (base == DockerBase.CLOUD ? 'cloud.tar' :
          (base == DockerBase.CLOUD_ESS ? 'cloud-ess.tar' :
            'docker.tar')))
    final String artifactName = "elasticsearch${arch}${base.suffix}_test"

    final String exportTaskName = taskName("export", architecture, base, 'DockerImage')
    final String buildTaskName = taskName('build', architecture, base, 'DockerImage')
    final String tarFile = "${parent.projectDir}/build/${artifactName}_${VersionProperties.elasticsearch}.${extension}"

    tasks.register(exportTaskName, LoggedExec) {
      inputs.file("${parent.projectDir}/build/markers/${buildTaskName}.marker")
      executable = 'docker'
      outputs.file(tarFile)
      args "save",
        "-o",
        tarFile,
        "elasticsearch${base.suffix}:${architecture.classifier}"

      dependsOn(parent.path + ":" + buildTaskName)
      onlyIf { isArchitectureSupported(architecture) }
    }

    artifacts.add('default', file(tarFile)) {
      type 'tar'
      name artifactName
      builtBy exportTaskName
    }
  }
}

tasks.named('resolveAllDependencies') {
  // Don't try and resolve filebeat or metricbeat snapshots as they may not always be available
  configs = configurations.matching { it.name.endsWith('beat') == false }
}