From 5928cdfc8a7e420166f598315889035d07491c44 Mon Sep 17 00:00:00 2001 From: Jason Gerlowski Date: Wed, 6 Dec 2023 16:40:58 -0500 Subject: [PATCH 1/7] SOLR-16880: Add OAS to 'assembleRelease' task This adds our OAS to the release artifacts gathered by the 'assembleRelease' task in gradle. It produces a checksum for the OAS file and stores the file in a separate 'oas' directory (similar to the 'changes' directory which already exists for Changes.html). One remaining piece here is signing - this commit attempts to produce an '*.asc' signature for the OAS when signing is enabled, but doesn't quite have it working yet. --- dev-tools/scripts/buildAndPushRelease.py | 3 ++- solr/api/build.gradle | 3 ++- solr/distribution/build.gradle | 13 +++++++++++++ 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/dev-tools/scripts/buildAndPushRelease.py b/dev-tools/scripts/buildAndPushRelease.py index 44416eed723..203337c79d0 100755 --- a/dev-tools/scripts/buildAndPushRelease.py +++ b/dev-tools/scripts/buildAndPushRelease.py @@ -30,7 +30,8 @@ import scriptutil LOG = '/tmp/release.log' -dev_mode = False +# NOCOMMIT +dev_mode = True def log(msg): f = open(LOG, mode='ab') diff --git a/solr/api/build.gradle b/solr/api/build.gradle index ecd93125e86..46548b40337 100644 --- a/solr/api/build.gradle +++ b/solr/api/build.gradle @@ -28,7 +28,7 @@ ext { jsClientDir = "${buildDir}/generated/js" pythonClientDir = "${buildDir}/generated/python" openApiSpecDir = "${buildDir}/generated/openapi" - openApiSpecFile = "${project.openApiSpecDir}/openapi.json" + openApiSpecFile = "${project.openApiSpecDir}/solr-${version}-openapi.json" } configurations { @@ -50,6 +50,7 @@ resolve { classpath = sourceSets.main.runtimeClasspath resourcePackages = ["org.apache.solr.client.api.util", "org.apache.solr.client.api.endpoint"] outputDir = file(project.openApiSpecDir) + outputFileName = "solr-${version}-openapi" prettyPrint = true } diff --git a/solr/distribution/build.gradle b/solr/distribution/build.gradle index d1ad6b62256..be3fcf8d37c 100644 --- a/solr/distribution/build.gradle +++ b/solr/distribution/build.gradle @@ -54,16 +54,19 @@ apply from: buildscript.sourceFile.toPath().resolveSibling("source-release.gradl // Set up the HTML-rendered "changes" distribution artifact by linking to documentation's output. configurations { changesHtml + openApiSpecFile docker } dependencies { changesHtml project(path: ":solr:documentation", configuration: "changesHtml") + openApiSpecFile project(path: ":solr:api", configuration: "openapiSpec") docker project(path: ':solr:docker', configuration: 'packagingOfficial') } def fullDistTarTask = rootProject.getTasksByName("fullDistTar", true)[0] def slimDistTarTask = rootProject.getTasksByName("slimDistTar", true)[0] +def oasTask = rootProject.getTasksByName("resolve", true)[0] // Compute checksums for release archives. task computeChecksums(type: Checksum) { @@ -74,6 +77,7 @@ task computeChecksums(type: Checksum) { tasks.assembleSourceTgz, fullDistTarTask, slimDistTarTask, + oasTask ].each { dep -> dependsOn dep files += dep.outputs.files @@ -93,11 +97,16 @@ task signSourceTgz(type: Sign) { dependsOn tasks.assembleSourceTgz sign tasks.assembleSourceTgz.destination } +task signOpenApiSpec(type: Sign) { + dependsOn oasTask + sign configurations.openApiSpecFile +} task signReleaseArchives(type: Sync) { from tasks.signFullBinaryTgz from tasks.signSlimBinaryTgz from tasks.signSourceTgz + from tasks.signOpenApiSpec into "${buildDir}/signatures" } @@ -125,6 +134,10 @@ task assembleRelease(type: Sync) { into "changes" }) + from(configurations.openApiSpecFile, { + into "oas" + }) + from(configurations.docker, { include 'Dockerfile.official*' into "docker" From 4a0ffab1a1877b778e11a0566d4f850c406e531e Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Thu, 7 Dec 2023 15:02:02 -0500 Subject: [PATCH 2/7] Add signatures and sha for openApiSpec --- solr/api/build.gradle | 2 +- solr/distribution/build.gradle | 15 +++++++++------ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/solr/api/build.gradle b/solr/api/build.gradle index 46548b40337..5062cd097fd 100644 --- a/solr/api/build.gradle +++ b/solr/api/build.gradle @@ -92,7 +92,7 @@ tasks.withType(org.openapitools.generator.gradle.plugin.tasks.GenerateTask) { artifacts { // Ensure the OAS is available to other modules who want to generate code (i.e. solrj) - openapiSpec resolve.outputDir, { + openapiSpec file(openApiSpecFile), { builtBy resolve } diff --git a/solr/distribution/build.gradle b/solr/distribution/build.gradle index be3fcf8d37c..7a0f1a308d7 100644 --- a/solr/distribution/build.gradle +++ b/solr/distribution/build.gradle @@ -66,7 +66,6 @@ dependencies { def fullDistTarTask = rootProject.getTasksByName("fullDistTar", true)[0] def slimDistTarTask = rootProject.getTasksByName("slimDistTar", true)[0] -def oasTask = rootProject.getTasksByName("resolve", true)[0] // Compute checksums for release archives. task computeChecksums(type: Checksum) { @@ -76,12 +75,13 @@ task computeChecksums(type: Checksum) { [ tasks.assembleSourceTgz, fullDistTarTask, - slimDistTarTask, - oasTask + slimDistTarTask ].each { dep -> dependsOn dep files += dep.outputs.files } + dependsOn configurations.openApiSpecFile + files += configurations.openApiSpecFile outputDir = file("${buildDir}/checksums") } @@ -98,8 +98,11 @@ task signSourceTgz(type: Sign) { sign tasks.assembleSourceTgz.destination } task signOpenApiSpec(type: Sign) { - dependsOn oasTask - sign configurations.openApiSpecFile + dependsOn configurations.openApiSpecFile + // This is not an artifact, so we need to use "file" not "configuration" signing + doFirst { + sign configurations.openApiSpecFile.singleFile + } } task signReleaseArchives(type: Sync) { @@ -135,7 +138,7 @@ task assembleRelease(type: Sync) { }) from(configurations.openApiSpecFile, { - into "oas" + into "openApi" }) from(configurations.docker, { From 475acef268494c347e1ea44a9dbb1f1cb9a38ac3 Mon Sep 17 00:00:00 2001 From: Jason Gerlowski Date: Thu, 7 Dec 2023 16:23:16 -0500 Subject: [PATCH 3/7] File tweaks --- solr/distribution/build.gradle | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/solr/distribution/build.gradle b/solr/distribution/build.gradle index 7a0f1a308d7..601a59fa3ef 100644 --- a/solr/distribution/build.gradle +++ b/solr/distribution/build.gradle @@ -155,11 +155,26 @@ task assembleRelease(type: Sync) { from fullDistTarTask from slimDistTarTask - from tasks.computeChecksums + from(tasks.computeChecksums, { + exclude { it.file.getName().contains("openapi") } + //excludes ["**/*openapi.json.sha512"] + }) + from(tasks.computeChecksums, { + include { it.file.getName().contains("openapi") } + //includes ["**/*openapi.json.sha512"] + into "openApi" + }) // Conditionally, attach signatures of all the release archives. if (project.ext.withSignedArtifacts) { - from tasks.signReleaseArchives + from(tasks.signReleaseArchives, { + exclude { it.file.getName().contains("openapi") } + }) + + from(tasks.signReleaseArchives, { + include { it.file.getName().contains("openapi") } + into "openApi" + }) } into releaseDir From 60f3fd3533fe6efe3935b35e65551f398785c537 Mon Sep 17 00:00:00 2001 From: Jason Gerlowski Date: Wed, 13 Dec 2023 12:50:11 -0500 Subject: [PATCH 4/7] Address review comments, rd 1 --- solr/api/build.gradle | 4 ++-- solr/distribution/build.gradle | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/solr/api/build.gradle b/solr/api/build.gradle index 5062cd097fd..bbcf66170e1 100644 --- a/solr/api/build.gradle +++ b/solr/api/build.gradle @@ -28,7 +28,7 @@ ext { jsClientDir = "${buildDir}/generated/js" pythonClientDir = "${buildDir}/generated/python" openApiSpecDir = "${buildDir}/generated/openapi" - openApiSpecFile = "${project.openApiSpecDir}/solr-${version}-openapi.json" + openApiSpecFile = "${project.openApiSpecDir}/solr-openapi-${version}.json" } configurations { @@ -50,7 +50,7 @@ resolve { classpath = sourceSets.main.runtimeClasspath resourcePackages = ["org.apache.solr.client.api.util", "org.apache.solr.client.api.endpoint"] outputDir = file(project.openApiSpecDir) - outputFileName = "solr-${version}-openapi" + outputFileName = "solr-openapi-${version}" prettyPrint = true } diff --git a/solr/distribution/build.gradle b/solr/distribution/build.gradle index 601a59fa3ef..8ebc5872f53 100644 --- a/solr/distribution/build.gradle +++ b/solr/distribution/build.gradle @@ -157,11 +157,9 @@ task assembleRelease(type: Sync) { from(tasks.computeChecksums, { exclude { it.file.getName().contains("openapi") } - //excludes ["**/*openapi.json.sha512"] }) from(tasks.computeChecksums, { include { it.file.getName().contains("openapi") } - //includes ["**/*openapi.json.sha512"] into "openApi" }) From 70ffc11c604978e1e4e5f2d31d59bca9333fb497 Mon Sep 17 00:00:00 2001 From: Jason Gerlowski Date: Thu, 14 Dec 2023 11:42:31 -0500 Subject: [PATCH 5/7] Update smoke-tester to validate OAS --- dev-tools/scripts/smokeTestRelease.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/dev-tools/scripts/smokeTestRelease.py b/dev-tools/scripts/smokeTestRelease.py index 554c664559d..3d57f6d98f9 100755 --- a/dev-tools/scripts/smokeTestRelease.py +++ b/dev-tools/scripts/smokeTestRelease.py @@ -220,6 +220,7 @@ def checkSigs(urlString, version, tmpDir, isSigned, keysFile): ents = getDirEntries(urlString) artifact = None changesURL = None + openApiURL = None mavenURL = None dockerURL = None artifactURL = None @@ -243,6 +244,10 @@ def checkSigs(urlString, version, tmpDir, isSigned, keysFile): if text not in ('changes/', 'changes-%s/' % version): raise RuntimeError('solr: found %s vs expected changes-%s/' % (text, version)) changesURL = subURL + elif text.startswith('openApi'): + if text not in ('openApi/', 'openApi-%s/' % version): + raise RuntimeError('solr: found %s vs expected openApi-%s/' % (text, version)) + openApiURL = subURL elif artifact is None: artifact = text artifactURL = subURL @@ -296,6 +301,12 @@ def checkSigs(urlString, version, tmpDir, isSigned, keysFile): raise RuntimeError('solr is missing changes-%s' % version) testChanges(version, changesURL) + if openApiURL is None: + stopGpgAgent(gpgHomeDir, logfile) + raise RuntimeError('solr is missing OpenAPI specification' % version) + testOpenApi(version, openApiURL) + + for artifact, urlString in artifacts: # pylint: disable=redefined-argument-from-local print(' download %s...' % artifact) scriptutil.download(artifact, urlString, tmpDir, force_clean=FORCE_CLEAN) @@ -349,6 +360,17 @@ def testChanges(version, changesURLString): s = load(changesURL) checkChangesContent(s, version, changesURL, True) +def testOpenApi(version, openApiDirUrl): + print(' check OpenAPI specification...') + specFound = False + expectedSpecFileName = 'solr-openapi-' + version + '.json' + for text, subURL in getDirEntries(openApiDirUrl): + if text == expectedSpecFileName: + specFound = True + + if not specFound: + raise RuntimeError('Did not see %s in %s' % expectedSpecFileName, openApiDirUrl) + def testChangesText(dir, version): "Checks all CHANGES.txt under this dir." From 23bc6451fab6fcded30f314c59aca68343cee7c4 Mon Sep 17 00:00:00 2001 From: Jason Gerlowski Date: Thu, 14 Dec 2023 11:48:30 -0500 Subject: [PATCH 6/7] Remove debug code --- dev-tools/scripts/buildAndPushRelease.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dev-tools/scripts/buildAndPushRelease.py b/dev-tools/scripts/buildAndPushRelease.py index 203337c79d0..44416eed723 100755 --- a/dev-tools/scripts/buildAndPushRelease.py +++ b/dev-tools/scripts/buildAndPushRelease.py @@ -30,8 +30,7 @@ import scriptutil LOG = '/tmp/release.log' -# NOCOMMIT -dev_mode = True +dev_mode = False def log(msg): f = open(LOG, mode='ab') From c4ae30b92655b5a8ab1879d764671a1fcba8af9c Mon Sep 17 00:00:00 2001 From: Jason Gerlowski Date: Thu, 14 Dec 2023 12:16:34 -0500 Subject: [PATCH 7/7] Add CHANGES.txt entry --- solr/CHANGES.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 2fd94304a24..9505b2c6f53 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -171,6 +171,9 @@ Other Changes * SOLR-16949: Restrict certain file types from being uploaded to or downloaded from Config Sets (janhoy, Houston Putman) +* SOLR-16880: Solr now produces an OpenAPI Specification artifact on releases ("solr-openapi-x.y.z.json") that covers + Solr's v2 APIs. (Jason Gerlowski, Houston Putman) + ================== 9.4.0 ================== New Features ---------------------