Skip to content

SOLR-15852 Update smoketester for 9.0 #581

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
janhoy merged 5 commits into from
Feb 2, 2022
Merged

SOLR-15852 Update smoketester for 9.0 #581

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
6bed449
Update-smoketester
janhoy Jan 31, 2022
1e0db8a
Merge branch 'main' into smoketester90
janhoy Jan 31, 2022
e0bc213
SOLR-15852 Update smokeTestRelease.py for 9.0
janhoy Jan 31, 2022
e65f13f
SOLR-15972 Revert allowing gradle-wrapper.jar in source release
janhoy Feb 2, 2022
8afc63e
Remove cp-deps
janhoy Feb 2, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion dev-tools/scripts/scriptutil.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -156,7 +156,10 @@ def download(name, urlString, tmpDir, quiet=False, force_clean=True):


def attemptDownload(urlString, fileName):
fIn = urllib.request.urlopen(urlString)
raw_request = urllib.request.Request(urlString)
raw_request.add_header('User-Agent', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:78.0) Gecko/20100101 Firefox/78.0')
Copy link
Contributor

@madrob madrob Feb 1, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm curious about what requires this

Copy link
Contributor Author

@janhoy janhoy Feb 1, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I generate some test releases before the real RC, and this is needed to be able to download artifacts from a plain httpd server like this, instead of dist.apache.org. It adds a header to avoid some 406 response, and also pulls the file name from href instead of display-part of the tag, since httpd will abbreviate long file names with dots.

raw_request.add_header('Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8')
fIn = urllib.request.urlopen(raw_request)
fOut = open(fileName, 'wb')
success = False
try:
Expand Down
24 changes: 18 additions & 6 deletions dev-tools/scripts/smokeTestRelease.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,14 +93,18 @@ def getHREFs(urlString):
raise

for subUrl, text in reHREF.findall(html):
#print("Got suburl %s and text %s" % (subUrl, text))
fullURL = urllib.parse.urljoin(urlString, subUrl)
links.append((text, fullURL))
return links


def load(urlString):
try:
content = urllib.request.urlopen(urlString).read().decode('utf-8')
raw_request = urllib.request.Request(urlString)
raw_request.add_header('User-Agent', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:78.0) Gecko/20100101 Firefox/78.0')
raw_request.add_header('Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8')
content = urllib.request.urlopen(raw_request).read().decode('utf-8')
except Exception as e:
print('Retrying download of url %s after exception: %s' % (urlString, e))
content = urllib.request.urlopen(urlString).read().decode('utf-8')
Expand Down Expand Up @@ -140,7 +144,7 @@ def checkJARMetaData(desc, jarFile, gitRevision, version):
for verify in (
'Specification-Vendor: The Apache Software Foundation',
'Implementation-Vendor: The Apache Software Foundation',
'Specification-Title: Solr Search Engine:',
'Specification-Title: Apache Solr Search Server:',
'Implementation-Title: org.apache.solr',
'X-Compile-Source-JDK: 11',
'X-Compile-Target-JDK: 11',
Expand Down Expand Up @@ -215,6 +219,7 @@ def checkSigs(urlString, version, tmpDir, isSigned, keysFile):
artifact = None
changesURL = None
mavenURL = None
dockerURL = None
artifactURL = None
expectedSigs = []
if isSigned:
Expand All @@ -228,6 +233,8 @@ def checkSigs(urlString, version, tmpDir, isSigned, keysFile):
raise RuntimeError('solr: release dir should not contain a KEYS file - only toplevel /dist/solr/KEYS is used')
elif text == 'maven/':
mavenURL = subURL
elif text == 'docker/':
dockerURL = subURL
elif text.startswith('changes'):
if text not in ('changes/', 'changes-%s/' % version):
raise RuntimeError('solr: found %s vs expected changes-%s/' % (text, version))
Expand All @@ -240,7 +247,7 @@ def checkSigs(urlString, version, tmpDir, isSigned, keysFile):
raise RuntimeError('solr: unknown artifact %s: expected prefix %s' % (text, expected))
sigs = []
elif text.startswith(artifact + '.'):
sigs.append(text[len(artifact)+1:])
sigs.append(subURL.rsplit(".")[-1:][0])
else:
if sigs != expectedSigs:
raise RuntimeError('solr: artifact %s has wrong sigs: expected %s but got %s' % (artifact, expectedSigs, sigs))
Expand Down Expand Up @@ -272,6 +279,9 @@ def checkSigs(urlString, version, tmpDir, isSigned, keysFile):
if mavenURL is None:
raise RuntimeError('solr is missing maven')

if dockerURL is None:
raise RuntimeError('solr is missing docker')

if changesURL is None:
raise RuntimeError('solr is missing changes-%s' % version)
testChanges(version, changesURL)
Expand Down Expand Up @@ -578,15 +588,15 @@ def verifyUnpacked(java, artifact, unpackPath, gitRevision, version, testArgs):
expected_src_root_folders = ['buildSrc', 'dev-docs', 'dev-tools', 'gradle', 'help', 'solr']
expected_src_root_files = ['build.gradle', 'gradlew', 'gradlew.bat', 'settings.gradle', 'versions.lock', 'versions.props']
expected_src_solr_files = ['build.gradle']
expected_src_solr_folders = ['benchmark', 'bin', 'bin-test', 'build', 'modules', 'core', 'docker', 'documentation', 'example', 'licenses', 'packaging', 'server', 'solr-ref-guide', 'solrj', 'test-framework', 'webapp']
expected_src_solr_folders = ['benchmark', 'bin', 'bin-test', 'modules', 'core', 'docker', 'documentation', 'example', 'licenses', 'packaging', 'distribution', 'prometheus-exporter', 'server', 'solr-ref-guide', 'solrj', 'test-framework', 'webapp', '.gitignore', '.gitattributes']
is_in_list(in_root_folder, expected_src_root_folders)
is_in_list(in_root_folder, expected_src_root_files)
is_in_list(in_solr_folder, expected_src_solr_folders)
is_in_list(in_solr_folder, expected_src_solr_files)
if len(in_solr_folder) > 0:
raise RuntimeError('solr: unexpected files/dirs in artifact %s solr/ folder: %s' % (artifact, in_solr_folder))
else:
is_in_list(in_root_folder, ['bin', 'modules', 'dist', 'docs', 'example', 'licenses', 'server'])
is_in_list(in_root_folder, ['bin', 'modules', 'docker', 'prometheus-exporter', 'docs', 'example', 'licenses', 'server'])

if len(in_root_folder) > 0:
raise RuntimeError('solr: unexpected files/dirs in artifact %s: %s' % (artifact, in_root_folder))
Expand Down Expand Up @@ -1011,6 +1021,8 @@ def parse_config():
help='Path to Java17 home directory, to run tests with if specified')
parser.add_argument('--download-only', action='store_true', default=False,
help='Only perform download and sha hash check steps')
parser.add_argument('--dev-mode', action='store_true', default=False,
help='Enable dev mode, will not check branch compatibility')
parser.add_argument('url', help='Url pointing to release to test')
parser.add_argument('test_args', nargs=argparse.REMAINDER,
help='Arguments to pass to gradle for testing, e.g. -Dwhat=ever.')
Expand Down Expand Up @@ -1058,7 +1070,7 @@ def main():

# Pick <major>.<minor> part of version and require script to be from same branch
scriptVersion = re.search(r'((\d+).(\d+)).(\d+)', scriptutil.find_current_version()).group(1).strip()
if not c.version.startswith(scriptVersion + '.'):
if not c.version.startswith(scriptVersion + '.') and not c.dev_mode:
raise RuntimeError('smokeTestRelease.py for %s.X is incompatible with a %s release.' % (scriptVersion, c.version))

print('NOTE: output encoding is %s' % sys.stdout.encoding)
Expand Down