Skip to content
This repository has been archived by the owner on Dec 31, 2019. It is now read-only.

Commit

Permalink
Fix wire
Browse files Browse the repository at this point in the history
  • Loading branch information
@JohanLorenzo
JohanLorenzo committed Jul 23, 2018
1 parent ef5fe70 commit d7f7a1d
Show file tree
Hide file tree
Showing 5 changed files with 63 additions and 32 deletions.
17 changes: 11 additions & 6 deletions beetmoverscript/script.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,15 +17,18 @@
from scriptworker.exceptions import ScriptWorkerTaskException, ScriptWorkerRetryException
from scriptworker.utils import retry_async, raise_future_exceptions

from beetmoverscript import task, zip

from beetmoverscript.constants import (
MIME_MAP, RELEASE_BRANCHES, CACHE_CONTROL_MAXAGE, RELEASE_EXCLUDE,
NORMALIZED_BALROG_PLATFORMS, PARTNER_REPACK_PUBLIC_PREFIX_TMPL,
PARTNER_REPACK_PRIVATE_REGEXES, PARTNER_REPACK_PUBLIC_REGEXES, BUILDHUB_ARTIFACT,
INSTALLER_ARTIFACTS, ZIP_MAX_COMPRESSION_RATIO
)
from beetmoverscript.maven import get_maven_expected_files_per_archive_per_task_id
from beetmoverscript.task import (
validate_task_schema, add_balrog_manifest_to_artifacts,
get_upstream_artifacts, get_upstream_artifacts_with_zip_extract_param, get_release_props,
get_upstream_artifacts, get_release_props,
add_checksums_to_artifacts, get_task_bucket, get_task_action, validate_bucket_paths,
get_updated_buildhub_artifact
)
Expand All @@ -38,7 +41,6 @@
get_product_name, is_partner_action, is_partner_private_task,
is_partner_public_task, write_json
)
from beetmoverscript.zip import check_and_extract_zip_archives

log = logging.getLogger(__name__)

Expand Down Expand Up @@ -157,18 +159,21 @@ async def push_to_releases(context):

async def push_to_maven(context):
"""Push artifacts to locations expected by maven clients (like mvn or gradle)"""
context.artifacts_to_beetmove = get_upstream_artifacts_with_zip_extract_param(context)
context.artifacts_to_beetmove = task.get_upstream_artifacts_with_zip_extract_param(context)
context.release_props = get_release_props(context)
context.checksums = dict() # Needed by downstream calls

mapping_manifest = generate_beetmover_manifest(context)
validate_bucket_paths(context.bucket, mapping_manifest['s3_bucket_path'])

expected_files = list(mapping_manifest['mapping'].keys())
expected_files = get_maven_expected_files_per_archive_per_task_id(
context.task['payload']['upstreamArtifacts'], mapping_manifest
)

context.artifacts_to_beetmove = {
'en-US': check_and_extract_zip_archives(
context.artifacts_to_beetmove, expected_files,
'en-US': zip.check_and_extract_zip_archives(
context.artifacts_to_beetmove,
expected_files,
context.config.get('zip_extract_max_file_size_in_mb', ZIP_MAX_COMPRESSION_RATIO)
)
}
Expand Down
24 changes: 12 additions & 12 deletions beetmoverscript/templates/maven_geckoview.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,51 +10,51 @@ mapping:
{% for locale in ['en-US'] %}
"{{ locale }}": # Locale is not needed for geckoview, it's used by move_beets, though
{% for product in ['geckoview'] %}
"{{ product }}.aar":
"{{ artifact_id }}-{{ version }}.aar":
s3_key: {{ artifact_id }}-{{ version }}.aar
destinations:
- {{ artifact_id }}-{{ version }}.aar
"{{ product }}.aar.md5":
"{{ artifact_id }}-{{ version }}.aar.md5":
s3_key: {{ artifact_id }}-{{ version }}.aar.md5
destinations:
- {{ artifact_id }}-{{ version }}.aar.md5
"{{ product }}.aar.sha1":
"{{ artifact_id }}-{{ version }}.aar.sha1":
s3_key: {{ artifact_id }}-{{ version }}.aar.sha1
destinations:
- {{ artifact_id }}-{{ version }}.aar.sha1
"{{ product }}.pom":
"{{ artifact_id }}-{{ version }}.pom":
s3_key: {{ artifact_id }}-{{ version }}.pom
destinations:
- {{ artifact_id }}-{{ version }}.pom
"{{ product }}.pom.md5":
"{{ artifact_id }}-{{ version }}.pom.md5":
s3_key: {{ artifact_id }}-{{ version }}.pom.md5
destinations:
- {{ artifact_id }}-{{ version }}.pom.md5
"{{ product }}.pom.sha1":
"{{ artifact_id }}-{{ version }}.pom.sha1":
s3_key: {{ artifact_id }}-{{ version }}.pom.sha1
destinations:
- {{ artifact_id }}-{{ version }}.pom.sha1
"{{ product }}-javadoc.jar":
"{{ artifact_id }}-{{ version }}-javadoc.jar":
s3_key: {{ artifact_id }}-{{ version }}-javadoc.jar
destinations:
- {{ artifact_id }}-{{ version }}-javadoc.jar
"{{ product }}-javadoc.jar.md5":
"{{ artifact_id }}-{{ version }}-javadoc.jar.md5":
s3_key: {{ artifact_id }}-{{ version }}-javadoc.jar.md5
destinations:
- {{ artifact_id }}-{{ version }}-javadoc.jar.md5
"{{ product }}-javadoc.jar.sha1":
"{{ artifact_id }}-{{ version }}-javadoc.jar.sha1":
s3_key: {{ artifact_id }}-{{ version }}-javadoc.jar.sha1
destinations:
- {{ artifact_id }}-{{ version }}-javadoc.jar.sha1
"{{ product }}-sources.jar":
"{{ artifact_id }}-{{ version }}-sources.jar":
s3_key: {{ artifact_id }}-{{ version }}-sources.jar
destinations:
- {{ artifact_id }}-{{ version }}-sources.jar
"{{ product }}-sources.jar.md5":
"{{ artifact_id }}-{{ version }}-sources.jar.md5":
s3_key: {{ artifact_id }}-{{ version }}-sources.jar.md5
destinations:
- {{ artifact_id }}-{{ version }}-sources.jar.md5
"{{ product }}-sources.jar.sha1":
"{{ artifact_id }}-{{ version }}-sources.jar.sha1":
s3_key: {{ artifact_id }}-{{ version }}-sources.jar.sha1
destinations:
- {{ artifact_id }}-{{ version }}-sources.jar.sha1
Expand Down
28 changes: 27 additions & 1 deletion beetmoverscript/test/test_script.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,8 +92,34 @@ def fake_list(*args):

@pytest.mark.asyncio
async def test_push_to_maven(context, mocker):
mocker.patch('beetmoverscript.script.move_beets', new=noop_async)
async def assert_artifacts_to_beetmove(_, artifacts_to_beetmove, __):
assert artifacts_to_beetmove == {'en-US': [
'/work_dir/cot/someTaskId/public/build/target.maven.zip.out/org/mozilla/geckoview-beta-x86/62.0b3/geckoview-beta-x86-62.0b3.aar',
'/work_dir/cot/someTaskId/public/build/target.maven.zip.out/org/mozilla/geckoview-beta-x86/62.0b3/geckoview-beta-x86-62.0b3.aar.md5',
'/work_dir/cot/someTaskId/public/build/target.maven.zip.out/org/mozilla/geckoview-beta-x86/62.0b3/geckoview-beta-x86-62.0b3.aar.sha1',
]}

mocker.patch('beetmoverscript.script.move_beets', new=assert_artifacts_to_beetmove)

mocker.patch('beetmoverscript.utils.JINJA_ENV', get_test_jinja_env())
context.task['payload']['upstreamArtifacts'] = [{
'paths': ['public/build/target.maven.zip'],
'taskId': 'someTaskId',
'taskType': 'build',
'zipExtract': True,
}]
mocker.patch('beetmoverscript.task.get_upstream_artifacts_with_zip_extract_param', new=lambda _: {
'someTaskId': [{
'paths': ['/work_dir/cot/someTaskId/public/build/target.maven.zip'],
'zip_extract': True,
}]
})
mocker.patch('beetmoverscript.zip.check_and_extract_zip_archives', new=lambda _, __, ___: [
'/work_dir/cot/someTaskId/public/build/target.maven.zip.out/org/mozilla/geckoview-beta-x86/62.0b3/geckoview-beta-x86-62.0b3.aar',
'/work_dir/cot/someTaskId/public/build/target.maven.zip.out/org/mozilla/geckoview-beta-x86/62.0b3/geckoview-beta-x86-62.0b3.aar.md5',
'/work_dir/cot/someTaskId/public/build/target.maven.zip.out/org/mozilla/geckoview-beta-x86/62.0b3/geckoview-beta-x86-62.0b3.aar.sha1',
])

await push_to_maven(context)


Expand Down
24 changes: 12 additions & 12 deletions beetmoverscript/test/test_utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,51 +98,51 @@ def test_generate_manifest_maven(context, mocker):
assert generate_beetmover_manifest(context) == {
'mapping': {
'en-US': {
'geckoview-javadoc.jar': {
'geckoview-nightly-x86-63.0.20181231120000-javadoc.jar': {
'destinations': ['geckoview-nightly-x86-63.0.20181231120000-javadoc.jar'],
's3_key': 'geckoview-nightly-x86-63.0.20181231120000-javadoc.jar',
},
'geckoview-javadoc.jar.md5': {
'geckoview-nightly-x86-63.0.20181231120000-javadoc.jar.md5': {
'destinations': ['geckoview-nightly-x86-63.0.20181231120000-javadoc.jar.md5'],
's3_key': 'geckoview-nightly-x86-63.0.20181231120000-javadoc.jar.md5',
},
'geckoview-javadoc.jar.sha1': {
'geckoview-nightly-x86-63.0.20181231120000-javadoc.jar.sha1': {
'destinations': ['geckoview-nightly-x86-63.0.20181231120000-javadoc.jar.sha1'],
's3_key': 'geckoview-nightly-x86-63.0.20181231120000-javadoc.jar.sha1',
},
'geckoview-sources.jar': {
'geckoview-nightly-x86-63.0.20181231120000-sources.jar': {
'destinations': ['geckoview-nightly-x86-63.0.20181231120000-sources.jar'],
's3_key': 'geckoview-nightly-x86-63.0.20181231120000-sources.jar',
},
'geckoview-sources.jar.md5': {
'geckoview-nightly-x86-63.0.20181231120000-sources.jar.md5': {
'destinations': ['geckoview-nightly-x86-63.0.20181231120000-sources.jar.md5'],
's3_key': 'geckoview-nightly-x86-63.0.20181231120000-sources.jar.md5',
},
'geckoview-sources.jar.sha1': {
'geckoview-nightly-x86-63.0.20181231120000-sources.jar.sha1': {
'destinations': ['geckoview-nightly-x86-63.0.20181231120000-sources.jar.sha1'],
's3_key': 'geckoview-nightly-x86-63.0.20181231120000-sources.jar.sha1',
},
'geckoview.aar': {
'geckoview-nightly-x86-63.0.20181231120000.aar': {
'destinations': ['geckoview-nightly-x86-63.0.20181231120000.aar'],
's3_key': 'geckoview-nightly-x86-63.0.20181231120000.aar',
},
'geckoview.aar.md5': {
'geckoview-nightly-x86-63.0.20181231120000.aar.md5': {
'destinations': ['geckoview-nightly-x86-63.0.20181231120000.aar.md5'],
's3_key': 'geckoview-nightly-x86-63.0.20181231120000.aar.md5',
},
'geckoview.aar.sha1': {
'geckoview-nightly-x86-63.0.20181231120000.aar.sha1': {
'destinations': ['geckoview-nightly-x86-63.0.20181231120000.aar.sha1'],
's3_key': 'geckoview-nightly-x86-63.0.20181231120000.aar.sha1',
},
'geckoview.pom': {
'geckoview-nightly-x86-63.0.20181231120000.pom': {
'destinations': ['geckoview-nightly-x86-63.0.20181231120000.pom'],
's3_key': 'geckoview-nightly-x86-63.0.20181231120000.pom',
},
'geckoview.pom.md5': {
'geckoview-nightly-x86-63.0.20181231120000.pom.md5': {
'destinations': ['geckoview-nightly-x86-63.0.20181231120000.pom.md5'],
's3_key': 'geckoview-nightly-x86-63.0.20181231120000.pom.md5',
},
'geckoview.pom.sha1': {
'geckoview-nightly-x86-63.0.20181231120000.pom.sha1': {
'destinations': ['geckoview-nightly-x86-63.0.20181231120000.pom.sha1'],
's3_key': 'geckoview-nightly-x86-63.0.20181231120000.pom.sha1'
},
Expand Down
2 changes: 1 addition & 1 deletion beetmoverscript/zip.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,7 @@ def _ensure_all_expected_files_are_present_in_archive(zip_path, files_in_archive
)
if file not in unique_expected_files:
raise TaskVerificationError(
'File "{}" is present in archive "{}" is not expected.'.format(
'File "{}" present in archive "{}" is not expected.'.format(
file, zip_path
)
)
Expand Down

0 comments on commit d7f7a1d

Please sign in to comment.