From 2353200fe87e6ca204a70c08fd84eeeb75d631bc Mon Sep 17 00:00:00 2001 From: Rok Garbas Date: Tue, 6 Aug 2019 07:50:21 -0600 Subject: [PATCH] Bug 1555970 - switch mapper auth to taskcluster. r=dhouse --HG-- extra : amend_source : 5614c9bae0b3dbc171cab0cf5dbb4807fe43074e --- configs/vcs_sync/beagle.py | 1 + configs/vcs_sync/build-repos.py | 1 + scripts/vcs-sync/vcs_sync.py | 62 +++++++++++++++++++++++++++++---- 3 files changed, 57 insertions(+), 7 deletions(-) diff --git a/configs/vcs_sync/beagle.py b/configs/vcs_sync/beagle.py index 1a3ee342..ce1079b3 100644 --- a/configs/vcs_sync/beagle.py +++ b/configs/vcs_sync/beagle.py @@ -271,6 +271,7 @@ "mozinfo==0.5", "mozprocess==0.11", "requests==2.8.1", + "mohawk==1.0.0", ], "find_links": [ "http://pypi.pvt.build.mozilla.org/pub", diff --git a/configs/vcs_sync/build-repos.py b/configs/vcs_sync/build-repos.py index 189e5cbb..96b052ee 100644 --- a/configs/vcs_sync/build-repos.py +++ b/configs/vcs_sync/build-repos.py @@ -164,6 +164,7 @@ "mozinfo==0.5", "mozprocess==0.11", "requests==2.8.1", + "mohawk==1.0.0", ], "find_links": [ "http://pypi.pub.build.mozilla.org/pub" diff --git a/scripts/vcs-sync/vcs_sync.py b/scripts/vcs-sync/vcs_sync.py index 6c8d5f5d..929d779b 100755 --- a/scripts/vcs-sync/vcs_sync.py +++ b/scripts/vcs-sync/vcs_sync.py @@ -983,10 +983,7 @@ def publish_to_mapper(self): mapper_url = mapper_config['url'] mapper_project = mapper_config['project'] insert_url = "%s/%s/insert/ignoredups" % (mapper_url, mapper_project) - headers = { - 'Content-Type': 'text/plain', - 'Authentication': 'Bearer %s' % os.environ["RELENGAPI_INSERT_HGGIT_MAPPINGS_AUTH_TOKEN"] - } + all_new_mappings = [] all_new_mappings.extend(self.pull_out_new_sha_lookups(published_to_mapper, complete_mapfile)) self.write_to_file(delta_for_mapper, "".join(all_new_mappings)) @@ -1018,8 +1015,12 @@ def check_output(*popenargs, **kwargs): error.output = output raise error return output - lines_last_time = int(check_output('wc -l <%s' % published_to_mapper, shell=True)) - lines_this_time = int(check_output('wc -l <%s' % complete_mapfile, shell=True)) + lines_last_time = 0 + if os.path.exists(published_to_mapper): + lines_last_time = int(check_output('wc -l <%s' % published_to_mapper, shell=True)) + lines_this_time = 0 + if os.path.exists(complete_mapfile): + lines_this_time = int(check_output('wc -l <%s' % complete_mapfile, shell=True)) if lines_this_time - lines_last_time != len(all_new_mappings): self.error("Bad calc of new mappings: last %d, now %d, diff %d, calc %d" % (lines_last_time, lines_this_time, lines_this_time - lines_last_time, @@ -1032,12 +1033,58 @@ def check_output(*popenargs, **kwargs): if retcode != 1: self.error("Bad selection of new mappings, some already there") + # create authentication headers + content_type = 'text/plain' + tc_client_id = os.environ.get( + 'RELENGAPI_INSERT_HGGIT_MAPPINGS_TASKCLUSTER_CLIENT_ID') + tc_access_token = os.environ.get( + 'RELENGAPI_INSERT_HGGIT_MAPPINGS_TASKCLUSTER_ACCESS_TOKEN') + relengapi_token = os.environ.get( + 'RELENGAPI_INSERT_HGGIT_MAPPINGS_AUTH_TOKEN') + + # For taskcluster auth, we only import mohawk since we need + # content to create the header + try: + import mohawk + except ImportError as e: + self.fatal("Can't import mohawk: %s\nDid you create-virtualenv?" % str(e)) + # due to timeouts on load balancer, we only push 200 lines at a time # this means that we should get http response back within 30 seconds # including the time it takes to insert the mappings in the database publish_successful = True + for i in range(0, len(all_new_mappings), 200): - r = requests.post(insert_url, data="".join(all_new_mappings[i:i+200]), headers=headers) + data = "".join(all_new_mappings[i:i+200]) + + if tc_client_id and tc_access_token: + headers = { + 'Content-Type': content_type, + 'Authentication': mohawk.Sender( + credentials=dict( + id=tc_client_id, + key=tc_access_token, + algorithm='sha256', + ), + ext=dict(), + url=insert_url, + content=data, + content_type=content_type, + method='POST', + ).request_header, + } + elif relengapi_token: + headers = { + 'Content-Type': content_type, + 'Authentication': 'Bearer %s' % relengapi_token, + } + else: + self.fatal( + "Please provide either:\n" + "- RELENGAPI_INSERT_HGGIT_MAPPINGS_AUTH_TOKEN\n" + "- RELENGAPI_INSERT_HGGIT_MAPPINGS_TASKCLUSTER_ACCESS_TOKEN and RELENGAPI_INSERT_HGGIT_MAPPINGS_TASKCLUSTER_CLIENT_ID") + + r = requests.post(insert_url, data=data, headers=headers) if (r.status_code != 200): self.error("Could not publish mapfile ('%s') line range [%s, %s] to mapper (%s) - received http %s code" % (delta_for_mapper, i, i+200, insert_url, r.status_code)) publish_successful = False @@ -1046,6 +1093,7 @@ def check_output(*popenargs, **kwargs): # time anyway else: self.info("Published mapfile ('%s') line range [%s, %s] to mapper (%s)" % (delta_for_mapper, i, i+200, insert_url)) + if publish_successful: # bug 1193011 says there are problems on occasion # with delta uploads. Check that items are in db in