diff --git a/.github/workflows/python-linters.yml b/.github/workflows/python-linters.yml index 5ca1d44..fb459a0 100644 --- a/.github/workflows/python-linters.yml +++ b/.github/workflows/python-linters.yml @@ -36,6 +36,7 @@ jobs: run: | python -m pip install --upgrade pip pip --cache-dir ~/pip-cache install pylint + pip --cache-dir ~/pip-cache install ldap3 - name: Run Pylint env: PYTHON_FILES: ${{ needs.python-files.outputs.filelist }} diff --git a/comanage_utils.py b/comanage_utils.py new file mode 100644 index 0000000..bf08385 --- /dev/null +++ b/comanage_utils.py @@ -0,0 +1,240 @@ +#!/usr/bin/env python3 + +import os +import re +import sys +import json +import urllib.error +import urllib.request +from ldap3 import Server, Connection, ALL, ALL_ATTRIBUTES, SAFE_SYNC + +#PRODUCTION VALUES + +PRODUCTION_ENDPOINT = "https://registry.cilogon.org/registry/" +PRODUCTION_LDAP_SERVER = "ldaps://ldap.cilogon.org" +PRODUCTION_LDAP_USER = "uid=readonly_user,ou=system,o=OSG,o=CO,dc=cilogon,dc=org" +PRODUCTION_OSG_CO_ID = 7 +PRODUCTION_UNIX_CLUSTER_ID = 1 +PRODUCTION_LDAP_TARGET_ID = 6 + +#TEST VALUES + +TEST_ENDPOINT = "https://registry-test.cilogon.org/registry/" +TEST_LDAP_SERVER = "ldaps://ldap-test.cilogon.org" +TEST_LDAP_USER ="uid=registry_user,ou=system,o=OSG,o=CO,dc=cilogon,dc=org" +TEST_OSG_CO_ID = 8 +TEST_UNIX_CLUSTER_ID = 10 +TEST_LDAP_TARGET_ID = 9 + + +MIN_TIMEOUT = 5 +MAX_TIMEOUT = 625 +TIMEOUTMULTIPLE = 5 + + +GET = "GET" +PUT = "PUT" +POST = "POST" +DELETE = "DELETE" + + +def getpw(user, passfd, passfile): + if ":" in user: + user, pw = user.split(":", 1) + elif passfd is not None: + pw = os.fdopen(passfd).readline().rstrip("\n") + elif passfile is not None: + pw = open(passfile).readline().rstrip("\n") + elif "PASS" in os.environ: + pw = os.environ["PASS"] + else: + raise PermissionError + #when script needs to say PASS required, raise a permission error + #usage("PASS required") + return user, pw + + +def mkauthstr(user, passwd): + from base64 import encodebytes + raw_authstr = "%s:%s" % (user, passwd) + return encodebytes(raw_authstr.encode()).decode().replace("\n", "") + + +def mkrequest(method, target, data, endpoint, authstr, **kw): + url = os.path.join(endpoint, target) + if kw: + url += "?" + "&".join("{}={}".format(k,v) for k,v in kw.items()) + req = urllib.request.Request(url, json.dumps(data).encode("utf-8")) + req.add_header("Authorization", "Basic %s" % authstr) + req.add_header("Content-Type", "application/json") + req.get_method = lambda: method + return req + + +def call_api(target, endpoint, authstr, **kw): + return call_api2(GET, target, endpoint, authstr, **kw) + + +def call_api2(method, target, endpoint, authstr, **kw): + return call_api3(method, target, data=None, endpoint=endpoint, authstr=authstr, **kw) + + +def call_api3(method, target, data, endpoint, authstr, **kw): + req = mkrequest(method, target, data, endpoint, authstr, **kw) + trying = True + currentTimeout = MIN_TIMEOUT + while trying: + try: + resp = urllib.request.urlopen(req, timeout=currentTimeout) + payload = resp.read() + trying = False + except urllib.error.URLError as exception: + if currentTimeout < MAX_TIMEOUT: + currentTimeout *= TIMEOUTMULTIPLE + else: + sys.exit( + f"Exception raised after maximum number of retries and/or timeout {MAX_TIMEOUT} seconds reached. " + + f"Exception reason: {exception.reason}.\n Request: {req.full_url}" + ) + + return json.loads(payload) if payload else None + + +def get_osg_co_groups(osg_co_id, endpoint, authstr): + return call_api("co_groups.json", endpoint, authstr, coid=osg_co_id) + + +def get_co_group_identifiers(gid, endpoint, authstr): + return call_api("identifiers.json", endpoint, authstr, cogroupid=gid) + + +def get_co_group_members(gid, endpoint, authstr): + return call_api("co_group_members.json", endpoint, authstr, cogroupid=gid) + + +def get_co_person_identifiers(pid, endpoint, authstr): + return call_api("identifiers.json", endpoint, authstr, copersonid=pid) + + +def get_co_group(gid, endpoint, authstr): + resp_data = call_api("co_groups/%s.json" % gid, endpoint, authstr) + grouplist = get_datalist(resp_data, "CoGroups") + if not grouplist: + raise RuntimeError("No such CO Group Id: %s" % gid) + return grouplist[0] + + +def get_identifier(id_, endpoint, authstr): + resp_data = call_api("identifiers/%s.json" % id_, endpoint, authstr) + idfs = get_datalist(resp_data, "Identifiers") + if not idfs: + raise RuntimeError("No such Identifier Id: %s" % id_) + return idfs[0] + + +def get_unix_cluster_groups(ucid, endpoint, authstr): + return call_api("unix_cluster/unix_cluster_groups.json", endpoint, authstr, unix_cluster_id=ucid) + + +def get_unix_cluster_groups_ids(ucid, endpoint, authstr): + unix_cluster_groups = get_unix_cluster_groups(ucid, endpoint, authstr) + return set(group["CoGroupId"] for group in unix_cluster_groups["UnixClusterGroups"]) + + +def delete_identifier(id_, endpoint, authstr): + return call_api2(DELETE, "identifiers/%s.json" % id_, endpoint, authstr) + + +def get_datalist(data, listname): + return data[listname] if data else [] + + +def get_ldap_groups(ldap_server, ldap_user, ldap_authtok): + ldap_group_osggids = set() + server = Server(ldap_server, get_info=ALL) + connection = Connection(server, ldap_user, ldap_authtok, client_strategy=SAFE_SYNC, auto_bind=True) + _, _, response, _ = connection.search("ou=groups,o=OSG,o=CO,dc=cilogon,dc=org", "(cn=*)", attributes=ALL_ATTRIBUTES) + for group in response: + ldap_group_osggids.add(group["attributes"]["gidNumber"]) + return ldap_group_osggids + + +def identifier_from_list(id_list, id_type): + id_type_list = [id["Type"] for id in id_list] + try: + id_index = id_type_list.index(id_type) + return id_list[id_index]["Identifier"] + except ValueError: + return None + + +def identifier_matches(id_list, id_type, regex_string): + pattern = re.compile(regex_string) + value = identifier_from_list(id_list, id_type) + return (value is not None) and (pattern.match(value) is not None) + + +def rename_co_group(gid, group, newname, endpoint, authstr): + # minimal edit CoGroup Request includes Name+CoId+Status+Version + new_group_info = { + "Name" : newname, + "CoId" : group["CoId"], + "Status" : group["Status"], + "Version" : group["Version"] + } + data = { + "CoGroups" : [new_group_info], + "RequestType" : "CoGroups", + "Version" : "1.0" + } + return call_api3(PUT, "co_groups/%s.json" % gid, data, endpoint, authstr) + + +def add_identifier_to_group(gid, type, identifier_value, endpoint, authstr): + new_identifier_info = { + "Version": "1.0", + "Type": type, + "Identifier": identifier_value, + "Login": False, + "Person": {"Type": "Group", "Id": str(gid)}, + "Status": "Active", + } + data = { + "RequestType": "Identifiers", + "Version": "1.0", + "Identifiers": [new_identifier_info], + } + return call_api3(POST, "identifiers.json", data, endpoint, authstr) + + +def add_unix_cluster_group(gid, ucid, endpoint, authstr): + data = { + "RequestType": "UnixClusterGroups", + "Version": "1.0", + "UnixClusterGroups": [{"Version": "1.0", "UnixClusterId": ucid, "CoGroupId": gid}], + } + return call_api3(POST, "unix_cluster/unix_cluster_groups.json", data, endpoint, authstr) + + +def provision_group(gid, provision_target, endpoint, authstr): + path = f"co_provisioning_targets/provision/{provision_target}/cogroupid:{gid}.json" + data = { + "RequestType" : "CoGroupProvisioning", + "Version" : "1.0", + "Synchronous" : True + } + return call_api3(POST, path, data, endpoint, authstr) + +def provision_group_members(gid, prov_id, endpoint, authstr): + data = { + "RequestType" : "CoPersonProvisioning", + "Version" : "1.0", + "Synchronous" : True + } + responses = {} + for member in get_co_group_members(gid, endpoint, authstr)["CoGroupMembers"]: + if member["Person"]["Type"] == "CO": + pid = member["Person"]["Id"] + path = f"co_provisioning_targets/provision/{prov_id}/copersonid:{pid}.json" + responses[pid] = call_api3(POST, path, data, endpoint, authstr) + return responses diff --git a/create_project.py b/create_project.py index fbc28c5..cc447a1 100755 --- a/create_project.py +++ b/create_project.py @@ -7,6 +7,7 @@ import getopt import urllib.error import urllib.request +import comanage_utils as utils SCRIPT = os.path.basename(__file__) @@ -14,11 +15,6 @@ USER = "co_7.group_fixup" OSG_CO_ID = 7 -GET = "GET" -PUT = "PUT" -POST = "POST" -DELETE = "DELETE" - _usage = f"""\ usage: [PASS=...] {SCRIPT} [OPTIONS] COGroupNameOrId ProjectName @@ -63,127 +59,17 @@ class Options: options = Options() -def getpw(user, passfd, passfile): - if ':' in user: - user, pw = user.split(':', 1) - elif passfd is not None: - pw = os.fdopen(passfd).readline().rstrip('\n') - elif passfile is not None: - pw = open(passfile).readline().rstrip('\n') - elif 'PASS' in os.environ: - pw = os.environ['PASS'] - else: - usage("PASS required") - return user, pw - - -def mkauthstr(user, passwd): - from base64 import encodebytes - raw_authstr = '%s:%s' % (user, passwd) - return encodebytes(raw_authstr.encode()).decode().replace('\n', '') - - -def mkrequest(target, **kw): - return mkrequest2(GET, target, **kw) - - -def mkrequest2(method, target, **kw): - return mkrequest3(method, target, data=None, **kw) - - -def mkrequest3(method, target, data, **kw): - url = os.path.join(options.endpoint, target) - if kw: - url += "?" + "&".join( "{}={}".format(k,v) for k,v in kw.items() ) - req = urllib.request.Request(url, json.dumps(data).encode("utf-8")) - req.add_header("Authorization", "Basic %s" % options.authstr) - req.add_header("Content-Type", "application/json") - req.get_method = lambda: method - return req - - -def call_api(target, **kw): - return call_api2(GET, target, **kw) - - -def call_api2(method, target, **kw): - return call_api3(method, target, data=None, **kw) - - -def call_api3(method, target, data, **kw): - req = mkrequest3(method, target, data, **kw) - resp = urllib.request.urlopen(req) - payload = resp.read() - return json.loads(payload) if payload else None - - -# primary api calls - - -def get_osg_co_groups(): - return call_api("co_groups.json", coid=options.osg_co_id) - - -def get_co_group_identifiers(gid): - return call_api("identifiers.json", cogroupid=gid) - - -def get_co_group_members(gid): - return call_api("co_group_members.json", cogroupid=gid) - - -def get_co_person_identifiers(pid): - return call_api("identifiers.json", copersonid=pid) - - -def get_co_group(gid): - resp_data = call_api("co_groups/%s.json" % gid) - grouplist = get_datalist(resp_data, "CoGroups") - if not grouplist: - raise RuntimeError("No such CO Group Id: %s" % gid) - return grouplist[0] - - -def get_identifier(id_): - resp_data = call_api("identifiers/%s.json" % id_) - idfs = get_datalist(resp_data, "Identifiers") - if not idfs: - raise RuntimeError("No such Identifier Id: %s" % id_) - return idfs[0] - - -def get_datalist(data, listname): - return data[listname] if data else [] - - # script-specific functions def add_project_identifier_to_group(gid, project_name): identifier_name = "Yes-%s" % project_name type_ = "ospoolproject" - return add_identifier_to_group(gid, type_, identifier_name) - - -def add_identifier_to_group(gid, type_, identifier_name): - new_identifier_info = { - "Version" : "1.0", - "Type" : type_, - "Identifier" : identifier_name, - "Login" : False, - "Person" : {"Type": "Group", "Id": str(gid)}, - "Status" : "Active" - } - data = { - "RequestType" : "Identifiers", - "Version" : "1.0", - "Identifiers" : [new_identifier_info] - } - return call_api3(POST, "identifiers.json", data) + return utils.add_identifier_to_group(gid, type_, identifier_name, options.endpoint, options.authstr) def gname_to_gid(gname): - resp_data = get_osg_co_groups() - groups = get_datalist(resp_data, "CoGroups") + resp_data = utils.get_osg_co_groups(options.osg_co_id, options.endpoint, options.authstr) + groups = utils.get_datalist(resp_data, "CoGroups") matching = [ g for g in groups if g["Name"] == gname ] if len(matching) > 1: @@ -225,8 +111,11 @@ def parse_options(args): if op == '-f': passfile = arg if op == '-e': options.endpoint = arg - user, passwd = getpw(options.user, passfd, passfile) - options.authstr = mkauthstr(user, passwd) + try: + user, passwd = utils.getpw(options.user, passfd, passfile) + options.authstr = utils.mkauthstr(user, passwd) + except PermissionError: + usage("PASS required") def main(args): @@ -235,7 +124,7 @@ def main(args): if options.gname: options.gid = gname_to_gid(options.gname) else: - options.gname = get_co_group(options.gid)["Name"] + options.gname = utils.get_co_group(options.gid, options.endpoint, options.authstr)["Name"] print('Creating new Identifier for project "%s"\n' 'for CO Group "%s" (%s)' @@ -247,7 +136,7 @@ def main(args): print("Server Response:") print(json.dumps(resp, indent=2, sort_keys=True)) - new_identifier = get_identifier(resp["Id"]) + new_identifier = utils.get_identifier(resp["Id"], options.endpoint, options.authstr) print("") print("New Identifier Object:") print(json.dumps(new_identifier, indent=2, sort_keys=True)) diff --git a/group_fixup.py b/group_fixup.py index e017568..34d3839 100755 --- a/group_fixup.py +++ b/group_fixup.py @@ -3,11 +3,11 @@ import os import re import sys -import json import getopt import collections import urllib.error import urllib.request +import comanage_utils as utils SCRIPT = os.path.basename(__file__) @@ -16,11 +16,6 @@ OSG_CO_ID = 7 LDAP_PROV_ID = 6 -GET = "GET" -PUT = "PUT" -POST = "POST" -DELETE = "DELETE" - _usage = f"""\ usage: [PASS=...] {SCRIPT} [OPTIONS] @@ -74,102 +69,17 @@ class Options: options = Options() -def getpw(user, passfd, passfile): - if ':' in user: - user, pw = user.split(':', 1) - elif passfd is not None: - pw = os.fdopen(passfd).readline().rstrip('\n') - elif passfile is not None: - pw = open(passfile).readline().rstrip('\n') - elif 'PASS' in os.environ: - pw = os.environ['PASS'] - else: - usage("PASS required") - return user, pw - - -def mkauthstr(user, passwd): - from base64 import encodebytes - raw_authstr = '%s:%s' % (user, passwd) - return encodebytes(raw_authstr.encode()).decode().replace('\n', '') - - -def mkrequest(target, **kw): - return mkrequest2(GET, target, **kw) - - -def mkrequest2(method, target, **kw): - return mkrequest3(method, target, data=None, **kw) - - -def mkrequest3(method, target, data, **kw): - url = os.path.join(options.endpoint, target) - if kw: - url += "?" + "&".join( "{}={}".format(k,v) for k,v in kw.items() ) - req = urllib.request.Request(url, json.dumps(data).encode("utf-8")) - req.add_header("Authorization", "Basic %s" % options.authstr) - req.add_header("Content-Type", "application/json") - req.get_method = lambda: method - return req - - -def call_api(target, **kw): - return call_api2(GET, target, **kw) - - -def call_api2(method, target, **kw): - return call_api3(method, target, data=None, **kw) - - -def call_api3(method, target, data, **kw): - req = mkrequest3(method, target, data, **kw) - resp = urllib.request.urlopen(req) - payload = resp.read() - return json.loads(payload) if payload else None - - -# primary api calls - - -def get_osg_co_groups(): - return call_api("co_groups.json", coid=options.osg_co_id) - - -def get_co_group_identifiers(gid): - return call_api("identifiers.json", cogroupid=gid) - - -def get_co_group_members(gid): - return call_api("co_group_members.json", cogroupid=gid) - - -def get_co_person_identifiers(pid): - return call_api("identifiers.json", copersonid=pid) - - -def get_co_group(gid): - resp_data = call_api("co_groups/%s.json" % gid) - grouplist = get_datalist(resp_data, "CoGroups") - if not grouplist: - raise RuntimeError("No such CO Group Id: %s" % gid) - return grouplist[0] - - -def get_datalist(data, listname): - return data[listname] if data else [] - - # api call results massagers def get_unixcluster_autogroups(): - groups = get_osg_co_groups() + groups = utils.get_osg_co_groups(options.osg_co_id, options.endpoint, options.authstr) return [ g for g in groups["CoGroups"] if "automatically by UnixCluster" in g["Description"] ] def get_misnamed_unixcluster_groups(): - groups = get_osg_co_groups() + groups = utils.get_osg_co_groups(options.osg_co_id, options.endpoint, options.authstr) return [ g for g in groups["CoGroups"] if "UnixCluster Group" in g["Name"] ] @@ -220,7 +130,7 @@ def show_all_unixcluster_groups(): def show_one_unixcluster_group(gid): - group = get_co_group(gid) + group = utils.get_co_group(gid, options.endpoint, options.authstr) show_misnamed_unixcluster_group(group) @@ -231,8 +141,8 @@ def show_misnamed_unixcluster_groups(): def show_group_identifiers(gid): - resp_data = get_co_group_identifiers(gid) - identifiers = get_datalist(resp_data, "Identifiers") + resp_data = utils.get_co_group_identifiers(gid, options.endpoint, options.authstr) + identifiers = utils.get_datalist(resp_data, "Identifiers") for i in identifiers: print(' - Identifier {Id}: ({Type}) "{Identifier}"'.format(**i)) @@ -241,73 +151,25 @@ def show_group_identifiers(gid): print(' ** Identifier Ids to delete: %s' % ', '.join(ids_to_delete)) - # fixup functions -def delete_identifier(id_): - return call_api2(DELETE, "identifiers/%s.json" % id_) - - -def rename_co_group(gid, group, newname): - # minimal edit CoGroup Request includes Name+CoId+Status+Version - new_group_info = { - "Name" : newname, - "CoId" : group["CoId"], - "Status" : group["Status"], - "Version" : group["Version"] - } - data = { - "CoGroups" : [new_group_info], - "RequestType" : "CoGroups", - "Version" : "1.0" - } - return call_api3(PUT, "co_groups/%s.json" % gid, data) - - -def provision_group(gid): - prov_id = options.prov_id - path = f"co_provisioning_targets/provision/{prov_id}/cogroupid:{gid}.json" - data = { - "RequestType" : "CoGroupProvisioning", - "Version" : "1.0", - "Synchronous" : True - } - return call_api3(POST, path, data) - - -def provision_group_members(gid): - prov_id = options.prov_id - data = { - "RequestType" : "CoPersonProvisioning", - "Version" : "1.0", - "Synchronous" : True - } - responses = {} - for member in get_co_group_members(gid)["CoGroupMembers"]: - if member["Person"]["Type"] == "CO": - pid = member["Person"]["Id"] - path = f"co_provisioning_targets/provision/{prov_id}/copersonid:{pid}.json" - responses[pid] = call_api3(POST, path, data) - return responses - - def fixup_unixcluster_group(gid): - group = get_co_group(gid) + group = utils.get_co_group(gid, options.endpoint, options.authstr) oldname = group["Name"] newname = get_fixed_unixcluster_group_name(oldname) - resp_data = get_co_group_identifiers(gid) - identifiers = get_datalist(resp_data, "Identifiers") + resp_data = utils.get_co_group_identifiers(gid, options.endpoint, options.authstr) + identifiers = utils.get_datalist(resp_data, "Identifiers") ids_to_delete = get_identifiers_to_delete(identifiers) show_misnamed_unixcluster_group(group) if oldname != newname: - rename_co_group(gid, group, newname) + utils.rename_co_group(gid, group, newname, options.endpoint, options.authstr) for id_ in ids_to_delete: - delete_identifier(id_) + utils.delete_identifier(id_, options.endpoint, options.authstr) - provision_group(gid) - provision_group_members(gid) + utils.provision_group(gid, options.prov_id, options.endpoint, options.authstr) + utils.provision_group_members(gid, options.prov_id, options.endpoint, options.authstr) # http errors raise exceptions, so at this point we apparently succeeded print(":thumbsup:") @@ -349,8 +211,11 @@ def parse_options(args): if op == '--fix-all': options.fix_all = True - user, passwd = getpw(options.user, passfd, passfile) - options.authstr = mkauthstr(user, passwd) + try: + user, passwd = utils.getpw(options.user, passfd, passfile) + options.authstr = utils.mkauthstr(user, passwd) + except PermissionError: + usage("PASS required") def main(args): @@ -376,4 +241,3 @@ def main(args): except (RuntimeError, urllib.error.HTTPError) as e: print(e, file=sys.stderr) sys.exit(1) - diff --git a/group_identifier_assigner.py b/group_identifier_assigner.py deleted file mode 100755 index 7eae705..0000000 --- a/group_identifier_assigner.py +++ /dev/null @@ -1,267 +0,0 @@ -#!/usr/bin/env python3 - -import os -import re -import sys -import json -import getopt -import urllib.error -import urllib.request - -SCRIPT = os.path.basename(__file__) -ENDPOINT = "https://registry-test.cilogon.org/registry/" -OSG_CO_ID = 8 -MINTIMEOUT = 5 -MAXTIMEOUT = 625 -TIMEOUTMULTIPLE = 5 - -GET = "GET" -PUT = "PUT" -POST = "POST" -DELETE = "DELETE" - -OSPOOL_PROJECT_PREFIX_STR = "Yes-" -PROJECT_GIDS_START = 200000 - -_usage = f"""\ -usage: [PASS=...] {SCRIPT} [OPTIONS] - -OPTIONS: - -u USER[:PASS] specify USER and optionally PASS on command line - -c OSG_CO_ID specify OSG CO ID (default = {OSG_CO_ID}) - -d passfd specify open fd to read PASS - -f passfile specify path to file to open and read PASS - -e ENDPOINT specify REST endpoint - (default = {ENDPOINT}) - -o outfile specify output file (default: write to stdout) - -t minTimeout set minimum timeout, in seconds, for API call (default to {MINTIMEOUT}) - -T maxTimeout set maximum timeout, in seconds, for API call (default to {MAXTIMEOUT}) - -h display this help text - -PASS for USER is taken from the first of: - 1. -u USER:PASS - 2. -d passfd (read from fd) - 3. -f passfile (read from file) - 4. read from $PASS env var -""" - - -def usage(msg=None): - if msg: - print(msg + "\n", file=sys.stderr) - - print(_usage, file=sys.stderr) - sys.exit() - - -class Options: - endpoint = ENDPOINT - user = "co_8.william_test" - osg_co_id = OSG_CO_ID - outfile = None - authstr = None - min_timeout = MINTIMEOUT - max_timeout = MAXTIMEOUT - project_gid_startval = PROJECT_GIDS_START - - -options = Options() - - -def getpw(user, passfd, passfile): - if ":" in user: - user, pw = user.split(":", 1) - elif passfd is not None: - pw = os.fdopen(passfd).readline().rstrip("\n") - elif passfile is not None: - pw = open(passfile).readline().rstrip("\n") - elif "PASS" in os.environ: - pw = os.environ["PASS"] - else: - usage("PASS required") - return user, pw - - -def mkauthstr(user, passwd): - from base64 import encodebytes - - raw_authstr = "%s:%s" % (user, passwd) - return encodebytes(raw_authstr.encode()).decode().replace("\n", "") - - -def mkrequest(method, target, data, **kw): - url = os.path.join(options.endpoint, target) - if kw: - url += "?" + "&".join("{}={}".format(k, v) for k, v in kw.items()) - req = urllib.request.Request(url, json.dumps(data).encode("utf-8")) - req.add_header("Authorization", "Basic %s" % options.authstr) - req.add_header("Content-Type", "application/json") - req.get_method = lambda: method - return req - - -def call_api(target, **kw): - return call_api2(GET, target, **kw) - - -def call_api2(method, target, **kw): - return call_api3(method, target, data=None, **kw) - - -def call_api3(method, target, data, **kw): - req = mkrequest(method, target, data, **kw) - trying = True - currentTimeout = options.min_timeout - while trying: - try: - resp = urllib.request.urlopen(req, timeout=currentTimeout) - payload = resp.read() - trying = False - except urllib.error.URLError as exception: - if currentTimeout < options.max_timeout: - currentTimeout *= TIMEOUTMULTIPLE - else: - sys.exit( - f"Exception raised after maximum timeout {options.max_timeout} seconds reached. " - + f"Exception reason: {exception.reason}.\n Request: {req.full_url}" - ) - - return json.loads(payload) if payload else None - - -def get_osg_co_groups(): - return call_api("co_groups.json", coid=options.osg_co_id) - - -# primary api calls - - -def get_co_group_identifiers(gid): - return call_api("identifiers.json", cogroupid=gid) - - -def get_co_group_members(gid): - return call_api("co_group_members.json", cogroupid=gid) - - -def get_co_person_identifiers(pid): - return call_api("identifiers.json", copersonid=pid) - - -def get_datalist(data, listname): - return data[listname] if data else [] - - -def identifier_index(id_list, id_type): - id_type_list = [id["Type"] for id in id_list] - try: - return id_type_list.index(id_type) - except ValueError: - return -1 - - -def identifier_matches(id_list, id_type, regex_string): - pattern = re.compile(regex_string) - index = identifier_index(id_list, id_type) - return (index != -1) & (pattern.match(id_list[index]["Identifier"]) is not None) - - -def add_identifier_to_group(gid, type, identifier_name): - new_identifier_info = { - "Version": "1.0", - "Type": type, - "Identifier": identifier_name, - "Login": False, - "Person": {"Type": "Group", "Id": str(gid)}, - "Status": "Active", - } - data = { - "RequestType": "Identifiers", - "Version": "1.0", - "Identifiers": [new_identifier_info], - } - return call_api3(POST, "identifiers.json", data) - - -def parse_options(args): - try: - ops, args = getopt.getopt(args, "u:c:d:f:e:o:t:T:h") - except getopt.GetoptError: - usage() - - if args: - usage("Extra arguments: %s" % repr(args)) - - passfd = None - passfile = None - - for op, arg in ops: - if op == "-h": - usage() - if op == "-u": - options.user = arg - if op == "-c": - options.osg_co_id = int(arg) - if op == "-d": - passfd = int(arg) - if op == "-f": - passfile = arg - if op == "-e": - options.endpoint = arg - if op == "-o": - options.outfile = arg - if op == "-t": - options.min_timeout = float(arg) - if op == "-T": - options.max_timeout = float(arg) - - user, passwd = getpw(options.user, passfd, passfile) - options.authstr = mkauthstr(user, passwd) - - -def main(args): - parse_options(args) - - # get groups with 'OSPool project name' matching "Yes-*" that don't have a 'OSG GID' - - co_groups = get_osg_co_groups()["CoGroups"] - highest_osggid = 0 - projects_to_assign_identifiers = [] - - for group in co_groups: - gid = group["Id"] - identifier_data = get_co_group_identifiers(gid) - - if identifier_data: - identifier_list = identifier_data["Identifiers"] - - project_id_index = identifier_index(identifier_list, "ospoolproject") - if project_id_index != -1: - project_id = str(identifier_list[project_id_index]["Identifier"]) - is_project = re.compile(OSPOOL_PROJECT_PREFIX_STR + "*").match(project_id) is not None - else: - is_project = False - - osggid_index = identifier_index(identifier_list, "osggid") - if osggid_index != -1: - highest_osggid = max(highest_osggid, int(identifier_list[osggid_index]["Identifier"])) - elif is_project is True: - project_name = project_id.replace(OSPOOL_PROJECT_PREFIX_STR, "", 1).lower() - projects_to_assign_identifiers.append((gid, project_name,)) - - for gid, project_name in projects_to_assign_identifiers: - # for each, set a 'OSG GID' starting from 200000 and a 'OSG Group Name' that is the group name - osggid_to_assign = max(highest_osggid + 1, options.project_gid_startval) - highest_osggid = osggid_to_assign - add_identifier_to_group(gid, type="osggid", identifier_name=osggid_to_assign) - add_identifier_to_group(gid, type="osggroup", identifier_name=project_name) - print(f"project {project_name}: added osggid {osggid_to_assign} and osg project name {project_name}") - - -if __name__ == "__main__": - try: - main(sys.argv[1:]) - except urllib.error.HTTPError as e: - print(e, file=sys.stderr) - sys.exit(1) - diff --git a/osg-comanage-project-usermap.py b/osg-comanage-project-usermap.py index 4c96979..f065cd7 100755 --- a/osg-comanage-project-usermap.py +++ b/osg-comanage-project-usermap.py @@ -2,19 +2,14 @@ import os import sys -import json import getopt import collections -import urllib.error -import urllib.request +import comanage_utils as utils SCRIPT = os.path.basename(__file__) -ENDPOINT = "https://registry-test.cilogon.org/registry/" -OSG_CO_ID = 8 -MINTIMEOUT = 5 -MAXTIMEOUT = 625 -TIMEOUTMULTIPLE = 5 +ENDPOINT = "https://registry.cilogon.org/registry/" +OSG_CO_ID = 7 _usage = f"""\ @@ -29,8 +24,6 @@ (default = {ENDPOINT}) -o outfile specify output file (default: write to stdout) -g filter_group filter users by group name (eg, 'ap1-login') - -t minTimeout set minimum timeout, in seconds, for API call (default to {MINTIMEOUT}) - -T maxTimeout set maximum timeout, in seconds, for API call (default to {MAXTIMEOUT}) -h display this help text PASS for USER is taken from the first of: @@ -50,124 +43,51 @@ def usage(msg=None): class Options: endpoint = ENDPOINT - user = "co_8.project_script" + user = "co_7.project_script" osg_co_id = OSG_CO_ID outfile = None authstr = None filtergrp = None - min_timeout = MINTIMEOUT - max_timeout = MAXTIMEOUT options = Options() -def getpw(user, passfd, passfile): - if ':' in user: - user, pw = user.split(':', 1) - elif passfd is not None: - pw = os.fdopen(passfd).readline().rstrip('\n') - elif passfile is not None: - pw = open(passfile).readline().rstrip('\n') - elif 'PASS' in os.environ: - pw = os.environ['PASS'] - else: - usage("PASS required") - return user, pw - - -def mkauthstr(user, passwd): - from base64 import encodebytes - raw_authstr = '%s:%s' % (user, passwd) - return encodebytes(raw_authstr.encode()).decode().replace('\n', '') - - -def mkrequest(target, **kw): - url = os.path.join(options.endpoint, target) - if kw: - url += "?" + "&".join( "{}={}".format(k,v) for k,v in kw.items() ) - req = urllib.request.Request(url) - req.add_header("Authorization", "Basic %s" % options.authstr) - req.get_method = lambda: 'GET' - return req - - -def call_api(target, **kw): - req = mkrequest(target, **kw) - trying = True - currentTimeout = options.min_timeout - while trying: - try: - resp = urllib.request.urlopen(req, timeout=currentTimeout) - payload = resp.read() - trying = False - except urllib.error.URLError as exception: - if (currentTimeout < options.max_timeout): - currentTimeout *= TIMEOUTMULTIPLE - else: - sys.exit(f"Exception raised after maximum timeout {options.max_timeout} seconds reached. " - + f"Exception reason: {exception.reason}.\n Request: {req.full_url}") - - return json.loads(payload) if payload else None - - -def get_osg_co_groups(): - return call_api("co_groups.json", coid=options.osg_co_id) - - -# primary api calls - -def get_co_group_identifiers(gid): - return call_api("identifiers.json", cogroupid=gid) - - -def get_co_group_members(gid): - return call_api("co_group_members.json", cogroupid=gid) - - -def get_co_person_identifiers(pid): - return call_api("identifiers.json", copersonid=pid) - - -def get_datalist(data, listname): - return data[listname] if data else [] - - # api call results massagers def get_osg_co_groups__map(): #print("get_osg_co_groups__map()") - resp_data = get_osg_co_groups() - data = get_datalist(resp_data, "CoGroups") + resp_data = utils.get_osg_co_groups(options.osg_co_id, options.endpoint, options.authstr) + data = utils.get_datalist(resp_data, "CoGroups") return { g["Id"]: g["Name"] for g in data } def co_group_is_ospool(gid): #print(f"co_group_is_ospool({gid})") - resp_data = get_co_group_identifiers(gid) - data = get_datalist(resp_data, "Identifiers") + resp_data = utils.get_co_group_identifiers(gid, options.endpoint, options.authstr) + data = utils.get_datalist(resp_data, "Identifiers") return any( i["Type"] == "ospoolproject" for i in data ) def get_co_group_members__pids(gid): #print(f"get_co_group_members__pids({gid})") - resp_data = get_co_group_members(gid) - data = get_datalist(resp_data, "CoGroupMembers") + resp_data = utils.get_co_group_members(gid, options.endpoint, options.authstr) + data = utils.get_datalist(resp_data, "CoGroupMembers") # For INF-1060: Temporary Fix until "The Great Project Provisioning" is finished return [ m["Person"]["Id"] for m in data if m["Member"] == True] def get_co_person_osguser(pid): #print(f"get_co_person_osguser({pid})") - resp_data = get_co_person_identifiers(pid) - data = get_datalist(resp_data, "Identifiers") + resp_data = utils.get_co_person_identifiers(pid, options.endpoint, options.authstr) + data = utils.get_datalist(resp_data, "Identifiers") typemap = { i["Type"]: i["Identifier"] for i in data } return typemap.get("osguser") def parse_options(args): try: - ops, args = getopt.getopt(args, 'u:c:d:f:g:e:o:t:T:h') + ops, args = getopt.getopt(args, 'u:c:d:f:g:e:o:h') except getopt.GetoptError: usage() @@ -186,11 +106,12 @@ def parse_options(args): if op == '-e': options.endpoint = arg if op == '-o': options.outfile = arg if op == '-g': options.filtergrp = arg - if op == '-t': options.min_timeout = float(arg) - if op == '-T': options.max_timeout = float(arg) - user, passwd = getpw(options.user, passfd, passfile) - options.authstr = mkauthstr(user, passwd) + try: + user, passwd = utils.getpw(options.user, passfd, passfile) + options.authstr = utils.mkauthstr(user, passwd) + except PermissionError: + usage("PASS required") def gid_pids_to_osguser_pid_gids(gid_pids, pid_osguser): @@ -248,7 +169,5 @@ def main(args): if __name__ == "__main__": try: main(sys.argv[1:]) - except urllib.error.HTTPError as e: - print(e, file=sys.stderr) - sys.exit(1) - + except Exception as e: + sys.exit(e) diff --git a/project_group_setup.py b/project_group_setup.py new file mode 100644 index 0000000..a263e04 --- /dev/null +++ b/project_group_setup.py @@ -0,0 +1,279 @@ +#!/usr/bin/env python3 + +import os +import sys +import getopt +import comanage_utils as utils + +SCRIPT = os.path.basename(__file__) +ENDPOINT = "https://registry.cilogon.org/registry/" +LDAP_SERVER = "ldaps://ldap.cilogon.org" +LDAP_USER = "uid=readonly_user,ou=system,o=OSG,o=CO,dc=cilogon,dc=org" +OSG_CO_ID = 7 +UNIX_CLUSTER_ID = 1 +LDAP_TARGET_ID = 6 + +OSPOOL_PROJECT_PREFIX_STR = "Yes-" +PROJECT_GIDS_START = 200000 + + +_usage = f"""\ +usage: [PASS=...] {SCRIPT} [OPTIONS] + +OPTIONS: + -u USER[:PASS] specify USER and optionally PASS on command line + -c OSG_CO_ID specify OSG CO ID (default = {OSG_CO_ID}) + -g CLUSTER_ID specify UNIX Cluster ID (default = {UNIX_CLUSTER_ID}) + -l LDAP_TARGET specify LDAP Provsion ID (defult = {LDAP_TARGET_ID}) + -s LDAP_SERVER specify LDAP server + -y LDAP_USER specify LDAP server user + -p LDAP authtok specify LDAP server authtok + -d passfd specify open fd to read PASS + -f passfile specify path to file to open and read PASS + -e ENDPOINT specify REST endpoint + (default = {ENDPOINT}) + -o outfile specify output file (default: write to stdout) + -h display this help text + +PASS for USER is taken from the first of: + 1. -u USER:PASS + 2. -d passfd (read from fd) + 3. -f passfile (read from file) + 4. read from $PASS env var +""" + + +def usage(msg=None): + if msg: + print(f"{msg}\n", file=sys.stderr) + + print(_usage, file=sys.stderr) + sys.exit() + + +class Options: + endpoint = ENDPOINT + user = "co_7.project_script" + osg_co_id = OSG_CO_ID + ucid = UNIX_CLUSTER_ID + provision_target = LDAP_TARGET_ID + ldap_user = LDAP_USER + ldap_server = LDAP_SERVER + outfile = None + authstr = None + ldap_authtok = None + project_gid_startval = PROJECT_GIDS_START + + +options = Options() + + +def parse_options(args): + try: + ops, args = getopt.getopt(args, "u:c:g:l:p:d:f:e:o:s:y:h") + except getopt.GetoptError: + usage() + + if args: + usage("Extra arguments: %s" % repr(args)) + + passfd = None + passfile = None + + for op, arg in ops: + if op == "-h": + usage() + if op == "-u": + options.user = arg + if op == "-c": + options.osg_co_id = int(arg) + if op == "-g": + options.ucid = int(arg) + if op == "-l": + options.provision_target = int(arg) + if op == "-p": + options.ldap_authtok = arg + if op == "-d": + passfd = int(arg) + if op == "-f": + passfile = arg + if op == "-e": + options.endpoint = arg + if op == "-o": + options.outfile = arg + if op == "-s": + options.ldap_server = arg + if op == "-y": + options.ldap_user = arg + + try: + user, passwd = utils.getpw(options.user, passfd, passfile) + options.authstr = utils.mkauthstr(user, passwd) + except PermissionError: + usage("PASS required") + + +def append_if_project(project_groups, group): + """If this group has a ospoolproject id, and it starts with "Yes-", it's a project""" + if utils.identifier_matches(group["ID_List"], "ospoolproject", (OSPOOL_PROJECT_PREFIX_STR + "*")): + # Add a dict of the relavent data for this project to the project_groups list + project_groups.append(group) + + +def update_highest_osggid(highest_osggid, group): + # Get the value of the osggid identifier, if this group has one + osggid = utils.identifier_from_list(group["ID_List"], "osggid") + # If this group has a osggid, keep a hold of the highest one we've seen so far + try: + return max(highest_osggid, int(osggid)) + except TypeError: + return highest_osggid + + +def get_comanage_data(): + projects_list = [] + highest_osggid = 0 + + co_groups = utils.get_osg_co_groups(options.osg_co_id, options.endpoint, options.authstr)["CoGroups"] + for group_data in co_groups: + identifier_list = utils.get_co_group_identifiers(group_data["Id"], options.endpoint, options.authstr) + if identifier_list is not None: + # Store this groups data in a dictionary to avoid repeated API calls + group = {"Gid": group_data["Id"], "Name": group_data["Name"], "ID_List": identifier_list["Identifiers"]} + + # Add this group to the project list if it's a project, otherwise skip. + append_if_project(projects_list, group) + + # Update highest_osggid, if this group has an osggid and it's higher than the current highest osggid. + highest_osggid = update_highest_osggid(highest_osggid, group) + return (projects_list, highest_osggid) + + +def get_projects_needing_identifiers(project_groups): + projects_needing_identifiers = [] + for project in project_groups: + # If this project doesn't have an osggid already assigned to it... + if utils.identifier_from_list(project["ID_List"], "osggid") is None: + # Prep the project to have the proper identifiers added to it + projects_needing_identifiers.append(project) + return projects_needing_identifiers + + +def get_projects_needing_cluster_groups(project_groups): + # CO Groups associated with a UNIX Cluster Group + clustered_group_ids = utils.get_unix_cluster_groups_ids(options.ucid, options.endpoint, options.authstr) + try: + # All project Gids + project_gids = set(project["Gid"] for project in project_groups) + # Project Gids for projects without UNIX cluster groups + project_gids_lacking_cluster_groups = project_gids.difference(clustered_group_ids) + # All projects needing UNIX cluster groups + projects_needing_unix_groups = ( + project + for project in project_groups + if project["Gid"] in project_gids_lacking_cluster_groups + ) + return projects_needing_unix_groups + except TypeError: + print("ERROR: TypeError raised while trying to determine which projects need UNIX cluster groups\n" + +f"clustered group ids: {clustered_group_ids} and project_gids: {project_gids}") + return set() + + +def get_projects_needing_provisioning(project_groups): + # project groups provisioned in LDAP + ldap_group_osggids = utils.get_ldap_groups(options.ldap_server, options.ldap_user, options.ldap_authtok) + try: + # All project osggids + project_osggids = set( + int(utils.identifier_from_list(project["ID_List"], "osggid")) for project in project_groups + ) + # project osggids not provisioned in ldap + project_osggids_to_provision = project_osggids.difference(ldap_group_osggids) + # All projects that aren't provisioned in ldap + projects_to_provision = ( + project + for project in project_groups + if int(utils.identifier_from_list(project["ID_List"], "osggid")) in project_osggids_to_provision + ) + return projects_to_provision + except TypeError: + print("TypeError raised while trying to determine which projects need provisioning\n" + +f"ldap group osggids: {ldap_group_osggids} and project osggids: {project_osggids}") + return set() + + +def add_missing_group_identifier(project, id_type, value): + # If the group doesn't already have an id of this type ... + if utils.identifier_from_list(project["ID_List"], id_type) is None: + # ... add the identifier to the group + utils.add_identifier_to_group(project["Gid"], id_type, value, options.endpoint, options.authstr) + print(f'project {project["Gid"]}: added id {value} of type {id_type}') + + +def assign_identifiers_to_project(project, id_dict): + for k, v in id_dict.items(): + # Add an identifier of type k and value v to this group, if it doesn't have them already + add_missing_group_identifier(project, k, v) + # Update the project object to include the new identifiers + new_identifiers = utils.get_co_group_identifiers(project["Gid"], options.endpoint, options.authstr)["Identifiers"] + project["ID_List"] = new_identifiers + + +def assign_identifiers(project_list, highest_osggid): + highest = highest_osggid + for project in project_list: + # Project name identifier is the CO Group name in lower case + project_name = project["Name"].lower() + + # Determine what osggid to assign this project, + # based on the starting range and the highest osggid seen in existing groups + osggid_to_assign = max(highest + 1, options.project_gid_startval) + highest = osggid_to_assign + + identifiers_to_add = {"osggid": osggid_to_assign, "osggroup": project_name} + + assign_identifiers_to_project(project, identifiers_to_add) + + +def create_unix_cluster_groups(project_list): + for project in project_list: + utils.add_unix_cluster_group(project["Gid"], options.ucid, options.endpoint, options.authstr) + print(f'project group {project["Gid"]}: added UNIX Cluster Group') + + +def provision_groups(project_list): + for project in project_list: + utils.provision_group(project["Gid"], options.provision_target, options.endpoint, options.authstr) + print(f'project group {project["Gid"]}: Provisioned Group') + + +def main(args): + parse_options(args) + + # Make all of the necessary calls to COManage's API for the data we'll need to set up projects. + # Projects is a List of dicts with keys Gid, Name, and Identifiers, the project's list of identifiers. + # Highest_current_osggid is the highest OSGGID that's currently assigned to any CO Group. + projects, highest_current_osggid = get_comanage_data() + + # From all the project groups in COManage, find the ones that need OSGGIDs or OSG GroupNames, + # then assign them the identifiers that they're missing. + projects_needing_identifiers = get_projects_needing_identifiers(projects) + assign_identifiers(projects_needing_identifiers, highest_current_osggid) + + # From all the project groups in COManage, find the ones that don't have UNIX Cluster Groups, + # then create UNIX Cluster Groups for them. + projects_needing_cluster_groups = get_projects_needing_cluster_groups(projects) + create_unix_cluster_groups(projects_needing_cluster_groups) + + # From all the project groups in COManage, find the ones that aren't already provisioned in LDAP, + # then have COManage provision the project/UNIX Cluster Group in LDAP. + projects_needing_provisioning = get_projects_needing_provisioning(projects) + provision_groups(projects_needing_provisioning) + + +if __name__ == "__main__": + try: + main(sys.argv[1:]) + except Exception as e: + sys.exit(e)