diff --git a/pylmod/__init__.py b/pylmod/__init__.py index 51142f2..aa902bd 100644 --- a/pylmod/__init__.py +++ b/pylmod/__init__.py @@ -1,25 +1,25 @@ - """ -PyLmod is a module that implements MIT Learning Modules API in python +PyLmod is a module that implements MIT Learning Modules API in Python """ import os.path from pkg_resources import get_distribution, DistributionNotFound from pylmod.client import Client -from pylmod.stellargradebook import StellarGradeBook +from pylmod.gradebook import GradeBook +from pylmod.membership import Membership try: - _dist = get_distribution('pylmod') + DIST = get_distribution('pylmod') # Normalize case for Windows systems - dist_loc = os.path.normcase(_dist.location) - here = os.path.normcase(__file__) - if not here.startswith(os.path.join(dist_loc, 'pylmod')): + DIST_LOC = os.path.normcase(DIST.location) + HERE = os.path.normcase(__file__) + if not HERE.startswith(os.path.join(DIST_LOC, 'pylmod')): # not installed, but there is another version that *is* raise DistributionNotFound except DistributionNotFound: __version__ = 'Please install this project with setup.py' else: - __version__ = _dist.version + __version__ = DIST.version -__all__ = ['Client', 'StellarGradeBook', ] +__all__ = ['Client', 'GradeBook', 'Membership'] diff --git a/pylmod/base.py b/pylmod/base.py new file mode 100644 index 0000000..a5a8606 --- /dev/null +++ b/pylmod/base.py @@ -0,0 +1,129 @@ +""" + Python class representing interface to MIT Learning Modules web service. +""" + +import json +import logging +import requests + + +log = logging.getLogger(__name__) # pylint: disable=C0103 + + +class Base(object): + """ + The Base class provides the transport for accessing MIT LM web service. + + The Base class implements the functions that underlie the HTTP calls to + the LM web service. It shouldn't be instantiated directly as it is + inherited by the classes that implement the API. + + Attributes: + cert: The certificate used to authenticate access to LM web service + urlbase: The URL of the LM web service + """ + + GETS = {'academicterms': '', + 'academicterm': '/{termCode}', + 'gradebook': '?uuid={uuid}'} + + GBUUID = 'STELLAR:/project/mitxdemosite' + TIMEOUT = 200 # connection timeout, seconds + + verbose = True + gradebookid = None + + def __init__( + self, + cert, + urlbase='https://learning-modules.mit.edu:8443/service/gradebook', + ): + """ + Initialize Base instance. + + - urlbase: URL base for gradebook API + (still needs certs); default False + """ + # pem with private and public key application certificate for access + self.cert = cert + + self.urlbase = urlbase + self.ses = requests.Session() + self.ses.cert = cert + self.ses.timeout = self.TIMEOUT # connection timeout + self.ses.verify = True # verify site certificate + + log.debug("------------------------------------------------------") + log.info("[PyLmod] init base=%s", urlbase) + + def rest_action(self, func, url, **kwargs): + """Routine to do low-level REST operation, with retry""" + cnt = 1 + while cnt < 10: + cnt += 1 + try: + return self.rest_action_actual(func, url, **kwargs) + except requests.ConnectionError, err: + log.error( + "[StellarGradeBook] Error - connection error in " + "rest_action, err=%s", err + ) + log.info(" Retrying...") + except requests.Timeout, err: + log.exception( + "[StellarGradeBook] Error - timeout in " + "rest_action, err=%s", err + ) + log.info(" Retrying...") + raise Exception( + "[StellarGradeBook] rest_action failure: exceed max retries" + ) + + def rest_action_actual(self, func, url, **kwargs): + """Routine to do low-level REST operation""" + log.info('Running request to %s', url) + resp = func(url, timeout=self.TIMEOUT, verify=False, **kwargs) + try: + retdat = json.loads(resp.content) + except Exception: + log.exception(resp.content) + raise + return retdat + + def get(self, service, params=None, **kwargs): + """ + Generic GET operation for retrieving data from Learning Modules API + Example: + gbk.get('students/{gradebookId}', params=params, gradebookId=gbid) + """ + urlfmt = '{base}/' + service + self.GETS.get(service, '') + url = urlfmt.format(base=self.urlbase, **kwargs) + if params is None: + params = {} + return self.rest_action(self.ses.get, url, params=params) + + def post(self, service, data, **kwargs): + """ + Generic POST operation for sending data to Learning Modules API. + data should be a JSON string or a dict. If it is not a string, + it is turned into a JSON string for the POST body. + """ + urlfmt = '{base}/' + service + url = urlfmt.format(base=self.urlbase, **kwargs) + if not (type(data) == str or type(data) == unicode): + data = json.dumps(data) + headers = {'content-type': 'application/json'} + return self.rest_action(self.ses.post, url, data=data, headers=headers) + + def delete(self, service, data, **kwargs): + """ + Generic DELETE operation for Learning Modules API. + """ + urlfmt = '{base}/' + service + url = urlfmt.format(base=self.urlbase, **kwargs) + if not (type(data) == str or type(data) == unicode): + data = json.dumps(data) + headers = {'content-type': 'application/json'} + return self.rest_action( + self.ses.delete, url, data=data, headers=headers + ) diff --git a/pylmod/client.py b/pylmod/client.py index 17ea7f6..796078e 100644 --- a/pylmod/client.py +++ b/pylmod/client.py @@ -1,26 +1,30 @@ """ -Python interface to MIT Learning Module +Contains the Client class for pylmod that exposes all API classes. """ import logging -from pylmod.stellargradebook import StellarGradeBook +from pylmod.gradebook import GradeBook +from pylmod.membership import Membership log = logging.getLogger(__name__) # pylint: disable=C0103 -class Client(StellarGradeBook): # pylint: disable=too-few-public-methods +class Client(GradeBook, Membership): # pylint: disable=too-few-public-methods """ - Python class representing interface to MIT Learning Modules. + Python class representing interface to MIT Learning Modules API. + + Use Client class to incorporate multiple Learning Modules APIs. + Example usage: - sg = Client('ichuang-cert.pem') - ats = sg.get('academicterms') + gradebook = Client('ichuang-cert.pem') + ats = gradebook.get('academicterms') tc = ats['data'][0]['termCode'] - sg.get('academicterm',termCode=tc) - students = sg.get_students() - assignments = sg.get_assignments() - sg.create_assignment('midterm1', 'mid1', 1.0, 100.0, '11-04-2013') - sid, student = sg.get_student_by_email(email) - aid, assignment = sg.get_assignment_by_name('midterm1') - sg.set_grade(aid, sid, 95.2) - sg.spreadsheet2gradebook(datafn) + gradebook.get('academicterm',termCode=tc) + students = gradebook.get_students() + assignments = gradebook.get_assignments() + gradebook.create_assignment('midterm1', 'mid1', 1.0, 100.0, '11-04-2013') + sid, student = gradebook.get_student_by_email(email) + aid, assignment = gradebook.get_assignment_by_name('midterm1') + gradebook.set_grade(aid, sid, 95.2) + gradebook.spreadsheet2gradebook(datafn) """ pass diff --git a/pylmod/stellargradebook.py b/pylmod/gradebook.py similarity index 68% rename from pylmod/stellargradebook.py rename to pylmod/gradebook.py index 8dbd78b..07ee0a5 100644 --- a/pylmod/stellargradebook.py +++ b/pylmod/gradebook.py @@ -1,53 +1,18 @@ -#!/usr/bin/python """ - Python interface to Stellar Grade Book module - - Defines the class StellarGradeBook +Contains GradeBook class """ - import csv import json import logging import time +from pylmod.base import Base -import requests - +log = logging.getLogger(__name__) # pylint: disable=invalid-name -log = logging.getLogger(__name__) # pylint: disable=C0103 - -class StellarGradeBook(object): +class GradeBook(Base): + """API for functions that return gradebook data from MIT LMod service. """ - Python class representing interface to Stellar gradebook. - - Example usage: - - sg = StellarGradeBook('ichuang-cert.pem') - ats = sg.get('academicterms') - tc = ats['data'][0]['termCode'] - sg.get('academicterm',termCode=tc) - - students = sg.get_students() - assignments = sg.get_assignments() - sg.create_assignment('midterm1', 'mid1', 1.0, 100.0, '11-04-2013') - - sid, student = sg.get_student_by_email(email) - aid, assignment = sg.get_assignment_by_name('midterm1') - sg.set_grade(aid, sid, 95.2) - - sg.spreadsheet2gradebook(datafn) - - """ - - GETS = {'academicterms': '', - 'academicterm': '/{termCode}', - 'gradebook': '?uuid={uuid}'} - - GBUUID = 'STELLAR:/project/mitxdemosite' - TIMEOUT = 200 # connection timeout, seconds - - verbose = True - gradebookid = None def __init__( self, @@ -55,215 +20,32 @@ def __init__( urlbase='https://learning-modules.mit.edu:8443/service/gradebook', gbuuid=None ): - """ - Initialize StellarGradeBook instance. - - - urlbase: URL base for gradebook API - (still needs certs); default False - - gbuuid: gradebook UUID (eg STELLAR:/project/mitxdemosite) - - """ - # pem with private and public key application certificate for access - self.cert = cert - - self.urlbase = urlbase - self.ses = requests.Session() - self.ses.cert = cert - self.ses.timeout = self.TIMEOUT # connection timeout - self.ses.verify = True # verify site certificate - - log.debug("------------------------------------------------------") - log.info("[StellarGradeBook] init base=%s", urlbase) - + super(GradeBook, self).__init__(cert, urlbase) if gbuuid is not None: self.gradebookid = self.get_gradebook_id(gbuuid) - def rest_action(self, func, url, **kwargs): - """Routine to do low-level REST operation, with retry""" - cnt = 1 - while cnt < 10: - cnt += 1 - try: - return self.rest_action_actual(func, url, **kwargs) - except requests.ConnectionError, err: - log.error( - "[StellarGradeBook] Error - connection error in " - "rest_action, err=%s", err - ) - log.info(" Retrying...") - except requests.Timeout, err: - log.exception( - "[StellarGradeBook] Error - timeout in " - "rest_action, err=%s", err - ) - log.info(" Retrying...") - raise Exception( - "[StellarGradeBook] rest_action failure: exceed max retries" - ) - - def rest_action_actual(self, func, url, **kwargs): - """Routine to do low-level REST operation""" - log.info('Running request to %s', url) - resp = func(url, timeout=self.TIMEOUT, verify=False, **kwargs) - try: - retdat = json.loads(resp.content) - except Exception: - log.exception(resp.content) - raise - return retdat - - def get(self, service, params=None, **kwargs): - """ - Generic GET operation for retrieving data from Gradebook API - Example: - sg.get('students/{gradebookId}', params=params, gradebookId=gbid) - """ - urlfmt = '{base}/' + service + self.GETS.get(service, '') - url = urlfmt.format(base=self.urlbase, **kwargs) - if params is None: - params = {} - return self.rest_action(self.ses.get, url, params=params) - - def post(self, service, data, **kwargs): - """ - Generic POST operation for sending data to Gradebook API. - data should be a JSON string or a dict. If it is not a string, - it is turned into a JSON string for the POST body. - """ - urlfmt = '{base}/' + service - url = urlfmt.format(base=self.urlbase, **kwargs) - if not (type(data) == str or type(data) == unicode): - data = json.dumps(data) - headers = {'content-type': 'application/json'} - return self.rest_action(self.ses.post, url, data=data, headers=headers) - - def delete(self, service, data, **kwargs): - """ - Generic DELETE operation for Gradebook API. - """ - urlfmt = '{base}/' + service - url = urlfmt.format(base=self.urlbase, **kwargs) - if not (type(data) == str or type(data) == unicode): - data = json.dumps(data) - headers = {'content-type': 'application/json'} - return self.rest_action( - self.ses.delete, url, data=data, headers=headers - ) - def get_gradebook_id(self, gbuuid): """return gradebookid for a given gradebook uuid.""" gradebook_id = self.get('gradebook', uuid=gbuuid) if 'data' not in gradebook_id: log.info(gradebook_id) - msg = "[StellarGradeBook] error in get_gradebook_id - no data" + msg = "[PyLmod] error in get_gradebook_id - no data" log.info(msg) raise Exception(msg) return gradebook_id['data']['gradebookId'] - def get_students(self, gradebookid='', simple=False, section_name=''): - """ - return list of students for a given gradebook, - specified by a gradebookid. - example return list element: - { - u'accountEmail': u'stellar.test2@gmail.com', - u'displayName': u'Molly Parker', - u'photoUrl': None, - u'middleName': None, - u'section': u'Unassigned', - u'sectionId': 1293925, - u'editable': False, - u'overallGradeInformation': None, - u'studentId': 1145, - u'studentAssignmentInfo': None, - u'sortableName': u'Parker, Molly', - u'surname': u'Parker', - u'givenName': u'Molly', - u'nickName': u'Molly', - u'email': u'stellar.test2@gmail.com' - } - """ - params = dict(includePhoto='false', includeGradeInfo='false', - includeGradeHistory='false', includeMakeupGrades='false') - - url = 'students/{gradebookId}' - if section_name: - groupid, _ = self.get_section_by_name(section_name) - if groupid is None: - msg = ( - 'in get_students -- Error: ' - 'No such section %s' % section_name - ) - log.critical(msg) - raise Exception(msg) - url += '/section/%s' % groupid - - sdat = self.get( - url, - params=params, - gradebookId=gradebookid or self.gradebookid - ) - - if simple: - # just return dict with keys email, name, section - student_map = dict( - accountEmail='email', - displayName='name', - section='section' - ) - - def remap(students): - """ make email domain name uppercase - """ - newx = dict((student_map[k], students[k]) for k in student_map) - # match certs - newx['email'] = newx['email'].replace('@mit.edu', '@MIT.EDU') - return newx - - return [remap(x) for x in sdat['data']] - - return sdat['data'] - - def get_section_by_name(self, section_name): + def get_assignment_by_name(self, assignment_name, assignments=None): """ - return section for a given section name - :param section_name: - :return: section + Get assignment by name; returns assignment ID value (numerical) + and assignment dict. """ - sections = self.get_sections() - for sec in sections: - if sec['name'] == section_name: - return sec['groupId'], sec + if assignments is None: + assignments = self.get_assignments() + for assignment in assignments: + if assignment['name'] == assignment_name: + return assignment['assignmentId'], assignment return None, None - def get_sections(self, gradebookid='', simple=False): - """ - return list of sections for a given gradebook, - specified by a gradebookid. - sample return: - [ - { - "name": "Unassigned", - "editable": false, - "members": null, - "shortName": "def", - "staffs": null, - "groupId": 1293925 - } - ] - """ - params = dict(includeMembers='false') - - sdat = self.get( - 'sections/{gradebookId}', - params=params, - gradebookId=gradebookid or self.gradebookid - ) - - if simple: - return [{'SectionName': x['name']} for x in sdat['data']] - return sdat['data'] - def get_assignments(self, gradebookid='', simple=False): """ return list of assignments for a given gradebook, @@ -281,11 +63,16 @@ def get_assignments(self, gradebookid='', simple=False): return [{'AssignmentName': x['name']} for x in dat['data']] return dat['data'] - def create_assignment( - self, name, shortname, weight, - maxpoints, duedatestr, gradebookid='', + def create_assignment( # pylint: disable=too-many-arguments + self, + name, + shortname, + weight, + maxpoints, + duedatestr, + gradebookid='', **kwargs - ): # pylint: disable=too-many-arguments + ): """ Create a new assignment. """ @@ -300,7 +87,7 @@ def create_assignment( data.update(kwargs) log.info("[StellaGradeBook] Creating assignment %s", name) ret = self.post('assignment', data) - log.debug('posted value=%s', ret) + log.debug('ret=%s', ret) return ret def delete_assignment(self, aid): @@ -312,19 +99,7 @@ def delete_assignment(self, aid): data={}, assignmentId=aid ) - def get_assignment_by_name(self, assignment_name, assignments=None): - """ - Get assignment by name; returns assignment ID value (numerical) - and assignment dict. - """ - if assignments is None: - assignments = self.get_assignments() - for assignment in assignments: - if assignment['name'] == assignment_name: - return assignment['assignmentId'], assignment - return None, None - - def set_grade( + def set_grade( # pylint: disable=too-many-arguments self, assignmentid, studentid, gradeval, gradebookid='', **kwargs ): """ @@ -339,10 +114,11 @@ def set_grade( "assignmentId": assignmentid, "mode": 2, "comment": 'from MITx %s' % time.ctime(time.time()), - "numericGradeValue": str(gradeval)} + "numericGradeValue": str(gradeval) + } gradeinfo.update(kwargs) log.info( - "[StellarGradeBook] student %s set_grade=%s for assignment %s", + "[PyLmod] student %s set_grade=%s for assignment %s", studentid, gradeval, assignmentid) @@ -352,7 +128,6 @@ def set_grade( gradebookId=gradebookid or self.gradebookid ) - # todo.consider renaming this to set_multi_grades() def multi_grade(self, garray, gradebookid=''): """ Set multiple grades for students. @@ -360,28 +135,13 @@ def multi_grade(self, garray, gradebookid=''): return self.post('multiGrades/{gradebookId}', data=garray, gradebookId=gradebookid or self.gradebookid) - def get_student_by_email(self, email, students=None): - """ - Get student based on email address. Calls self.get_students - to get list of all students, if not passed as the students - argument. Returns studentid, student dict, if found. - - return None, None if not found. - """ - if students is None: - students = self.get_students() - - email = email.lower() - for student in students: - if student['accountEmail'].lower() == email: - return student['studentId'], student - return None, None - - def spreadsheet2gradebook(self, datafn, email_field=None, single=False): + def spreadsheet2gradebook( + self, datafn, email_field=None, single=False + ): """ Upload grades from CSV format spreadsheet file into the - Stellar gradebook. The spreadsheet should have a column named - "External email"; this will be used as the student's email + Learning Modules gradebook. The spreadsheet should have a column + named "External email"; this will be used as the student's email address (for looking up and matching studentId). Columns ID,Username,Full Name,edX email,External email are otherwise @@ -428,9 +188,134 @@ def spreadsheet2gradebook(self, datafn, email_field=None, single=False): return resp + def get_sections(self, gradebookid='', simple=False): + """ + return list of sections for a given gradebook, + specified by a gradebookid. + sample return: + [ + { + "name": "Unassigned", + "editable": false, + "members": null, + "shortName": "def", + "staffs": null, + "groupId": 1293925 + } + ] + """ + params = dict(includeMembers='false') + + sdat = self.get( + 'sections/{gradebookId}', + params=params, + gradebookId=gradebookid or self.gradebookid + ) + + if simple: + return [{'SectionName': x['name']} for x in sdat['data']] + return sdat['data'] + + def get_section_by_name(self, section_name): + """ + return section for a given section name + + :param section_name: + :return: + """ + sections = self.get_sections() + for sec in sections: + if sec['name'] == section_name: + return sec['groupId'], sec + return None, None + + def get_student_by_email(self, email, students=None): + """ + Get student based on email address. Calls self.get_students + to get list of all students, if not passed as the students + argument. Returns studentid, student dict, if found. + + return None, None if not found. + """ + if students is None: + students = self.get_students() + + email = email.lower() + for student in students: + if student['accountEmail'].lower() == email: + return student['studentId'], student + return None, None + + def get_students(self, gradebookid='', simple=False, section_name=''): + """ + return list of students for a given gradebook, + specified by a gradebookid. + example return list element: + { + u'accountEmail': u'stellar.test2@gmail.com', + u'displayName': u'Molly Parker', + u'photoUrl': None, + u'middleName': None, + u'section': u'Unassigned', + u'sectionId': 1293925, + u'editable': False, + u'overallGradeInformation': None, + u'studentId': 1145, + u'studentAssignmentInfo': None, + u'sortableName': u'Parker, Molly', + u'surname': u'Parker', + u'givenName': u'Molly', + u'nickName': u'Molly', + u'email': u'stellar.test2@gmail.com' + } + """ + params = dict(includePhoto='false', includeGradeInfo='false', + includeGradeHistory='false', includeMakeupGrades='false') + + url = 'students/{gradebookId}' + if section_name: + groupid, _ = self.get_section_by_name(section_name) + if groupid is None: + msg = ( + 'in get_students -- Error: ' + 'No such section %s' % section_name + ) + log.critical(msg) + raise Exception(msg) + url += '/section/%s' % groupid + + sdat = self.get( + url, + params=params, + gradebookId=gradebookid or self.gradebookid + ) + + if simple: + # just return dict with keys email, name, section + student_map = dict( + accountEmail='email', + displayName='name', + section='section' + ) + + def remap(students): + """ + Convert mit.edu domain to upper-case for student emails + + :param students: + :return: + """ + newx = dict((student_map[k], students[k]) for k in student_map) + # match certs + newx['email'] = newx['email'].replace('@mit.edu', '@MIT.EDU') + return newx + + return [remap(x) for x in sdat['data']] + + return sdat['data'] + def _spreadsheet2gradebook_multi( # pylint: disable=too-many-locals - self, creader, - email_field, non_assignment_fields + self, creader, email_field, non_assignment_fields ): """ Helper function: Transfer grades from spreadsheet using @@ -469,7 +354,8 @@ def _spreadsheet2gradebook_multi( # pylint: disable=too-many-locals 'assignmentId' not in resp.get('data') ): log.warning( - 'Failed to create assignment %s', name) + 'Failed to create assignment %s', name + ) log.info(resp) msg = ( "Error ! Failed to create assignment %s", name @@ -481,15 +367,16 @@ def _spreadsheet2gradebook_multi( # pylint: disable=too-many-locals assignment2id[field] = aid aid = assignment2id[field] - is_successful = True + successful = True try: gradeval = float(cdat[field]) * 1.0 except ValueError as err: log.exception( - "Failed in converting grade for student %s," - " cdat=%s, err=%s", sid, cdat, err) - is_successful = False - if is_successful: + "Failed in converting grade for student %s" + ", cdat=%s, err=%s", sid, cdat, err + ) + successful = False + if successful: garray.append( {"studentId": sid, "assignmentId": aid, @@ -511,8 +398,7 @@ def _spreadsheet2gradebook_multi( # pylint: disable=too-many-locals return resp, duration def _spreadsheet2gradebook_slow( # pylint: disable=too-many-locals - self, creader, - email_field, non_assignment_fields + self, creader, email_field, non_assignment_fields ): """ Helper function: Transfer grades from spreadsheet one at a time @@ -527,10 +413,10 @@ def _spreadsheet2gradebook_slow( # pylint: disable=too-many-locals if field in non_assignment_fields: continue if field not in assignment2id: - aid, _ = self.get_assignment_by_name( + assignment_id, _ = self.get_assignment_by_name( field, assignments=assignments ) - if aid is None: + if assignment_id is None: name = field shortname = field[0:3] + field[-2:] resp = self.create_assignment( @@ -539,17 +425,20 @@ def _spreadsheet2gradebook_slow( # pylint: disable=too-many-locals if 'assignmentId' not in resp['data']: log.info(resp) msg = ( - "Error ! Failed to create assignment %s" % name + "Error ! Failed to create assignment %s", name ) log.error(msg) raise Exception(msg) - aid = resp['data']['assignmentId'] - log.info("Assignment %s has Id=%s", field, aid) - assignment2id[field] = aid + assignment_id = resp['data']['assignmentId'] + log.info("Assignment %s has Id=%s", + field, assignment_id + ) + assignment2id[field] = assignment_id - aid = assignment2id[field] + assignment_id = assignment2id[field] gradeval = float(cdat[field]) * 1.0 log.info( "--> Student %s assignment %s grade %s", - email, field, gradeval) - self.set_grade(aid, sid, gradeval) + email, field, gradeval + ) + self.set_grade(assignment_id, sid, gradeval) diff --git a/pylmod/membership.py b/pylmod/membership.py new file mode 100644 index 0000000..ca79c44 --- /dev/null +++ b/pylmod/membership.py @@ -0,0 +1,15 @@ +""" +Membership class +""" +import logging +from pylmod.base import Base + +log = logging.getLogger(__name__) # pylint: disable=invalid-name + + +class Membership(Base): # pylint: disable=too-few-public-methods + """ + Provide API for functions that return group membership data from MIT + Learning Modules service. + """ + pass