From 9f6277ee13c3e2b3905ec1bfdae01b56d6461b99 Mon Sep 17 00:00:00 2001 From: Mike Graves Date: Fri, 11 Mar 2016 10:28:29 -0500 Subject: [PATCH] Use new HR table --- carbon/__init__.py | 3 +-- carbon/app.py | 56 +++++++++++++++++++++++++++++++++++----- carbon/cli.py | 3 ++- carbon/db.py | 51 ++++++++++++------------------------ tests/conftest.py | 43 ++++++++++++++++++++---------- tests/fixtures/data.json | 15 ----------- tests/fixtures/data.yml | 33 +++++++++++++++++++++++ tests/test_app.py | 17 +++++++++--- tox.ini | 1 + 9 files changed, 146 insertions(+), 76 deletions(-) delete mode 100644 tests/fixtures/data.json create mode 100644 tests/fixtures/data.yml diff --git a/carbon/__init__.py b/carbon/__init__.py index b65bcd0..52cd9ad 100644 --- a/carbon/__init__.py +++ b/carbon/__init__.py @@ -6,7 +6,6 @@ A tool for generating people. """ -__version__ = '0.0.1' +__version__ = '0.1.0' from .app import people, person_feed -from .db import engine, session diff --git a/carbon/app.py b/carbon/app.py index ea3f155..54b847c 100644 --- a/carbon/app.py +++ b/carbon/app.py @@ -1,13 +1,29 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import -from contextlib import contextmanager +from contextlib import contextmanager, closing +from datetime import datetime from functools import partial import re from lxml import etree as ET -from sqlalchemy.sql import select +from sqlalchemy import func, select, and_, or_ -from carbon.db import persons, session +from carbon.db import persons, orcids, engine + + +PS_CODES_1 = ('CFAC', 'CFAN', 'CFAT', 'CFEL', 'CSRS', 'CSRR') +PS_CODES_2 = ('COAC', 'COAR') +TITLES = ( + 'ADJUNCT ASSOCIATE PROFESSOR', 'ADJUNCT PROFESSOR', + 'ASSOCIATE PROFESSOR OF THE PRACTICE', 'INSTITUTE PROFESSOR (WOT)', + 'INSTITUTE PROFESSOR EMERITUS', 'INSTRUCTOR', 'LECTURER', 'LECTURER II', + 'PROFESSOR (WOT)', 'PROFESSOR EMERITUS', 'SENIOR LECTURER', + 'VISITING ASSISTANT PROFESSOR', 'VISITING ASSOCIATE PROFESSOR', + 'VISITING PROFESSOR', 'VISITING SCHOLAR', 'VISITING SCIENTIST', + 'POSTDOCTORAL ASSOCIATE', 'POSTDOCTORAL FELLOW', + 'SENIOR POSTDOCTORAL ASSOCIATE', 'VISITING ENGINEER', 'VISITING SCHOLAR', + 'VISITING SCIENTIST', +) def people(): @@ -15,8 +31,24 @@ def people(): Returns an iterator of person dictionaries. """ - sql = select([persons]).distinct('MIT_ID') - with session() as conn: + sql = select([persons.c.MIT_ID, persons.c.KRB_NAME_UPPERCASE, + persons.c.FIRST_NAME, persons.c.MIDDLE_NAME, + persons.c.LAST_NAME, persons.c.EMAIL_ADDRESS, + persons.c.ORIGINAL_HIRE_DATE, persons.c.HR_ORG_UNIT_TITLE, + persons.c.PERSONNEL_SUBAREA_CODE, orcids.c.ORCID]) \ + .select_from(persons.outerjoin(orcids)) \ + .where(persons.c.EMAIL_ADDRESS != None) \ + .where(persons.c.APPOINTMENT_END_DATE >= datetime(2009, 1, 1)) \ + .where( + or_( + persons.c.PERSONNEL_SUBAREA_CODE.in_(PS_CODES_1), + and_( + persons.c.PERSONNEL_SUBAREA_CODE.in_(PS_CODES_2), + func.upper(persons.c.JOB_TITLE).in_(TITLES) + ) + ) + ) + with closing(engine().connect()) as conn: for row in conn.execute(sql): yield dict(zip(row.keys(), row)) @@ -92,13 +124,23 @@ def person_feed(out): def _add_person(xf, person): record = ET.Element('record') add_child(record, 'field', person['MIT_ID'], name='[Proprietary_ID]') - add_child(record, 'field', person['KRB_NAME'], name='[Username]') + add_child(record, 'field', person['KRB_NAME_UPPERCASE'], name='[Username]') add_child(record, 'field', initials(person['FIRST_NAME'], person['MIDDLE_NAME']), name='[Initials]') add_child(record, 'field', person['LAST_NAME'], name='[LastName]') add_child(record, 'field', person['FIRST_NAME'], name='[FirstName]') - add_child(record, 'field', person['EMAIL'], name='[Email]') + add_child(record, 'field', person['EMAIL_ADDRESS'], name='[Email]') add_child(record, 'field', 'MIT', name='[AuthenticatingAuthority]') add_child(record, 'field', '1', name='[IsAcademic]') + add_child(record, 'field', '1', name='[IsCurrent]') + add_child(record, 'field', '0', name='[LoginAllowed]') + add_child(record, 'field', person['HR_ORG_UNIT_TITLE'], + name='[PrimaryGroupDescriptor]') + add_child(record, 'field', person['ORCID'], name='[Generic01]') + add_child(record, 'field', person['PERSONNEL_SUBAREA_CODE'], + name='[Generic02]') + add_child(record, 'field', + person['ORIGINAL_HIRE_DATE'].strftime("%Y-%m-%d"), + name='[ArriveDate]') xf.write(record) diff --git a/carbon/cli.py b/carbon/cli.py index 8044fe6..410aead 100644 --- a/carbon/cli.py +++ b/carbon/cli.py @@ -3,7 +3,8 @@ import click -from carbon import engine, people, person_feed +from carbon import people, person_feed +from carbon.db import engine @click.group() diff --git a/carbon/db.py b/carbon/db.py index 0d5eec4..72386f7 100644 --- a/carbon/db.py +++ b/carbon/db.py @@ -1,23 +1,31 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import -from contextlib import contextmanager -from sqlalchemy import (create_engine, Table, Column, String, Date, MetaData,) +from sqlalchemy import (create_engine, Table, Column, String, Date, MetaData, + ForeignKey) metadata = MetaData() -persons = Table('library_person_lookup', metadata, +persons = Table('HR_PERSON_EMPLOYEE_LIMITED', metadata, Column('MIT_ID', String), - Column('KRB_NAME', String), + Column('KRB_NAME_UPPERCASE', String), Column('FIRST_NAME', String), - Column('MIDDLE_NAME', String), Column('LAST_NAME', String), - Column('EMAIL', String), - Column('START_DATE', Date), - Column('END_DATE', Date), - Column('DEPARTMENT_NAME', String)) + Column('MIDDLE_NAME', String), + Column('EMAIL_ADDRESS', String), + Column('ORIGINAL_HIRE_DATE', Date), + Column('APPOINTMENT_END_DATE', Date), + Column('HR_ORG_UNIT_TITLE', String), + Column('PERSONNEL_SUBAREA_CODE', String), + Column('JOB_TITLE', String)) + + +orcids = Table('ORCID_TO_MITID', metadata, + Column('MIT_ID', String, + ForeignKey('HR_PERSON_EMPLOYEE_LIMITED.MIT_ID')), + Column('ORCID', String)) class Engine(object): @@ -37,30 +45,5 @@ def configure(self, conn): self._engine = self._engine or create_engine(conn) -@contextmanager -def session(): - """Scoped session context for performing database operations. - - The database engine needs to be configured before using a - session. For example:: - - from carbon import engine, session - - engine.configure('sqlite:///:memory:') - with session() as s: - s.execute() ... - """ - conn = engine().connect() - tx = conn.begin() - try: - yield conn - tx.commit() - except: - tx.rollback() - raise - finally: - conn.close() - - engine = Engine() """Application database engine.""" diff --git a/tests/conftest.py b/tests/conftest.py index 7c5d48c..8f26d44 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,12 +1,13 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import -import json +from contextlib import closing import os from lxml.builder import ElementMaker import pytest +import yaml -from carbon.db import engine, session, metadata, persons +from carbon.db import engine, metadata, persons, orcids @pytest.fixture(scope="session", autouse=True) @@ -19,20 +20,24 @@ def app_init(): @pytest.fixture(scope="session") def records(): current_dir = os.path.dirname(os.path.realpath(__file__)) - data = os.path.join(current_dir, 'fixtures/data.json') + data = os.path.join(current_dir, 'fixtures/data.yml') with open(data) as fp: - r = json.load(fp) + r = list(yaml.load_all(fp)) return r @pytest.yield_fixture def load_data(records): - with session() as s: - s.execute(persons.delete()) - s.execute(persons.insert(), records) + with closing(engine().connect()) as conn: + conn.execute(persons.delete()) + conn.execute(orcids.delete()) + for r in records: + conn.execute(persons.insert(), r['person']) + conn.execute(orcids.insert(), r['orcid']) yield - with session() as s: - s.execute(persons.delete()) + with closing(engine().connect()) as conn: + conn.execute(persons.delete()) + conn.execute(orcids.delete()) @pytest.fixture @@ -40,23 +45,35 @@ def xml_records(E): return [ E.record( E.field('123456', {'name': '[Proprietary_ID]'}), - E.field('foobar', {'name': '[Username]'}), + E.field('FOOBAR', {'name': '[Username]'}), E.field('F B', {'name': '[Initials]'}), E.field('Gaz', {'name': '[LastName]'}), E.field('Foobar', {'name': '[FirstName]'}), E.field('foobar@example.com', {'name': '[Email]'}), E.field('MIT', {'name': '[AuthenticatingAuthority]'}), - E.field('1', {'name': '[IsAcademic]'}) + E.field('1', {'name': '[IsAcademic]'}), + E.field('1', {'name': '[IsCurrent]'}), + E.field('0', {'name': '[LoginAllowed]'}), + E.field('Chemisty', {'name': '[PrimaryGroupDescriptor]'}), + E.field('http://example.com/1', {'name': '[Generic01]'}), + E.field('CFAC', {'name': '[Generic02]'}), + E.field('2001-01-01', {'name': '[ArriveDate]'}) ), E.record( E.field('098754', name='[Proprietary_ID]'), - E.field('thor', name='[Username]'), + E.field('THOR', name='[Username]'), E.field(u'Þ H', name='[Initials]'), E.field('Hammerson', name='[LastName]'), E.field(u'Þorgerðr', name='[FirstName]'), E.field('thor@example.com', name='[Email]'), E.field('MIT', {'name': '[AuthenticatingAuthority]'}), - E.field('1', {'name': '[IsAcademic]'}) + E.field('1', {'name': '[IsAcademic]'}), + E.field('1', {'name': '[IsCurrent]'}), + E.field('0', {'name': '[LoginAllowed]'}), + E.field('Nuclear Science', {'name': '[PrimaryGroupDescriptor]'}), + E.field('http://example.com/2', {'name': '[Generic01]'}), + E.field('COAC', {'name': '[Generic02]'}), + E.field('2015-01-01', {'name': '[ArriveDate]'}) ) ] diff --git a/tests/fixtures/data.json b/tests/fixtures/data.json deleted file mode 100644 index e068cda..0000000 --- a/tests/fixtures/data.json +++ /dev/null @@ -1,15 +0,0 @@ -[{ - "MIT_ID": "123456", - "KRB_NAME": "foobar", - "FIRST_NAME": "Foobar", - "MIDDLE_NAME": "Baz", - "LAST_NAME": "Gaz", - "EMAIL": "foobar@example.com" -}, { - "MIT_ID": "098754", - "KRB_NAME": "thor", - "FIRST_NAME": "Þorgerðr", - "MIDDLE_NAME": "Hǫlgabrúðr", - "LAST_NAME": "Hammerson", - "EMAIL": "thor@example.com" -}] diff --git a/tests/fixtures/data.yml b/tests/fixtures/data.yml new file mode 100644 index 0000000..4133f29 --- /dev/null +++ b/tests/fixtures/data.yml @@ -0,0 +1,33 @@ +--- +person: + MIT_ID: "123456" + KRB_NAME_UPPERCASE: FOOBAR + FIRST_NAME: Foobar + MIDDLE_NAME: Baz + LAST_NAME: Gaz + EMAIL_ADDRESS: foobar@example.com + ORIGINAL_HIRE_DATE: !!timestamp 2001-01-01 00:00:00.0 + APPOINTMENT_END_DATE: !!timestamp 2010-01-01 00:00:00.0 + JOB_TITLE: Adjunct Professor + PERSONNEL_SUBAREA_CODE: CFAC + HR_ORG_UNIT_TITLE: Chemisty +orcid: + MIT_ID: "123456" + ORCID: http://example.com/1 + +--- +person: + MIT_ID: "098754" + KRB_NAME_UPPERCASE: THOR + FIRST_NAME: Þorgerðr + MIDDLE_NAME: Hǫlgabrúðr + LAST_NAME: Hammerson + EMAIL_ADDRESS: thor@example.com + ORIGINAL_HIRE_DATE: !!timestamp 2015-01-01 00:00:00.0 + APPOINTMENT_END_DATE: !!timestamp 2999-01-01 00:00:00.0 + JOB_TITLE: Visiting Engineer + PERSONNEL_SUBAREA_CODE: COAC + HR_ORG_UNIT_TITLE: Nuclear Science +orcid: + MIT_ID: "098754" + ORCID: http://example.com/2 diff --git a/tests/test_app.py b/tests/test_app.py index 5211078..19a2275 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -15,9 +15,14 @@ def test_people_generates_people(): peeps = list(people()) person = peeps[0] - assert person['KRB_NAME'] == 'foobar' + assert person['KRB_NAME_UPPERCASE'] == 'FOOBAR' person = peeps[1] - assert person['KRB_NAME'] == 'thor' + assert person['KRB_NAME_UPPERCASE'] == 'THOR' + + +def test_people_adds_orcids(): + peeps = list(people()) + assert peeps[0]['ORCID'] == 'http://example.com/1' def test_initials_returns_first_and_middle(): @@ -53,8 +58,10 @@ def test_person_feed_uses_namespace(): def test_person_feed_adds_person(records, xml_records, E): b = BytesIO() xml = E.records(xml_records[0]) + r = records[0]['person'].copy() + r.update(records[0]['orcid']) with person_feed(b) as f: - f(records[0]) + f(r) assert b.getvalue() == ET.tostring(xml, encoding="UTF-8", xml_declaration=True) @@ -62,7 +69,9 @@ def test_person_feed_adds_person(records, xml_records, E): def test_person_feed_uses_utf8_encoding(records, xml_records, E): b = BytesIO() xml = E.records(xml_records[1]) + r = records[1]['person'].copy() + r.update(records[1]['orcid']) with person_feed(b) as f: - f(records[1]) + f(r) assert b.getvalue() == ET.tostring(xml, encoding="UTF-8", xml_declaration=True) diff --git a/tox.ini b/tox.ini index 0154285..7c2b8fb 100644 --- a/tox.ini +++ b/tox.ini @@ -6,6 +6,7 @@ skipsdist = True commands = py.test {posargs:--tb=short} deps = pytest + pyyaml mock requests-mock -r{toxinidir}/requirements.txt