Use new HR table

carbon/__init__.py

@@ -6,7 +6,6 @@
     A tool for generating people.
 """
 
-__version__ = '0.0.1'
+__version__ = '0.1.0'
 
 from .app import people, person_feed
-from .db import engine, session

carbon/app.py

@@ -1,22 +1,54 @@
 # -*- coding: utf-8 -*-
 from __future__ import absolute_import
-from contextlib import contextmanager
+from contextlib import contextmanager, closing
+from datetime import datetime
 from functools import partial
 import re
 
 from lxml import etree as ET
-from sqlalchemy.sql import select
+from sqlalchemy import func, select, and_, or_
 
-from carbon.db import persons, session
+from carbon.db import persons, orcids, engine
+
+
+PS_CODES_1 = ('CFAC', 'CFAN', 'CFAT', 'CFEL', 'CSRS', 'CSRR')
+PS_CODES_2 = ('COAC', 'COAR')
+TITLES = (
+    'ADJUNCT ASSOCIATE PROFESSOR', 'ADJUNCT PROFESSOR',
+    'ASSOCIATE PROFESSOR OF THE PRACTICE', 'INSTITUTE PROFESSOR (WOT)',
+    'INSTITUTE PROFESSOR EMERITUS', 'INSTRUCTOR', 'LECTURER', 'LECTURER II',
+    'PROFESSOR (WOT)', 'PROFESSOR EMERITUS', 'SENIOR LECTURER',
+    'VISITING ASSISTANT PROFESSOR', 'VISITING ASSOCIATE PROFESSOR',
+    'VISITING PROFESSOR', 'VISITING SCHOLAR', 'VISITING SCIENTIST',
+    'POSTDOCTORAL ASSOCIATE', 'POSTDOCTORAL FELLOW',
+    'SENIOR POSTDOCTORAL ASSOCIATE', 'VISITING ENGINEER', 'VISITING SCHOLAR',
+    'VISITING SCIENTIST',
+)
 
 
 def people():
     """A person generator.
 
     Returns an iterator of person dictionaries.
     """
-    sql = select([persons]).distinct('MIT_ID')
-    with session() as conn:
+    sql = select([persons.c.MIT_ID, persons.c.KRB_NAME_UPPERCASE,
+                  persons.c.FIRST_NAME, persons.c.MIDDLE_NAME,
+                  persons.c.LAST_NAME, persons.c.EMAIL_ADDRESS,
+                  persons.c.ORIGINAL_HIRE_DATE, persons.c.HR_ORG_UNIT_TITLE,
+                  persons.c.PERSONNEL_SUBAREA_CODE, orcids.c.ORCID]) \
+        .select_from(persons.outerjoin(orcids)) \
+        .where(persons.c.EMAIL_ADDRESS != None) \
+        .where(persons.c.APPOINTMENT_END_DATE >= datetime(2009, 1, 1)) \
+        .where(
+            or_(
+                persons.c.PERSONNEL_SUBAREA_CODE.in_(PS_CODES_1),
+                and_(
+                    persons.c.PERSONNEL_SUBAREA_CODE.in_(PS_CODES_2),
+                    func.upper(persons.c.JOB_TITLE).in_(TITLES)
+                )
+            )
+        )
+    with closing(engine().connect()) as conn:
         for row in conn.execute(sql):
             yield dict(zip(row.keys(), row))
 
@@ -92,13 +124,23 @@ def person_feed(out):
 def _add_person(xf, person):
     record = ET.Element('record')
     add_child(record, 'field', person['MIT_ID'], name='[Proprietary_ID]')
-    add_child(record, 'field', person['KRB_NAME'], name='[Username]')
+    add_child(record, 'field', person['KRB_NAME_UPPERCASE'], name='[Username]')
     add_child(record, 'field', initials(person['FIRST_NAME'],
                                         person['MIDDLE_NAME']),
               name='[Initials]')
     add_child(record, 'field', person['LAST_NAME'], name='[LastName]')
     add_child(record, 'field', person['FIRST_NAME'], name='[FirstName]')
-    add_child(record, 'field', person['EMAIL'], name='[Email]')
+    add_child(record, 'field', person['EMAIL_ADDRESS'], name='[Email]')
     add_child(record, 'field', 'MIT', name='[AuthenticatingAuthority]')
     add_child(record, 'field', '1', name='[IsAcademic]')
+    add_child(record, 'field', '1', name='[IsCurrent]')
+    add_child(record, 'field', '0', name='[LoginAllowed]')
+    add_child(record, 'field', person['HR_ORG_UNIT_TITLE'],
+              name='[PrimaryGroupDescriptor]')
+    add_child(record, 'field', person['ORCID'], name='[Generic01]')
+    add_child(record, 'field', person['PERSONNEL_SUBAREA_CODE'],
+              name='[Generic02]')
+    add_child(record, 'field',
+              person['ORIGINAL_HIRE_DATE'].strftime("%Y-%m-%d"),
+              name='[ArriveDate]')
     xf.write(record)

carbon/cli.py

@@ -3,7 +3,8 @@
 
 import click
 
-from carbon import engine, people, person_feed
+from carbon import people, person_feed
+from carbon.db import engine
 
 
 @click.group()

carbon/db.py

@@ -1,23 +1,31 @@
 # -*- coding: utf-8 -*-
 from __future__ import absolute_import
-from contextlib import contextmanager
 
-from sqlalchemy import (create_engine, Table, Column, String, Date, MetaData,)
+from sqlalchemy import (create_engine, Table, Column, String, Date, MetaData,
+                        ForeignKey)
 
 
 metadata = MetaData()
 
 
-persons = Table('library_person_lookup', metadata,
+persons = Table('HR_PERSON_EMPLOYEE_LIMITED', metadata,
                 Column('MIT_ID', String),
-                Column('KRB_NAME', String),
+                Column('KRB_NAME_UPPERCASE', String),
                 Column('FIRST_NAME', String),
-                Column('MIDDLE_NAME', String),
                 Column('LAST_NAME', String),
-                Column('EMAIL', String),
-                Column('START_DATE', Date),
-                Column('END_DATE', Date),
-                Column('DEPARTMENT_NAME', String))
+                Column('MIDDLE_NAME', String),
+                Column('EMAIL_ADDRESS', String),
+                Column('ORIGINAL_HIRE_DATE', Date),
+                Column('APPOINTMENT_END_DATE', Date),
+                Column('HR_ORG_UNIT_TITLE', String),
+                Column('PERSONNEL_SUBAREA_CODE', String),
+                Column('JOB_TITLE', String))
+
+
+orcids = Table('ORCID_TO_MITID', metadata,
+               Column('MIT_ID', String,
+                      ForeignKey('HR_PERSON_EMPLOYEE_LIMITED.MIT_ID')),
+               Column('ORCID', String))
 
 
 class Engine(object):
@@ -37,30 +45,5 @@ def configure(self, conn):
         self._engine = self._engine or create_engine(conn)
 
 
-@contextmanager
-def session():
-    """Scoped session context for performing database operations.
-
-    The database engine needs to be configured before using a
-    session. For example::
-
-        from carbon import engine, session
-
-        engine.configure('sqlite:///:memory:')
-        with session() as s:
-            s.execute() ...
-    """
-    conn = engine().connect()
-    tx = conn.begin()
-    try:
-        yield conn
-        tx.commit()
-    except:
-        tx.rollback()
-        raise
-    finally:
-        conn.close()
-
-
 engine = Engine()
 """Application database engine."""

tests/conftest.py

@@ -1,12 +1,13 @@
 # -*- coding: utf-8 -*-
 from __future__ import absolute_import
-import json
+from contextlib import closing
 import os
 
 from lxml.builder import ElementMaker
 import pytest
+import yaml
 
-from carbon.db import engine, session, metadata, persons
+from carbon.db import engine, metadata, persons, orcids
 
 
 @pytest.fixture(scope="session", autouse=True)
@@ -19,44 +20,60 @@ def app_init():
 @pytest.fixture(scope="session")
 def records():
     current_dir = os.path.dirname(os.path.realpath(__file__))
-    data = os.path.join(current_dir, 'fixtures/data.json')
+    data = os.path.join(current_dir, 'fixtures/data.yml')
     with open(data) as fp:
-        r = json.load(fp)
+        r = list(yaml.load_all(fp))
     return r
 
 
 @pytest.yield_fixture
 def load_data(records):
-    with session() as s:
-        s.execute(persons.delete())
-        s.execute(persons.insert(), records)
+    with closing(engine().connect()) as conn:
+        conn.execute(persons.delete())
+        conn.execute(orcids.delete())
+        for r in records:
+            conn.execute(persons.insert(), r['person'])
+            conn.execute(orcids.insert(), r['orcid'])
     yield
-    with session() as s:
-        s.execute(persons.delete())
+    with closing(engine().connect()) as conn:
+        conn.execute(persons.delete())
+        conn.execute(orcids.delete())
 
 
 @pytest.fixture
 def xml_records(E):
     return [
         E.record(
             E.field('123456', {'name': '[Proprietary_ID]'}),
-            E.field('foobar', {'name': '[Username]'}),
+            E.field('FOOBAR', {'name': '[Username]'}),
             E.field('F B', {'name': '[Initials]'}),
             E.field('Gaz', {'name': '[LastName]'}),
             E.field('Foobar', {'name': '[FirstName]'}),
             E.field('foobar@example.com', {'name': '[Email]'}),
             E.field('MIT', {'name': '[AuthenticatingAuthority]'}),
-            E.field('1', {'name': '[IsAcademic]'})
+            E.field('1', {'name': '[IsAcademic]'}),
+            E.field('1', {'name': '[IsCurrent]'}),
+            E.field('0', {'name': '[LoginAllowed]'}),
+            E.field('Chemisty', {'name': '[PrimaryGroupDescriptor]'}),
+            E.field('http://example.com/1', {'name': '[Generic01]'}),
+            E.field('CFAC', {'name': '[Generic02]'}),
+            E.field('2001-01-01', {'name': '[ArriveDate]'})
         ),
         E.record(
             E.field('098754', name='[Proprietary_ID]'),
-            E.field('thor', name='[Username]'),
+            E.field('THOR', name='[Username]'),
             E.field(u'Þ H', name='[Initials]'),
             E.field('Hammerson', name='[LastName]'),
             E.field(u'Þorgerðr', name='[FirstName]'),
             E.field('thor@example.com', name='[Email]'),
             E.field('MIT', {'name': '[AuthenticatingAuthority]'}),
-            E.field('1', {'name': '[IsAcademic]'})
+            E.field('1', {'name': '[IsAcademic]'}),
+            E.field('1', {'name': '[IsCurrent]'}),
+            E.field('0', {'name': '[LoginAllowed]'}),
+            E.field('Nuclear Science', {'name': '[PrimaryGroupDescriptor]'}),
+            E.field('http://example.com/2', {'name': '[Generic01]'}),
+            E.field('COAC', {'name': '[Generic02]'}),
+            E.field('2015-01-01', {'name': '[ArriveDate]'})
         )
     ]
 

tests/fixtures/data.yml

@@ -0,0 +1,33 @@
+---
+person:
+    MIT_ID: "123456"
+    KRB_NAME_UPPERCASE: FOOBAR
+    FIRST_NAME: Foobar
+    MIDDLE_NAME: Baz
+    LAST_NAME: Gaz
+    EMAIL_ADDRESS: foobar@example.com
+    ORIGINAL_HIRE_DATE: !!timestamp 2001-01-01 00:00:00.0
+    APPOINTMENT_END_DATE: !!timestamp 2010-01-01 00:00:00.0
+    JOB_TITLE: Adjunct Professor
+    PERSONNEL_SUBAREA_CODE: CFAC
+    HR_ORG_UNIT_TITLE: Chemisty
+orcid:
+    MIT_ID: "123456"
+    ORCID: http://example.com/1
+
+---
+person:
+    MIT_ID: "098754"
+    KRB_NAME_UPPERCASE: THOR
+    FIRST_NAME: Þorgerðr
+    MIDDLE_NAME: Hǫlgabrúðr
+    LAST_NAME: Hammerson
+    EMAIL_ADDRESS: thor@example.com
+    ORIGINAL_HIRE_DATE: !!timestamp 2015-01-01 00:00:00.0
+    APPOINTMENT_END_DATE: !!timestamp 2999-01-01 00:00:00.0
+    JOB_TITLE: Visiting Engineer
+    PERSONNEL_SUBAREA_CODE: COAC
+    HR_ORG_UNIT_TITLE: Nuclear Science
+orcid:
+    MIT_ID: "098754"
+    ORCID: http://example.com/2

tests/test_app.py

@@ -15,9 +15,14 @@
 def test_people_generates_people():
     peeps = list(people())
     person = peeps[0]
-    assert person['KRB_NAME'] == 'foobar'
+    assert person['KRB_NAME_UPPERCASE'] == 'FOOBAR'
     person = peeps[1]
-    assert person['KRB_NAME'] == 'thor'
+    assert person['KRB_NAME_UPPERCASE'] == 'THOR'
+
+
+def test_people_adds_orcids():
+    peeps = list(people())
+    assert peeps[0]['ORCID'] == 'http://example.com/1'
 
 
 def test_initials_returns_first_and_middle():
@@ -53,16 +58,20 @@ def test_person_feed_uses_namespace():
 def test_person_feed_adds_person(records, xml_records, E):
     b = BytesIO()
     xml = E.records(xml_records[0])
+    r = records[0]['person'].copy()
+    r.update(records[0]['orcid'])
     with person_feed(b) as f:
-        f(records[0])
+        f(r)
     assert b.getvalue() == ET.tostring(xml, encoding="UTF-8",
                                        xml_declaration=True)
 
 
 def test_person_feed_uses_utf8_encoding(records, xml_records, E):
     b = BytesIO()
     xml = E.records(xml_records[1])
+    r = records[1]['person'].copy()
+    r.update(records[1]['orcid'])
     with person_feed(b) as f:
-        f(records[1])
+        f(r)
     assert b.getvalue() == ET.tostring(xml, encoding="UTF-8",
                                        xml_declaration=True)

tox.ini

@@ -6,6 +6,7 @@ skipsdist = True
 commands = py.test {posargs:--tb=short}
 deps =
     pytest
+    pyyaml
     mock
     requests-mock
     -r{toxinidir}/requirements.txt