ANN_Text_Classifier_K-fold_Cross-validation_9_17_18.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# use natural language toolkit\n",
    "import nltk\n",
    "from nltk import punkt\n",
    "from nltk.stem.lancaster import LancasterStemmer\n",
    "from nltk.corpus import stopwords\n",
    "from sklearn.feature_extraction import stop_words\n",
    "import sklearn.datasets\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.metrics import accuracy_score, f1_score, precision_score, \\\n",
    "recall_score, classification_report, confusion_matrix\n",
    "import os\n",
    "import json\n",
    "import datetime\n",
    "stemmer = LancasterStemmer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 4 classes of training data\n",
    "# 0 = Performance\n",
    "# 1 = Usability\n",
    "# 2 = Security\n",
    "# 3 = Operability\n",
    "\n",
    "training_data = []\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The system shall refresh the display every 60 seconds.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The product shall respond fast to keep up-to-date data in the display.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The product shall produce search results in an acceptable time\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The search results shall be returned no later 30      seconds  after the user has entered the search criteria\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The product shall generate a CMA report in an acceptable time.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The CMA report shall be returned no later 60 seconds after the user has entered the CMA report criteria.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The product shall synchronize contacts and appointments in an acceptable time.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The product shall synchronize with the office system every hour.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"Staff members shall be able to complete a set of tasks in a timely manner.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The response time of general student management tasks shall take no longer than 5 seconds and the response time of schedule generation shall take no longer than 30 seconds.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The maximum wait time for a user navigating from one screen to another within the Disputes application shall be no more than 5 seconds.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The Disputes application shall support 350 concurrent users without any degradation of performance in the application.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The search for recycled parts shall take no longer than 15 seconds.  The search results shall be returned in under 15 seconds.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The search for the preferred repair facility shall take no longer than 8 seconds.  The preferred repair facility is returned within 8 seconds\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The recycled parts audit report shall be returned to the user within 10 seconds.  The audit report shall be returned within 10 seconds.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The preferred repair facility ratings shall be saved within 5 seconds. The save shall occur within 5 seconds.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The response shall be fast enough to avoid interrupting the userís flow of thought.  90% of tasks shall complete within 5 seconds.  98% of tasks shall complete within 8 seconds.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"Recreation of C360Inventory Quantity Adjustment for the period of previous 30 days shall take no more than 30min.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The System shall allow for a minimum of  6 users to work at the same time.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"Administrator shall be able to activate a pre-paid card via the Administration section in under 5 seconds.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"Customer shall be able to check the status of their prepaid card  by entering in the PIN number  in under 5 seconds.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"System shall let customers register on the website as a ìpay as you goî user in under 5 minutes.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"System shall let customers register on the website as a ìunlimited movie subscriberî user in under 10 minutes.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"System shall let existing customers log into the website with their email address and password in under 5 seconds\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"System shall let customers cancel their account  process shall take no longer than 2 minutes.  Database will mark the account as inactive  and user will not be able to log into the website.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"System shall let administrator de-activate a customer account in under 1 minute. Customer will no longer be able to access the website.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"System shall let administrator add/remove movies on the website in under 5 minutes.  Entered movie information will be stored in the database and will now be available on the website.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"System shall let the administrator add/remove categories on the website in under 5 minutes.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"System shall let Izogn Manager access sales and usage statistics in under 2 minutes.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"System shall let administrators generate unique pre-paid pins in under 2 minutes and generate an Excel file with the generated PINs.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"Any interface between a user and the automated system shall have a maximum response time of 5 seconds  unless noted by an exception below.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"When purchasing a streaming movie or pre-paid card via credit card  the processing time should have a maximum response time of 15 seconds.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"When streaming a movie  the buffering time should take no longer than 10 seconds (plus any latency on the userís connection.)\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"While streaming a movie  buffering should only occur if there is latency in the userís connection.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"Izogn Administrator must be able to update the category listings on the website within 2 minutes.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"Total lead cycle time must be completed in 5 minutes for 90% of lead volume between the business hours of 8AM CST to 8PM CST; then 3 minutes for 99% of lead volume during off business hours.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"No lead evaluation will complete in a time period longer than 25 seconds for 97% of all leads processed.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"For leads that process longer than 25 seconds the system will record the event and duration\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The process must fit within the processing time conditions set forth by the Auto Rep Assignment process in which a lead is assigned within 3 minutes of the total life cycle.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The product shall able to handle a large number of games simultaneously.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The response time shall be fast enough to maintain the flow of the game. The response time shall be no more than 2 seconds for 95% of responses and no more than 4% for the remaining responses.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"90% of tasks shall complete within 5 seconds.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"98% of tasks shall complete within 8 seconds.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"Any interface between a user and the automated system shall have a maximum response time of 5 seconds  unless noted by an exception below.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"Aside from server failure  the software product shall achieve 99.99% up time.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"For 10% of the time  the response time should not be more than 10 seconds.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"No lead evaluation will complete in a time period longer than 25 seconds for 97% of all leads processed.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"On a 112k connection or faster  the system response time for all operations must be no more than 3 seconds 90% of the time.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"On a 56k connection  the system response time for all operations must be no more than 6 seconds 90% of the time.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"Recreation of Inventory Quantity Adjustment for the period of previous 30 days shall take no more than 30min.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"system shall be able to handle all of the user requests or usage during business hours.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The application shall be able to complete every user transaction within one fourth of a second of receiving the request over the network.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The audit report shall be returned within 10 seconds.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The average number of recycled parts records per day shall be 50 000.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The CMA report shall be returned no later 60 seconds after the user has entered the CMA report criteria.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The concurrency capacity must be able to handle peak scheduling times such as early morning and late afternoon hours.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The Disputes application shall support 350 concurrent users without any degradation of performance in the application.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The maximum wait time for a user navigating from one screen to another within the Disputes application shall be no more than 5 seconds.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The peak CPU utilization shall never exceed 70% of CPU capacity at any given time.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The preferred repair facility is returned within 8 seconds\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The preferred repair facility ratings shall be saved within 5 seconds.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The process must fit within the processing time conditions set forth by the Auto Rep Assignment process in which a lead is assigned within 3 minutes of the total life cycle.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The product interface should be fast.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The product shall able to handle a large number of games simultaneously.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The product shall be able support 1000 simultaneous users.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The product shall be able to handle 10 000 concurrent users within 2 years of the initial launch.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The product shall be able to handle up to 1500 simultaneous users.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The product shall be able to process 10 000 transactions per hour within two years of its launch.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The product shall be able to support multiple remote users\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The product shall be capable of handling the existing 1000 users.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The product shall be capable of handling up to 1 000 concurrent requests.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The product shall be expected to operate for at least 5 years for each customer installation.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The product shall generate a CMA report in an acceptable time.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The product shall have a maximum message processing time of 2 seconds\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The product shall process messages within 2 seconds.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The product shall process minimum of 1 million transactions per year.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The product shall provide Multi-threading and Load Balancing.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The product shall provide multi-threading technology as well as load balancing to overcome data load spikes and traffic.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The product shall respond fast to keep up-to-date data in the display.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The product shall synchronize contacts and appointments in an acceptable time.\"})\n",
    "#training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The product shall use symbols and words that are naturally understandable by the realtor community.\"\"})The product shall produce search results in an acceptable time\"\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The recycled parts audit report shall be returned to the user within 10 seconds.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The response time for the most common requests to reach a user shall not exceed 3 seconds.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The response time of general student management tasks shall take no longer than 5 seconds and the response time of schedule generation shall take no longer than 30 seconds.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The response time of the product interface should not exceed 2 seconds 90% of the time.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The response time shall be fast enough to maintain the flow of the game.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The response time shall be no more than 2 seconds for 95% of responses and no more than 4% for the remaining responses.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The save shall occur within 5 seconds.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The search for recycled parts shall take no longer than 15 seconds.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The search for the preferred repair facility shall take no longer than 8 seconds.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The search results shall be returned in under 15 seconds.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The search results shall be returned no later 30      seconds  after the user has entered the search criteria\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The server will support a maximum of 1 000 simultaneous users.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The solution shall complete an average of 99% of all online update transactions in under 5 seconds over any 60-minute period, during peak usage.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The solution shall track system uptime and transaction response times in order to demonstrate operation within acceptable levels.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The System shall allow for a minimum of  6 users to work at the same time.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The system shall be able to handle an average transaction load with an average CPU utilization of no more than 35%-40% of the CPU capacity.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The system shall be capable of processing 100% of nursing students and their classes for the next 10 years.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The system shall be expected to manage the nursing program curriculum and class or clinical scheduling for a minimum of 5 years.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The system shall be scalable and adaptable to meet future growth and expansion needs.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The system shall cater to 10 simultaneous users from 9am to 5pm.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The system shall have a response time where the average transaction on the server needs to occur on average less than 1 second.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The system shall include documentation of product capacities (e.g. number of users, number of transactions per second, number of records, network load, etc.) and the baseline representative configurations assumed for these capacities (e.g. number or type of processors, server or workstation configuration and network capacity, etc).\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The system shall perform with 1500 users as a time.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The system shall refresh the display every 60 seconds.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The system will need the support capability of up to and beyond 5000 default schedules.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The system, including programs, database, and ancillary hardware and related software systems shall be able to retain its performance levels when adding additional users, functions, and data.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The website shall be capable of supporting 100 000 customers.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"There is no limitation to the number of accepted formats.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"This number is expected to grow 5 times within the next year\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"This number will increase to 2 000 by Release 2.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"This number will increase to 20 000 by Release 2.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"Total lead cycle time must be completed in 5 minutes for 90% of lead volume between the business hours of 8AM CST to 8PM CST; then 3 minutes for 99% of lead volume during off business hours.\"})\n",
    "training_data.append({\"class\":\"0\", \"class_name\":\"Performance\", \"sentence\":\"The response time shall be fast enough to maintain the flow of the game.\"})\n",
    "\n",
    "\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"If projected  the data must be readable.  On a 10x10 projection screen  90% of viewers must be able to read Event / Activity data from a viewing distance of 30.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The system shall be used by realtors with no training\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The system shall be intuitive and self explanatory.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The product shall be easy for a realtor to learn.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The product shall use symbols and words that are naturally understandable by the realtor community.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The product shall be installed by an untrained realtor   without recourse to separately-printed instructions.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"90% of untrained realtors shall be able to install the product on their device without printed instructions\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The list of dispute cases that are displayed after a search is performed must be color coded for easy identification of dispute cases based upon the dispute case status.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"100% of the cardmember and merchant services representatives shall use the Disputes application regularly after a 2-day training course.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"100% of cardmember services representatives shall be able to successfully create a dispute case on the first encounter after completing the training course.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"100% of merchant services representatives shall be able to successfully perform a follow up action on a dispute case on the first encounter after completing the training course.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The product shall be easy to use by Adjusters and Collision Estimators.  95% of Adjusters and Collision Estimators shall find the product easy to use.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The product shall increase productivity of Collision Estimators. 80% of the Collision Estimators shall agree their productivity has increase within 1 month of using the product.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"Users shall feel satisfied using the product.  85% of all users will be satisfied with the product.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The product shall be easy to learn by Adjusters and Collision Estimators.  The product shall be learned with two days onsite training\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The user shall easily locate instructions while using the product.  User help can be found within 90% of the system.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The product shall make the users want to use it. 80% of the users surveyed report they are regularly using the product after the first 2 weeks post-launch.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The product shall be easy to use. 90% of users will be able to successfully reserve a conference room within 5 minutes of product use.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The product shall give users feedback when necessary. 80% of the users surveyed report that the product accurately confirms their actions.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The product shall allow the user to select a chosen language from one of the target market countries.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The product shall allow for customization of start page and views preferences.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"An employee will be able to successfully use the product within a few minutes.  After informally navigating the product for less than 15 minutes  users shall be able to successfully setup meetings and reserve conference rooms.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The product shall use a standard navigation menu familiar to most web users.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The product shall allow for intuitive searching of available conference rooms.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The product shall conform to the Americans with Disabilities Act.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The product should be able to be used by 90% of novice users on the Internet.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"Users should only have to navigate through a maximum of 4 pages  to create an account with the website.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"Users should be able to access their streaming movies in under 2 clicks after logging into the website.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"An anonymous survey will show that 70% of customers are comfortable purchasing streaming movies after 2 weeks using the website.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"An anonymous survey will show that 95% of customers are comfortable using and purchasing pre-paid cards after a month of using the system.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The product shall retain the customerís buying preferences and information.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The product shall be easy for customers with novice skills on the  Internet to purchase and watch streaming movies.  Customers should be able to purchase and watch a pre-determined movie in under 3 minutes\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The product shall use symbols and words that are naturally understandable by users familiar with websites\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The website will use standard Internet conventions when interacting with the user.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The product shall be intuitive and self-explanatory.  Nine out ten users shall be able to successfully complete a game with no training.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The product shall be available to players who understand English  Spanish and French.  The product shall allow the user to select the language upon accessing the product.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"If projected  the data must be understandable. On a 10x10 projection screen  90% of viewers must be able to determine that Events or Activities are occuring in current time from a viewing distance of 100.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The product shall be intuitive and self-explanatory.  :  90% of new users shall be able to start the display of Events or Activities within 90 minutes of using the product.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The system shall be easy to use by the Program Administrators/Nursing Staff Members.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"All of the users shall be able to use the system after one day of training.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The system shall help the user avoid making mistakes while scheduling classes and clinicals for the nursing students.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The number of mistakes noted by the students shall be decreased by 50% in the first year.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The system shall be intuitive to the Program Administrators/ Nursing Staff Members.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"2 out of 3 Program Administrators/Nursing Staff Members shall successfully be able to use the system to manage the scheduling of classes and clinicals.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The system shall use symbols and words that are naturally understandable by the user community.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The verbiage used within the system shall be consistent with Standard English and the terminology of the national Nursing Community.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"Every registered user will have access to the productís support site via the Internet.  70% of registered users shall find a solution to their problem within 5 minutes of using the support site.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The product shall be translated into foreign languages other than the target market countriesí languages in future releases.  For each emerging target market  it shall take no more than 5 days to configure the product for that marketís language.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The product shall be able to display calendar dates and times according to the userís locale.  90% of survey respondents from each target market country will find that the date and time formatting of the product is correct.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The website will provide a help section with answers and solutions to common problems.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"(iii) The covered entity informs the individual that it is terminating its agreement to a restriction, except that such termination is only effective with respect to protected health information created or received after it has so informed the individual.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"100% of all selection and navigation functionality must be able to be activated or selected by either a mouse or a keyboard\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"100% of cardmember services representatives shall be able to successfully create a dispute case on the first encounter after completing the training course.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"100% of merchant services representatives shall be able to successfully perform a follow up action on a dispute case on the first encounter after completing the training course.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"100% of the cardmember and merchant services representatives shall use the Disputes application regularly after a 2-day training course.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"2 out of 3 Program Administrators or Nursing Staff Members shall successfully be able to use the system to manage the scheduling of classes and clinicals.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"70% of registered users shall find a solution to their problem within 5 minutes of using the support site\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"80% of the Collision Estimators shall agree their productivity has increase within 1 month of using the product.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"80% of the users surveyed report that the product accurately confirms their actions.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"80% of the users surveyed report they are regularly using the product after the first 2 weeks post-launch.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"85% of all users will be satisfied with the product.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"90% of new users are able to log into the system within 10 minutes.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"90% of new users shall be able to start the display of Events or Activities within 90 minutes of using the product.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"90% of untrained realtors shall be able to install the product on their device without printed instructions\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"90% of users should be able to successfully use the system to generate a report on their first use.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"90% of users will be able to successfully reserve a conference room within 5 minutes of product use.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"95% of Adjusters and Collision Estimators shall find the product easy to use.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"A new user must be able to be added to the system within 10 minutes.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"A new user shall be able to navigate through the league and team pages within 30 seconds of reaching the start-up page.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"A user shall be able to use the system without instruction after a short amount of time.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"After informally navigating the product for less than 15 minutes  users shall be able to successfully setup meetings and reserve conference rooms.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"After one use a user should understand and remember 90% of the system usage and navigation.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"All CCR users shall receive training before the WCS system is deployed.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"All interfaces and reports will undergo usability tests by CCR users.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"All of the users shall be able to use the system after one day of training.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"An employee will be able to successfully use the product within a few minutes.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"Callers and supervisors must be able to accomplish any system task within 2 minutes.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"Callers and supervisors shall learn how to use the system within 10 minutes.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"Every registered user will have access to the product's support site via the Internet.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"If projected  the data must be readable.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"If projected  the data must be understandable.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"Integration of new maps with the product shall be possible with little to no support from product support staff.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"Intelligent spell checking of text fields\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"Nine out ten users shall be able to successfully complete a game with no training.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"NOTE: Clicking the calendar button next to the 'DOB' field will pop up an easy to use date selection tool found throughout the OpenEMR program.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"Notice that the 'Add New Issue' dialog also uses the OpenEMR date selection tool described earlier.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"Notice that the source and pay date from the previous window have been copied over; this saves time in the common case where a single check from the insurance company pays for multiple claims.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"On a 10x10 projection screen  90% of viewers must be able to determine that Events or Activities are occuring in current time from a viewing distance of 100\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"On a 10x10 projection screen  90% of viewers must be able to read Event  or  Activity data from a viewing distance of 30\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"One training session will be held for 100% of CCR users before system deployment (based on a CCR user being available)\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"On-line, interactive help with support for hyperlink technology and industry standard formats (e.g., HTML file formats)\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"Reports must be generated within one button click and exported within one button click (after selecting the report)\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"Rreports must be easy to generate.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"Staff members shall be able to complete a set of tasks in a timely manner.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The application shall ask the user to confirm dropping a player or trading a player before performing the action.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The customer shall be able to easily integrate new building maps with the product throughout the product's lifecycle.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The data elements required to complete a job function, whether to inquire or read only or data entry must be readily available.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The enterer or editor is presented with a success or failure message and the form is updated so that the enterer or editor may correct the form or add more information.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The form validation of the system shall show the errors of all the fields in a form at the same time.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The interface shall be easy to use.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The interface should adhere to IBM usability standards.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The list of dispute cases that are displayed after a search is performed must be color coded for easy identification of dispute cases based upon the dispute case status.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The number of mistakes noted by the students shall be decreased by 50% in the first year.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The product shall allow for intuitive searching of available conference rooms.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The product shall be easy for a realtor to learn.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The product shall be easy for System Administrators and DBAs to use after two weeks of usage.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The product shall be easy to learn by Adjusters and Collision Estimators.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The product shall be easy to use by Adjusters and Collision Estimators.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The product shall be easy to use\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The product shall be installed by an untrained realtor   without recourse to separately-printed instructions.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The product shall be intuitive and self-explanatory.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The product shall be intuitive and self-explanatory.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The product shall be learned with two days onsite training\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The product shall give users feedback when necessary.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The product shall have an intuitive user interface.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The product shall help the System Administrators and DBAs to resolve transaction errors 99% of the time.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The product shall increase productivity of Collision Estimators.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The product shall make the users want to use it.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The product shall use a standard navigation menu familiar to most web users.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The product should be able to be used by 90% of novice users on the Internet.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The Proposer shall allow for field level edit checks for transactions during data entry and provide immediate user feedback, including error messages and possible corrective actions\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The response shall be fast enough to avoid interrupting the user's flow of thought.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The system has capability to have voice recognition.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The system has capability to use visual cues to highlight abnormal results.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The system has the ability to display critical relevant data when creating orders.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The system has the capability to alert the user of lookalike and sound alike drugs.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The system has the capability to detect duplicate orders issuing visual and auditory warnings.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The system has the capability to use rules associated with required prompts that can either be used as warning to the usre or prevent moving to a next step until a specific action has been taken.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The system must use a point and click interface.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The system shall allow for the option of auto-fill capability per transaction or field entry throughout all modules\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The system shall allow for user-friendly end-user report creation without requiring technical staff or expertise to create and publish reports within the modules\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The system shall be easy to learn by callers and supervisors.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The system shall be easy to use by callers and supervisors.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The system shall be easy to use by the Program Administrators or Nursing Staff Members.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The system shall be intuitive and self explanatory.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The system shall be intuitive to the Program Administrators or Nursing Staff Members.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The system shall be used by realtors with no training.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The system shall display patient annotations in a manner which distinguishes them from other content in the system.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The system shall have a customizable online documentation and training materials such as context-specific help, search capability, organization-specific business process documentation and process maps.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The system shall have the ability to design a preferred sequence to make data-entry columns and fields match the order of information in organization source documents.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The system shall have the ability to restrict free form entry (e.g., provide drop down calendar for date field)\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The system shall help the user avoid making mistakes while scheduling classes and clinicals for the nursing students.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The system shall include documentation that explains system error or performance messages to users and administrators, with the actions required.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The system shall present data to users such that a minimum of navigational effort is required, including:\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The system shall provide templates for inputting data in a structured format as part of clinical documentation.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The system shall provide the ability to document encounters by one or more of the following means: direct keyboard entry of text; structured data entry utilizing templates, forms, pick lists or macro substitution; dictation with subsequent transcription of voice to text, either manually or via voice recognition system.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The system shall use symbols and words that are naturally understandable by the user community.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The system supports grouping like problems into disease categories (e.g., cerebrovascular disease, congenital heart disease, heart failure, hypertensive heart disease, ischemic heart disease, etc.) and enable one-click search when adding a problem to the list.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The user interface shall integrate information from multiple components into a unified display by business area or work type.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The user shall easily locate instructions while using the product.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The users should be able to easily use the system to successfully complete their budgets  within the stipulated time for completion.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The verbiage used within the system shall be consistent with Standard English and the terminology of the national Nursing Community.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The WCS system shall use appropriate nomenclature and terminology as defined by the Corporate Community Grants organization.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"The website will use standard Internet conventions when interacting with the user.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"User help can be found within 90% of the system.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"Users shall feel satisfied using the product.\"})\n",
    "training_data.append({\"class\":\"1\", \"class_name\":\"Usability\", \"sentence\":\"With a week of training prior to product implementation  98% of the users shall complete their budgets within the stipulated time.\"})\n",
    "\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The product shall ensure that it can only be accessed by authorized users.  The product will be able to distinguish between authorized and unauthorized users in all access attempts.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Only registered realtors shall be able to access the system.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Every user of the system shall be authenticated and authorized.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The product shall prevent its data from incorrect data being introduced\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Only authorized users shall have access to clinical site information.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Program Administrators/Nursing Staff Members are the only people who shall have access to clinical site details.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Only authorized users shall have access to studentsí personal information.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Dr Susan Poslusny and Dr Julie Donalek are the only people who shall have access to studentsí personal information.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Only authorized users shall have access to the portion of the system that interfaces with CampusConnect.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Fit Criterion: Dr Susan Poslusny and Karen Sysol are the only people who shall have access to the final class section scheduling for the system that interfaces with CampusConnect.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The system shall have basic data integrity checking to reduce the possibility of incorrect or invalid data being introduced.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The system shall protect private information in accordance with the organizationís information policy.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The system shall be built such that it is as secure as possible from malicious interference.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The system shall use the organizationís standard virus protection software to help monitor threats to itself.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The Disputes application shall ensure that only users with the appropriate access level are allowed to initiate dispute requests.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Only users with Update access must be able to initiate dispute requests.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The Disputes application shall ensure that only authorized users are allowed to logon to the application.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"All authorized users  once authenticated must be able to logon to the Disputes System\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The Disputes System shall generate a User Report that details any new users that have been added and any changes to an existing userís access level.  The report will be reviewed for auditing purposes.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"All additions of new users and modifications to userís access shall be recorded on the User Report.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"If a user account is revoked  it can only be re-instantiated by the System Administrator.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Only the System Administrator shall be able to reset a revoked user logon account\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The Disputes application shall manipulate and view business data. Users will not have direct access to data files or databases.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"All updates to data files or database must be initiated from the Disputes System.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Only adjusters can request recycled parts audit reports. No users without an adjuster role shall request recycled parts audits.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Only adjusters with a supervisor role can update preferred repair facility ratings. Users without the supervisor role cannot access the repair facility ratings.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Only collision estimators shall search for recycled parts.  Users without the collision estimator role shall not have access to the search for recycled parts.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Only valid data shall be entered into the system.  No invalid data shall be entered into the system.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"One insurance company shall not be able to view another insurance companys claim data.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The product shall free of computer viruses.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The system shall prevent malicious attacks including denial of service.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The product shall ensure that only company employees or external users with company-approved user IDs may have product access.  100% of all logons shall be by either company employees or external users with company-approved user IDs.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Only managers are able to perform search queries for reservations by user.  100% of all search queries for reservations by user shall be from logons of only management logons.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The product shall ensure that the databaseís data corresponds to the data entered by the user.  Each conference room reservation in the system will contain the same information as was entered by the user.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The product shall prevent the input of malicious data.  The product and/or adjacent hardware/software systemsí data shall remain 100% uncorrupted each time malicious data is input into the product.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The product shall have the ability to receive automatic software updates as new threats emerge.  100% of customers will be able to receive automatic software updates transmitted to the installed product.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The System shall maintain consistent security.  Any changes in user passwords and/or permissions should take affect in all parts of the System.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The System shall granularly control access rights to all major subsystems.  It should be possible to assign access rights to all subsystems independently.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"All credit card information will be secured on the server  and only accessible by authorized Izogn administrators.  Information will be encrypted in the database.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Only registered customers can purchase streaming movies.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"A customer can only stream a movie if they purchased it  and are within the 2 day viewing period.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Only authorized personnel can access customer records in the database.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Only administrators can activate a pre-paid card.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Only authorized personnel can access sales information.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The website shall prevent its data from being corrupted when incorrect data is entered in.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The website shall protect itself from intentional abuse  and notify the administrator at all occurrences.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Data integrity scripts will be run on a weekly basis to verify the integrity of the database.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The website shall make its user aware of its information practices before collection data from them via a Privacy Policy  accessible on all pages of the website.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The system shall notify customers of changes to its information policy.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The system shall reveal private information only in compliance with the organizationís information policy.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"All customer information will be stored on a secure database  accessible only to authorized personnel.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"All credit card information will be encrypted in the database.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The streaming server will have anti-virus software to prevent infection of malicious viruses.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The product shall provide players no access to information that might reveal the identity of another player\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"A session that has been inactive for more than ten minutes is terminated \"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"- Disable accounts after 90 days of inactivity\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"- Passwords can be changed by the associated user only once in a 2-day period\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"- Security decisions are not made based on user-supplied file names and paths.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"- System does not accept file names or file paths from calling functions.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"- Users are re-authenticated before changing passwords\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Measures are in place to protect information from being accidentally overwritten.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"(1) Ensure the confidentiality, integrity, and availability of all electronic protected health information the covered entity creates, receives, maintains, or transmits.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"(2) Protect against any reasonably anticipated threats or hazards to the security or integrity of such information.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"(2) Use appropriate safeguards to prevent use or disclosure of the information other than as provided for by the data use agreement;\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"(3) Protect against any reasonably anticipated uses or disclosures of such information that are not permitted or required under subpart E of this part.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"(7) A description of the process in place to ensure that no amounts higher than 100 percent of FFP will be claimed by the State for reimbursement of expenditures for State payments to Medicaid eligible providers for the certified EHR technology incentive payment program and a methodology for verifying such information.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"(8) A description of the process in place for ensuring improvements in health outcomes, clinical quality, or efficiency resulting from the adoption of certified EHR technology by recipients of Medicaid incentive payments and a methodology for verifying such information.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"(9) A description of the process in place for ensuring that any certified EHR technology used as the basis for a payment incentive to Medicaid providers is compatible with State or Federal administrative management systems, including the MMIS or other automated claims processing system or information retrieval system and a methodology for verifying such information.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"(A) Implement administrative, physical, and technical safeguards that reasonably and appropriately protect the confidentiality, integrity, and availability of the electronic protected health information that it creates, receives, maintains, or transmits on behalf of the covered entity as required by this subpart;\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"(B) No protected health information is to be removed from the covered entity by the researcher in the course of the review; and\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"(B) The right to receive confidential communications of protected health information as provided by section 164.522(b), as applicable;\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"(B) Use appropriate safeguards to prevent use or disclosure of the information other than as provided for by its contract;\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"(i) Except with respect to disclosures under section 164.510, verify the identity of a person requesting protected health information and the authority of any such person to have access to protected health information under this subpart, if the identity or any such authority of such person is not known to the covered entity; and\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"(i) Implement administrative, physical, and technical safeguards that reasonably and appropriately protect the confidentiality, integrity, and availability of the electronic protected health  information that it creates, receives, maintains, or transmits on behalf of the group health plan;\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"(ii) Customer service, including the provision of data analyses for policy holders, plan sponsors, or other customers, provided that protected health information is not disclosed to such policy holder, plan sponsor, or customer.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"(ii) Ensure that the adequate separation required by section 164.504(f)(2)(iii) is supported by reasonable and appropriate security measures;\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"100% of all logons shall be by either company employees or external users with company-approved user IDs.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"A covered entity must have in place appropriate administrative, technical, and physical safeguards to protect the privacy of protected health information.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"A patient has authenticated him or herself in the iTrust Medical Records system (UC2).\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"A security risk assessment is a requirement for compliance with HIPAA and Meaningful Use.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"A software tester has a login and password.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"After installation or upgrade consider removing(or ensuring no access to) to the following scripts, which are not needed for general OpenEMR use:\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Consider a firewall that only allows port 443 (https) traffic to the server.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Consider adding a specific sqlconfDOTphp override settings when using the patient portal (especially the onsite portal), which can be used to use a separate mysql user (and possibly database).\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Do not allow direct web access to the following directories\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Enforce password expiration, which can be set in Administration->Globals->Security.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Enforce strong or unique passwords, which can be set in Administration->Globals->Security\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Ensure the root password is set to something.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"If not using portal and want to allow users to access over the internet, then consider using client-sided certificates to only allow users access to site\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"If want to give access to the patient portals over the internet, then consider using a client-sided certificate to only allow users access to the main OpenEMR login page\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"If you are using XAMPP, note that this software is extremely unsecure after installation and you need to secure it (See the XAMPP security instructions to secure your XAMPP installation.)\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"On server, consider only opening port 443 (https).\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Only allow https (ie turn off http)\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Only allowing the IP address of the third party portal to access the SOAP APIs (for patient access)\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Passwords should be held to a standard and be required to change at intervals.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The third party portal could also set up a ssh tunnel method, so the local OpenEMR instance is not open to the web (and possibly not even require an IP address).\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The third party portal could set up a certificate to identify their server as an option to further secure.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Ability for an administrator to delegate authority, by user group, to reset password\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Ability for an administrator to delegate authority, by user group, to restore system access of locked out user\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Administration interfaces require strong authentication and authorization.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"After installation is finished, then secure files (this should be done as root user):\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"After three failed attempts with a userid in a given session, disallow attempts to log in via IP address for 15 minutes (see comments in the source code).\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"All additions of new users and modifications to user's access shall be recorded on the User Report.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"All authorized users  once authenticated must be able to logon to the Disputes System\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"All credit card information will be encrypted in the database.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"All customer information will be stored on a secure database  accessible only to authorized personnel.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"All input parameters are validated (including form fields, query strings, cookies, and HTTP headers).\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"All system and user accounts are identified.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"All updates to data files or database must be initiated from the Disputes System.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"All user sessions involving financial transactions or sensitive data are encrypted using SSL/HTTPS.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"An authenticated session ends when the user logs out or closes the iTrust application.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"An example of reflexive testing is when a confirmatory test is automatically ordered after a screening test is positive.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"And remember that OpenEMR's username and passwords are case sensitive.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Any changes in user passwords possibly and possibly permissions should take affect in all parts of the System.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Apply appropriate sanctions against workforce members who fail to comply with the security policies and procedures of the covered entity.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Attempts to access resources using parent paths are blocked.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Conduct an accurate and thorough assessment of the potential risks and vulnerabilities to the confidentiality, integrity, and availability of electronic protected health information held by the covered entity.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Conduct or review a security risk analysis in accordance with the requirements under 45 CFR 164.308(a)(1) and implement security updates as necessary and correct identified security deficiencies as part of its risk management process.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Conduct or review a security risk analysis in accordance with the requirements under 45 CFR 164.308(a)(1) and implement security updates as necessary and correct identified security deficiencies as part of its risk management process.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Configuration credentials and authentication tokens are not held in plain text in configuration files.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Configuration stores are secured from unauthorized access and tampering.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Cookies containing WCS login information about a user will not be stored on a user's computer.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Cookies with sensitive data (e.g. authentication cookies) are encrypted.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Cryptographic algorithm and key size for the System's data encryption requirements is AES 256 bit or stronger.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Customer will no longer be able to access the website.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Data collection devices (e.g. â€“ handheld devices, etc.) synchronize with the System securely using authentication, authorization and encryption mechanisms.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Data integrity scripts will be run on a weekly basis to verify the integrity of the database.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Data is validated for type, length, format, and range.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Data Recipient shall use appropriate safeguards to protect the IIHI from misuse or inappropriate disclosure and to prevent any use or disclosure of the IIHI other than as provided in this DUA or as otherwise required by law or regulation.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Data validation is consistent across the System.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Database keys, connections, passwords, or other secrets are not stored in plain text.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Database or file encryption for protection of sensitive data fields while the data is at rest (i.e., stored data) is provided.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Database will mark the account as inactive  and user will not be able to log into the website\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Dr Susan Poslusny and Dr Julie Donalek are the only people who shall have access to students' personal information.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Each conference room reservation in the system will contain the same information as was entered by the user.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Each team owner will need to use a username and password in order to be authenticated to reach the team owner's page\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Electronic sessions must terminate after ten minutes of inactivity.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Encrypted data delivered over the Internet is transmitted via open protocols (e.g., SSL, XML encryption)\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Encryption capability for certain data transmissions that require security protection.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Encryption keys are secured.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Ensure that authentication is reset after a period of inactivity that exceeds ten minutes.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Every user of the system shall be authenticated and authorized.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Fit Criterion: Dr Susan Poslusny and Karen Sysol are the only people who shall have access to the final class section scheduling for the system that interfaces with CampusConnect.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"For e-prescribing and for CCHIT certification, EHR systems shall have the ability to support two-factor authentication.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"For systems that provide access to PHI through a web browser interface (i.e. HTML over HTTP) shall include the capability to encrypt the data communicated over the network via SSL (HTML over HTTPS).\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Furthermore, if CMS determines that the risk of harm requires notification of affected individual persons of the  security breach possibly and possibly other remedies, the User agrees to carry out these remedies without cost to CMS.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Identify and respond to suspected or known security incidents; mitigate, to the extent practicable, harmful effects of security incidents that are known to the covered entity; and document security incidents and their outcomes.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Identify security standard (SSL or FIPS encryption).\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"If a user account is revoked  it can only be re-instantiated by the System Administrator.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"If Structured Query Language (SQL) authentication is used (e.g., communication between the application server and the database server) credentials are secured in storage and over the wire via Secure Socket Layer (SSL) or IP Security (IPSec).\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"If the system includes hardware, the system shall include documentation that covers the expected physical environment necessary for proper secure and reliable operation of the system including: electrical, HVAC, sterilization, and work area.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"If this is your first time logging in to OpenEMR it is recommended that you change your password to something more secure.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Implement a mechanism to encrypt and decrypt electronic protected health information.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Implement a mechanism to encrypt electronic protected health information whenever deemed appropriate.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Implement a security awareness and training program for all members of its workforce (including management).\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Implement electronic mechanisms to corroborate that electronic protected health information has not been altered or destroyed in an unauthorized manner.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Implement electronic procedures that terminate an electronic session after a predetermined time of inactivity.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Implement physical safeguards for all workstations that access electronic protected health information, to restrict access to authorized users.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Implement policies and procedures that govern the receipt and removal of hardware and electronic media that contain electronic protected health information into and out of a facility, and the movement of these items within the facility.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Implement policies and procedures that specify the proper functions to be performed, the manner in which those functions are to be performed, and the physical attributes of the surroundings of a specific workstation or class of workstation that can access electronic protected health information.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Implement policies and procedures to address security incidents.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Implement policies and procedures to address the final disposition of electronic protected health information, possibly and possibly the hardware or electronic media on which it is stored.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Implement policies and procedures to document repairs and modifications to the physical components of a facility which are related to security (for example, hardware, walls, doors, and locks).\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Implement policies and procedures to prevent, detect, contain, and correct security violations.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Implement policies and procedures to protect electronic protected health information from improper alteration or destruction.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Implement policies and procedures to safeguard the facility and the equipment therein from unauthorized  physical access, tampering, and theft.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Implement procedures for removal of electronic protected health infoImplement policies and procedures to address the final disposition of electronic protected health information, possibly and possibly the hardware or electronic media on which it is stored.rmation from electronic media before the media are made available for re-use.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Implement procedures to regularly review records of information system activity, such as audit logs, access reports, and security incident tracking reports.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Implement procedures to verify that a person or entity seeking access to electronic protected health information is the one claimed.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Implement security measures sufficient to reduce risks and vulnerabilities to a reasonable and appropriate level to comply with section 164.306(a).\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Implement security measures to ensure that electronically transmitted electronic protected health information is not improperly modified without detection until disposed of.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Implement technical security measures to guard against unauthorized access to electronic protected health information that is being transmitted over an electronic communications network.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"In the event that a system does not support pre-login capabilities, the system shall display the banner immediately following authorization.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Information will be encrypted in the database.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"It should be possible to assign access rights to all subsystems independently.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Key management procedure to secure and manage the encryption keys is defined.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Login to OpenEMR using your assigned username and password.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Measures are in place to detect unauthorized changes to information\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Measures are in place to prevent the upload of unauthorized files (e.g., executable files).\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Measures are in place to prevent, detect and log unauthorized attempts to access sensitive or confidential data.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Measures are in place to prevent, detect and log unauthorized attempts to access the System.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"MIDs should be considered private, sensitive information.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"No invalid data shall be entered into the system.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Note that the OpenEMR login is case-sensitive.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"On this page is displayed recommendations and warnings regarding directories Apache should NOT allow the outside world to see, as these directories contain Protected Health Information.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Only authorized user shall access the RFS system.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Only authorized users shall have access to clinical site information.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Only authorized users shall have access to students' personal information.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Only authorized users shall have access to the portion of the system that interfaces with CampusConnect.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Only managers are able to perform search queries for reservations by user.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Only registered realtors shall be able to access the system.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Only the System Administrator shall be able to reset a revoked user logon account\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"ONLY users that exist in the UAS system  with the RFS system assigned to their role(s)  shall access the RFS system and use it.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Only users with Update access must be able to initiate dispute requests.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Only valid data shall be entered into the system.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Periodic security updates.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Physical security protects the providers, the patients, clinic assets, and the physical manifestation of the business, financial, and health information.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Platform-level cryptography is used with no custom implementations.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Private and sensitive data (for example, passwords) are not logged\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Procedures for creating, changing, and safeguarding passwords.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Procedures for guarding against, detecting, and reporting malicious software.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Procedures for monitoring log-in attempts and reporting discrepancies.\"})\n",
    "#training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Program Administrators or Nursing Staff Members are the only people who shall have access to clinical site details.\"\",\"\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Protect electronic health information created or maintained by the certified EHR technology through the implementation of appropriate technical capabilities.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Protect electronic health information created or maintained by the certified EHR technology through the implementation of appropriate technical capabilities.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Protection mechanisms are in place for sensitive data that is sent over the network.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Remote administration channels are secured (e.g., SSL, VPN)\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Remove MID from being displayed on all pages and URLs.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Secrets are stored securely using a one-way hash.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Security decisions do not rely on HTTP header information.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Security measures implemented to comply with standards and implementation specifications adopted under section 164.105 and this subpart must be reviewed and modified as needed to continue provision of reasonable and appropriate protection of electronic protected health information as described at section 164.316.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Sensitive data and secrets are not incorporated in  code.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Sensitive data is not logged in clear text by the System.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Sensitive data is not passed in query strings or form fields.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Sensitive data is not stored in persistent cookies.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Sensitive data is not transmitted using insecure protocols, such as FTP, telnet, tftp etc., unless tunneled through an authenticated encrypted connection (e.g. VPN).\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Session identifiers are not passed in query strings.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Session lifetime is limited to a pre-specified and configurable duration.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Session state is protected from unauthorized access.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"SSL is used to protect authentication cookies.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"System allows addition of user-defined messages to logon screen\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"System allows secure password resets in case passwords are forgotten.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"System allows users to re-authenticate and remotely log out of an active user session before logging in at another location.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"System allows viewing of list of Users logged on to System in real-time\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"System authenticates standalone devices before access is allowed to protected resources.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"System authenticates the user before any access is allowed to protected resources (e.g., Protected Health Information)\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"System avoids un-trusted input of file name and file paths.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"System code does not rely on internal system generated error handling.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"System does not rely on client-side validation.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"System does not use parent paths when data within the System is being accessed.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"System encrypts passwords before being stored or transmitted.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"System encrypts sensitive data transmitted between clients and servers using Secure Socket Layer (SSL) Certificates, Transport Layer Security (TLS), or by other means\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"System exception handling minimizes information disclosure in case of an exception.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"System has the ability to disallow more than one active session per sign-on identification.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"System insures that input validation is applied whenever input is received through user or external data interfaces.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"System insures that minimum error information is returned in the event of authentication failure.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"System must support password complexity that meets the following requirements:\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"System performs secure and seamless logon for all third party integrated systems.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"System provides a web (HTTPS) interface and provides an SSL configuration mechanism.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"System provides an automatic timeout if the session is idle for a pre-specified and configurable duration.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"System provides integrated security managed in a central accounts database\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"System provides reminder alerts to users to reset passwords.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"System provides secure information delivery over the Internet via encryption by using triple-DES (Data Encryption Standard) or the Advanced Encryption Standard (AES)\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"System provides the following password change rules for user accounts:\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"System provides the option for multi-factor authentication for users with higher security access.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"System restricts users from directly accessing the database.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"System returns generic error messages to the client, to avoid disclosure of sensitive information.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"System shall let existing customers log into the website with their email address and password in under 5 seconds.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"System uses least-privileged accounts.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"System warns the user before the timeout and prompts the user to re-enter their password.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Temporary objects are removed from the system, database connections are closed, and memory is released.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The administrator has authenticated himself or herself in the iTrust Medical Records system (UC3).\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The County standard for users is 90 days.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The Disputes application shall ensure that only authorized users are allowed to logon to the application.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The Disputes application shall ensure that only users with the appropriate access level are allowed to initiate dispute requests.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The Disputes application shall manipulate and view business data.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The Disputes System shall generate a User Report that details any new users that have been added and any changes to an existing user's access level.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The equipment storing PHI and BI should be subject to: Physical access restrictions: Place the server in a protected, locked area and limit access.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The first screen gives some general instructions and ensures file and directory permissions are set up correctly.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The HCP has authenticated himself or herself in the iTrust Medical Records system (UC2),(UC3).\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The HCP has authenticated himself or herself in the iTrust Medical Records system (UC3).\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The HCP has authenticated himself or herself in the iTrust Medical Records system(UC3).\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The Healthcare Information and Management Systems Society (HIMSS) provides a Risk Assessment Toolkit for conducting a risk assessment and developing a compliant security plan.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The inactivity timeout shall be configurable.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The iTrust user (LCHP) has been authenticated in the iTrust Medical Records system (UC3).\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The iTrust user (LHCP or patient) has been authenticated in the iTrust Medical Records system (UC3).\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The iTrust user (patient or HCP) has been authenticated in the iTrust Medical Records system (UC3).\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The iTrust user (patient) has authenticated himself or herself in the iTrust Medical Records system (UC3).\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The iTrust user (patient) has been authenticated in the iTrust Medical Records system (UC3).\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The iTrust user (patient, LHCP) or administrator has authenticated himself or herself in the iTrust Medical Records system (UC3),(UC2).\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The iTrust user has authenticated himself or herself in the iTrust Medical Records system (UC3).\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The iTrust user has been authenticated in the iTrust Medical Records system (UC3).\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The leads washing functionality will verify the users authentication and rights to make parameter updates.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The password is not displayed as clear text (Password Masking)\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The patient has authenticated himself or herself in the iTrust Medical Records system (UC3).\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The patient has authenticated himself or herself in the iTrust Medical Records system(UC3).\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The product possibly and possibly adjacent hardware or software systems' data shall remain 100% uncorrupted each time malicious data is input into the product.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The product shall ensure that it can only be accessed by authorized users.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The product shall ensure that only company employees or external users with company-approved user IDs may have product access.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The product shall ensure that the database's data corresponds to the data entered by the user.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The product shall free of computer viruses.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The product shall have security.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The product shall information Security.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The product shall prevent its data from incorrect data being introduced.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The product shall prevent the input of malicious data.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The product shall provide authentication  and authorization.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The product will be able to distinguish between authorized and unauthorized users in all access attempts\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The Proposer shall provide adequate firewall protection in order to secure Personal Data and other Confidential Information users of the EHR from unauthorized access by third parties.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The Proposer shall provide the facility required to host the computing and network environment, including appropriate physical security, required third-party software, and 24 by 7 staff support and monitoring environmental conditions (e.g. HVAC, port, fire detection, suppression, moisture, humidity and temperature.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The Proposer shall review security notifications and alerts relevant to the hosting platform (e.g., notifications of bugs, attacks, patches), and apply as appropriate to maintain the highest level of defense.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The purpose of the criteria is to provide the ability to remove a user's privileges, but maintain a history of the user in the system.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The report will be reviewed for auditing purposes.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The safeguards shall provide a level  and scope of security that is not less than the level and scope of security requirements established by the Office  of Management and Budget (OMB) in OMB Circular No A-130, Appendix III--Security of Federal Automated  Information Systems (http://www.whitehouse.gov/omb/circulars/a130/a130.html) as well as Federal  Information Processing Standard 200 entitled Minimum Security Requirements for Federal Information and  Information Systems (http://csrc.nist.gov/publications/fips/fips200/FIPS-200-final-march.pdf); and, Special Publication 800-53 Recommended Security Controls for Federal Information Systems (http://csrc.nist.gov/ publications/nistpubs/800-53-Rev2/sp800-53-rev2-final.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The software tester authenticates himself or herself in the iTrust Medical Records system (UC2).\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The software used to install and update the system, independent of the mode or method of conveyance, shall be certified free of malevolent software (malware).\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The streaming server will have anti-virus software to prevent infection of malicious viruses.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The System design assumes that user input is malicious.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The system must be able to associate permissions with a user using one or more of the following access controls: 1) user-based (access rights assigned to each user); 2) role-based (users are grouped and access rights assigned to these groups); or 3) context-based (role-based with additional access rights assigned or restricted based on the context of the transaction such as time-of-day, workstation-location, emergency-mode, etc.)  The system shall support removal of a user's privileges without deleting the user from the system.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The System provides error-handling processes.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The system shall authenticate the user before any access to Protected Resources (e.g. PHI) is allowed, including when not connected to a network e.g. mobile devices.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The system shall automatically notify users and force them to change passwords on a pre-defined frequency.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The system shall be built such that it is as secure as possible from malicious interference.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The system shall enable multiple simultaneous users, each with his or her own exclusive authentication.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The system shall enforce a limit of (configurable) consecutive invalid access attempts by a user.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The system shall enforce the most restrictive set of rights or privileges or accesses needed by users or groups (e.g. System Administration, Clerical, Nurse, Doctor, etc.), or processes acting on behalf of users, for the performance of specified tasks.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The System shall granularly control access rights to all major subsystems.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The system shall have basic data integrity checking to reduce the possibility of incorrect or invalid data being introduced.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The system shall have the ability for security module to be maintained by an in-house System Administrator\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The system shall have the ability to assign application access rights across entire suite of applications at a single point of entry.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The system shall have the ability to display the last date and time the user logged onto the system at the time of logon.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The system shall have the ability to support 128-bit SSL encryption, or higher, between the client browser and the application tier for any or all modules or sub-modules at organization discretion.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The system shall have the ability to support biometrics and biometrics plus passwords (e.g., fingerprint scan and fingerprint scan plus password).\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The system shall have the ability to support web based client access or other internet based client access technologies, with appropriate security access controls\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The system shall include documentation available to the customer stating whether or not there are known issues or conflicts with security services  in at least the following service areas:  antivirus, intrusion detection, malware eradication, host-based firewall and the resolution of that conflict (e.g. most  systems should note that full virus scanning should be done outside of peak usage times and should exclude the databases.)\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The system shall include documentation available to the customer that provides guidelines for configuration and use of the security controls necessary to support secure and reliable operation of the system, including but not limited to: creation, modification, and deactivation of user accounts, management of roles, reset of passwords, configuration of password constraints, and audit logs.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The system shall include documentation that itemizes the services (e.g. PHP, web services) and network protocols/ports (e.g. HL-7,  HTTP, FTP)  that are necessary for proper operation and servicing of the system, including justification of the need for that service and protocol.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The System shall maintain consistent security.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The system shall only be accessed by authorized Corporate Community Grants (CCR) employees.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The system shall prevent malicious attacks including denial of service.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The system shall protect against further, possibly malicious, user authentication attempts using an appropriate mechanism (e.g. locks the account or node until released by an administrator, locks the account or node for a configurable time period, or delays the next login prompt according to a  configurable delay algorithm).\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The system shall provide ability to use a single user sign-on for all modules with security configured for each module.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The system shall provide automatic logout of users when there has been no activity for a pre-defined period, maintaining transaction integrity\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The system shall provide expiration dates for passwords.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The system shall provide lock-out capability after a pre-defined number of unsuccessful user sign-on attempts.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The system shall provide only limited feedback information to the user during the authentication.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The system shall support ensuring the authenticity of remote nodes (mutual node authentication) when communicating Protected Health Information (PHI) over the Internet or other known open networks using an open protocol (e.g. TLS, SSL, IPSec, XML sig, S or MIME).\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The system shall support protection of confidentiality of all Protected Health Information (PHI) delivered over the Internet or other known open networks via encryption using triple-DES (3DES) or the Advanced Encryption Standard (AES) and an open protocol such as TLS, SSL, IPSec, XML encryptions, or S or MIME or their successors.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The system shall use the organization's standard virus protection software to help monitor threats to itself.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The system upon detection of inactivity of an interactive session shall prevent further viewing and access to the system by that session by terminating the session, or by initiating a session lock that remains in effect until the user reestablishes access using appropriate identification and authentication procedures.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The system, prior to access to any PHI, shall display a configurable warning or login banner (e.g. 'The system should only be accessed by authorized users').\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The system, when storing PHI on any device intended to be portable or removable (e.g. thumb-drives, CD-ROM, PDA, Notebook), shall support use of a standards based encrypted format using triple-DES (3DES), or the Advanced Encryption Standard (AES), or their successors.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The time for passwords to be changed is predefined as per user's role and access level.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The User agrees that,  within the User organization and the organizations of its agents, access to the data covered by this Agreement  shall be limited to the minimum amount of data and minimum number of individuals necessary to achieve the purpose stated in this section (i.e., individual's access to the data will be on a need-to-know basis).\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The User agrees to establish appropriate administrative, technical, and physical safeguards to protect the  confidentiality of the data and to prevent unauthorized use or access to it.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The user may try three times.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The validation approach is to constrain, reject, and then sanitize input.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The WCS system shall not allow automatic logins by any user.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The web server always asserts a character set: a locale and a country code, such as en_US.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The Web-enabled portions of the application shall meet state-of-the-art Internet standards for graphics and design and for speed, reliability, and security for dynamic content and user interaction.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The website shall prevent its data from being corrupted when incorrect data is entered in.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"The website shall protect itself from intentional abuse  and notify the administrator at all occurrences.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"This information may be used by the healthcare facility to properly configure their network defenses (firewalls and routers).\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"To begin using OpenEMR, navigate to your server's installation directory and log in using your assigned username and password.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"To begin, log in with your designated username and password.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Traditional security matters involve locks, points of entry, lighting and cameras, fire precautions, power outage procedures, and water protection, and disaster recovery.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"User accounts and service accounts used for configuration management have only the minimum privileges required for the task.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Users must be authenticated using their IBM intranet user name and password.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Users will not have direct access to data files or databases.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Vendor may self-certify compliance with this standard through procedures that make use of commercial malware scanning software.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"Web sites are partitioned into un-restricted and restricted areas using separate folders\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"When answer to the security question is typed in, the answer should not appear on the screen (similar to how a password normally appears) and the answer should be confirmed (by the patient or personal health representative) before it is saved.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"When passwords are used, the system shall allow an authenticated user to change their password consistent with password strength rules (SC 03.02).\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"When passwords are used, the system shall prevent the reuse of passwords previously used within a specific (configurable) timeframe (i.e., within the last X days, etc. - e.g. last 180 days), or shall prevent the reuse of a certain (configurable) number of the most recently used passwords (e.g. last 5 passwords).\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"When passwords are used, the system shall provide an administrative function that resets passwords.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"When passwords are used, the system shall support password strength rules that allow for minimum number of characters, and inclusion of alpha-numeric complexity.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"When passwords are used, the system shall use either standards-based encryption, e.g., 3DES, AES, or standards-based hashing, e.g., SHA1 to store or transport passwords.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"When passwords are used, user accounts that have been reset by an administrator shall require the user to change the password at next successful logon.\"})\n",
    "training_data.append({\"class\":\"2\", \"class_name\":\"Security\", \"sentence\":\"You will then be required to login again with your new password.\"})\n",
    "\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The system shall able to operate within a business office environment typical of the Nursing Department at DePaul University.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The system shall be used within the specifications defined by the computers used by the Program Administrators/Nursing Staff Members.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The system shall operate within the Windows XP Professional operating system.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The system shall interface with CampusConnectís central server\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The system shall interface with the faculty central server.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The system shall interface with the student master server.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"We must be able to interface with any HTML browser.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The Disputes application shall be available 24x7x365  with the exception of the following scheduled maintenance windows.  Monday ñ Saturday 3:00am to 4:00am EST Sunday 1:00amd to 5:00am EST\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The Disputes application shall interface with the Statement Database. The Statement Database provides the transaction details to the Disputes System. All transaction details must be obtained from the Statement Database\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The Disputes application shall interface with the Cardmember Information Database.  The Cardmember Information Database provides detailed information with regard to a cardmember.  All cardmember detail information must be obtained from the Cardmember Information Database.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The Disputes applications shall interface with the Merchant Information Database.  The Merchant Information Database provides detailed information with regard to the merchant. All merchant detail information shall be obtained from the Merchant Information Database.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The Disputes application shall interface with the Letters application.  This will allow the Disputes application to request letters as part of the dispute initiation and dispute follow up process. All letter requests must be sent to the Print Letter Utility application.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The Disputes application shall interface with the Cardmember Posting and Billing System.  This will allow the Disputes application to request adjustments to cardmember and merchantís accounts. All adjustment requests must be sent to the Cardmember Posting and Billing System.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The product must support Internet Explorer 5.5 and above.'\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The product must support Mozilla Firefox 1.0 and above.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The product shall interface with the Choice Parts System.  This provides the feed of recycled parts data.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The product shall run on the existing hardware for all environments.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The product shall adhere to the corporate Architecture guidelines\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"For estimators  the product shall be able to be operated in a repair facility during dirty and noisy conditions.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The product shall interface with the ChoiceParts system once per day at approximately 1:00 AM.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The maintenance of the product will fall within the established corporate maintenance windows.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The product shall be supported using the corporate support center.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"Product installations and upgrades shall be handled by the middleware technology team within established launch time frames.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The product must work with most database management systems (DBMS) on the market  whether the DBMS is colocated with the product on the same machine or is located on a different machine on the computer network.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The product will function alongside server software on any operating system where the Java runtime can be installed.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The product will require collaboration with a database management system (DBMS).  The DBMS may be located on the same machine as the product or on a separate machine residing on the same computer network.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The product must make use of web/application server technology.  Open source examples include Apache web server  Tomcat  and the JBoss application server.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"A database management system such as Oracle  DB2  MySql or HSQL will need to be integrated with the product.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The product shall be used in office environments.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The product must be able to interface with any HTML browser.  The product shall transmit data between the user and the product without problems. The product shall display HTML properly in 80% of all HTML browsers tested (minimum is to test 8 browsers).\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The product must be able to interface with various database management systems. The product shall communicate successfully with the database management system on 100% of all transactions.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The product must be able to interface with various email servers. The product shall be able to send email.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The product shall be available for distribution via the Internet as a binary or ZIP file.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The product shall be available for distribution as a packaged CD.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The product will be available for licensing as a one-server  two-five servers  or five-or-more servers license.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The product is expected to integrate with multiple database management systems.  The product will operate with Oracle  SQL Server  DB2  MySQL  HSQL  and MS Access.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The product shall be able to be installed in any operating environment within 2 days.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The product must be developed with the J2SE/J2EE programming language libraries\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The System shall utilize currently owned computer equipment.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The System shall be able to automatically backup business data and restore them on demand.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The System shall not require additional third party licenses resulting in royalty fees.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"All aspects of the website shall be accessed by a web browser over the Internet.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The same hardware supporting the current website will be used to support the new system.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"We must be able to interface with the following browsers: IE 5.X  6.0  Netscape 6.x  7.x  8.x and Firefox 1.0.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"Website must interface with the CyberSource API to process credit card transactions.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"Website must be fully operational with MSN TV2.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"Movies shall be encoded and streamed via Windows Media Player 9.0.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"Movies shall be encoded for 56k and 256k streams.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The product shall be able to be installed in the specified environment within 1 working day.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The website should use Flash to provide animation on the home page.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The lead washing process will operate within the established physical web service structure.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The washing process must use a non binding  neutral data format  such as xml  when receiving and processing lead data.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"100% of the entities (mentioned above) available as of October month-end  will be loaded into the RFS system  prior to the budget start process.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"7 data files shall be received from the BDW and loaded into the RFS system within 3 hours one day prior to the start of budgeting process  every year.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"98% of the user workstations shall meet the minimum hardware requirements prior to implementation.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"A database management system such as Oracle  DB2  MySql or HSQL will need to be integrated with the product.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"A primary database will be used to store cached data for active users.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"A secondary database will be used to store historical statistics and data.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"All adjustment requests must be sent to the Cardmember Posting and Billing System.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"All aspects of the website shall be accessed by a web browser over the Internet.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"All cardmember detail information must be obtained from the Cardmember Information Database.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"All letter requests must be sent to the Print Letter Utility application.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"All merchant detail information shall be obtained from the Merchant Information Database.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"All transaction details must be obtained from the Statement Database\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"Allow access to the EMH solution over the Internet and provide secure and confidential storage of all information transmitted to and from the LA DHS.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"Brio will provide buttons in the user interface that produce or export reports in xls or .123 file formats.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"First install and configure MySQL, Apache (or another PHP-capable webserver), and PHP5.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"For estimators  the product shall be able to be operated in a repair facility during dirty and noisy conditions.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"If using CentOS or Redhat, then ensure when configuring MySQL for OpenEMR, STRICT_MODE must be disabled.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"Irregular activities are identified based on County departments' rules and regulations.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"Minimum Hard Disk Space:  400MB.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"Movies shall be encoded and streamed via Windows Media Player 9.0.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"Movies shall be encoded for 56k and 256k streams.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"Open source examples include Apache web server  Tomcat  and the JBoss application server.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"Product installations and upgrades shall be handled by the middleware technology team within established launch time frames.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"Queries must be made to external databases for data refresh once a week.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"System integrates with Microsoft Active Directory for authentication and has the capability of notifying the end user of near domain account password expiration date as well as the ability to reset the password through the system's user interface\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"System provides configuration, user accounts, roles and privileges reports.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"System restricts access to financial transactions and other sensitive data by authorized users outside the County Intranet to Read Only mode.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The application shall run on a web application server.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The Brio portion of the WCS system must be able to export files in spreadsheet form (Microsoft Excel and Lotus 1-2-3 formats).\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The Cardmember Information Database provides detailed information with regard to a cardmember.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The DBMS may be located on the same machine as the product or on a separate machine residing on the same computer network.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The Disputes application shall interface with the Cardmember Information Database.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The Disputes application shall interface with the Cardmember Posting and Billing System.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The Disputes application shall interface with the Letters application.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The Disputes application shall interface with the Statement Database.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The Disputes applications shall interface with the Merchant Information Database.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The Disputes System must be accessible by both internal and external users.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The equipment storing PHI and BI should be subject to: Physical access restrictions: Place the server in a protected, locked area and limit access.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The lead washing process will operate within the established physical web service structure.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The maintenance of the product will fall within the established corporate maintenance windows.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The Merchant Information Database provides detailed information with regard to the merchant.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The product is expected to integrate with multiple database management systems.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The product is expected to run on Windows CE and Palm operating systems.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The product must be able to interface with any HTML browser.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The product must be able to interface with various database management systems.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The product must make use of web or application server technology.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The product must support Internet Explorer 5.5 and above.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The product must support Mozilla Firefox 1.0 and above.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The product must work with most database management systems (DBMS) on the market  whether the DBMS is colocated with the product on the same machine or is located on a different machine on the computer network.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The product shall be able to be installed in any operating environment within 2 days.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The product shall be able to be installed in the specified environment within 1 working day.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The product shall be available for distribution as a packaged CD.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The product shall be available for distribution via the Internet as a binary or ZIP file.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The product shall be platform independent.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The product shall be supported using the corporate support center.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The product shall be used in office environments.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The product shall communicate successfully with the database management system on 100% of all transactions.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The product shall display HTML properly in 80% of all HTML browsers tested (minimum is to test 8 browsers).\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The product shall enable access to any type of development environment and platform.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The product shall exchange messages between CSR and EAM on a real time basis without any direct interfaces between two systems.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The product shall interface CSR and Datastream.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The product shall interface with the Choice Parts System.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The product shall interface with the ChoiceParts system once per day at approximately 1:00 AM.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The product shall operate with every platform.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The product shall run on the existing hardware for all environments.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The product shall transmit data between the user and the product without problems.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The product will function alongside server software on any operating system where the Java runtime can be installed.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The product will operate with Oracle  SQL Server  DB2  MySQL  HSQL  and MS Access.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The product will require collaboration with a database management system (DBMS).\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The proposed EHR System is able to manage the resource roles necessary to support the environment similar to LAC DHS.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The Proposer shall provide and manage all required infrastructure and network equipment within the data center, such as servers, routers, switches, load balancers and consoles.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The RFS system should be able to easily interface with the BDW environment  in order to load the data.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The same hardware supporting the current website will be used to support the new system.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The Statement Database provides the transaction details to the Disputes System.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The system shall able to operate within a business office environment typical of the Nursing Department at DePaul University.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The system shall accommodate separate instances of databases.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The system shall be used within the specifications defined by the computers used by the Program Administrators or Nursing Staff Members.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The system shall interface with CampusConnect's central server\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The system shall interface with the faculty central server.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The system shall interface with the student master server.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The system shall lock database records based on organization parameters (e.g., at row level, field level, or at the application level).\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The system shall operate within the Windows XP Professional operating system.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The system shall provide data export functionality that creates common export file format (e.g. comma delimited, tab delimited, space delimited, quotation delimited, etc.).\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The system shall provide data import functionality to receive standard format data from external parties.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The system shall provide the ability for on-line access by any site connected to the organization WAN.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The system shall provide the ability for remote access by authorized individuals (i.e. web based VPN access).\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The system shall provide the ability to maintain multiple operating environments for development, test, training and production.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The System shall utilize currently owned computer equipment.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The WCS shall be accessed by users through the Internet on an IBM computer running a Windows 2000 or above operating system.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The WCS system must run queries to the WCS databases once again (on a specified day) to update the available data available to CCR users.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The WCS system shall be run in a Windows environment.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"The WCS system shall have the ability to connect other designated WCS databases currently not included in the system.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"This can be done in the MySQL configuration (myDOTcnf) file.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"This provides the feed of recycled parts data.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"This will allow the Disputes application to request adjustments to cardmember and merchant's accounts.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"This will allow the Disputes application to request letters as part of the dispute initiation and dispute follow up process.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"Velkat staff will be available for phone support between 7am-7pm  M-F to provide technical support to the Izogn administrators.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"We must be able to interface with any HTML browser.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"We must be able to interface with the following browsers: IE 5x, IE 6x, Netscape 6, 7, 8, and Firefox 1.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"Website must be fully operational with MSN TV2.\"})\n",
    "training_data.append({\"class\":\"3\", \"class_name\":\"Operability\", \"sentence\":\"You can check the setting in an existing MySQL instance by executing SHOW VARIABLES; in the MySQL Command Line Client and checking the value of SQL_MODE.\"})\n",
    "\n",
    "print (\"%s sentences in training data\" % len(training_data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "words = []\n",
    "classes = []\n",
    "documents = []\n",
    "ignore_words = [\n",
    "    'i', 'me', 'my', 'myself', 'we', 'our', 'ours', 'ourselves', 'you', \n",
    "    \"you're\", \"you've\", \"you'll\", \"you'd\", 'your', 'yours', 'yourself', \n",
    "    'yourselves', 'he', 'him', 'his', 'himself', 'she', \"she's\", 'her', \n",
    "    'hers', 'herself', 'it', \"it's\", 'its', 'itself', 'they', 'them', \n",
    "    'their', 'theirs', 'themselves', 'what', 'which', 'who', 'whom', 'this', \n",
    "    'that', \"that'll\", 'these', 'those', 'am', 'is', 'are', 'was', 'were', \n",
    "    'be', 'been', 'being', 'have', 'has', 'had', 'having', 'do', 'does', \n",
    "    'did', 'doing', 'a', 'an', 'the', 'and', 'but', 'if', 'or', 'because', \n",
    "    'as', 'until', 'while', 'of', 'at', 'by', 'for', 'with', 'about', \n",
    "    'against', 'between', 'into', 'through', 'during', 'before', 'after', \n",
    "    'above', 'below', 'to', 'from', 'up', 'down', 'in', 'out', 'on', 'off', \n",
    "    'over', 'under', 'again', 'further', 'then', 'once', 'here', 'there', \n",
    "    'when', 'where', 'why', 'how', 'all', 'any', 'both', 'each', 'few', \n",
    "    'more', 'most', 'other', 'some', 'such', 'no', 'nor', 'not', 'only', \n",
    "    'own', 'same', 'so', 'than', 'too', 'very', 's', 't', 'can', 'will', \n",
    "    'just', 'don', \"don't\", 'should', \"should've\", 'now', 'd', 'll', 'm', \n",
    "    'o', 're', 've', 'y', 'ain', 'aren', \"aren't\", 'couldn', \"couldn't\", \n",
    "    'didn', \"didn't\", 'doesn', \"doesn't\", 'hadn', \"hadn't\", 'hasn', \"hasn't\", \n",
    "    'haven', \"haven't\", 'isn', \"isn't\", 'ma', 'mightn', \"mightn't\", 'mustn', \n",
    "    \"mustn't\", 'needn', \"needn't\", 'shan', \"shan't\", 'shouldn', \"shouldn't\", \n",
    "    'wasn', \"wasn't\", 'weren', \"weren't\", 'won', \"won't\", 'wouldn', \"wouldn't\", \n",
    "    '?', '%', '/', '(', ')', '[', ']', '-', ':', ';', 'system', 'product', \n",
    "    'application', 'should', 'would', 'shall', 'go', 'System', 'system.', \n",
    "    'System.']\n",
    "\n",
    "class_dict = {}\n",
    "\n",
    "# loop through each sentence in our training data\n",
    "for pattern in training_data:\n",
    "    # tokenize each word in the sentence\n",
    "    w = nltk.word_tokenize(pattern['sentence'])\n",
    "    # add to our words list\n",
    "    words.extend(w)\n",
    "    # add to documents in our corpus\n",
    "    documents.append((w, pattern['class'], pattern['class_name']))\n",
    "    # add to our classes list\n",
    "    if pattern['class'] not in classes:\n",
    "        classes.append(pattern['class'])\n",
    "    # add class and name definition to dictionary\n",
    "    if pattern['class'] not in class_dict:\n",
    "        class_dict[pattern['class']] = pattern['class_name']\n",
    "\n",
    "# stem and lower each word and remove duplicates\n",
    "words = [stemmer.stem(w.lower()) for w in words if w not in ignore_words]\n",
    "words = list(set(words))\n",
    "\n",
    "print(len(documents), 'documents\\n')\n",
    "print(len(classes), 'classes', classes, '\\n')\n",
    "print(len(class_dict), 'class dict', class_dict, '\\n')\n",
    "print(len(words), 'unique stemmed words\\n', words)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Start Work Area 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# create our training data\n",
    "training = []\n",
    "output = []\n",
    "output2 = []\n",
    "# create an empty array for our output\n",
    "output_empty = [0] * len(classes)\n",
    "\n",
    "# training set, bag of words for each sentence\n",
    "for doc in documents:\n",
    "    # initialize our bag of words\n",
    "    bag = []\n",
    "    # list of tokenized words for the pattern\n",
    "    pattern_words = doc[0]\n",
    "    # stem each word\n",
    "    pattern_words = [stemmer.stem(word.lower()) for word in pattern_words]\n",
    "    # create our bag of words array\n",
    "    for w in words:\n",
    "        bag.append(1) if w in pattern_words else bag.append(0)\n",
    "\n",
    "    training.append(bag)\n",
    "    # output is a '0' for each tag and '1' for current tag\n",
    "    output_row = list(output_empty)\n",
    "    output_row[classes.index(doc[1])] = 1\n",
    "    output.append(output_row)\n",
    "    \n",
    "    # Output is a '0', '1', '2', or '3' depending on which class it belongs to.\n",
    "    # This is used to build the stratified folds for cross validation.\n",
    "    output_row2 = list(output_empty)\n",
    "    output_row2[classes.index(doc[1])] = classes.index(doc[1])\n",
    "    output2.append(classes.index(doc[1]))\n",
    "    \n",
    "print (\"# words\", len(words))\n",
    "print (\"# classes\", len(classes))\n",
    "\n",
    "print(classes.index(doc[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# sample training/output\n",
    "i = 192\n",
    "w = documents[i][0]\n",
    "print(documents[i][1], documents[i][2])\n",
    "print([stemmer.stem(word.lower()) for word in w])\n",
    "print(training[i])\n",
    "print(output[i])\n",
    "print(output2[i])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Output Lists Explanation\n",
    "\n",
    "+ **Terminology**: classification = target\n",
    "\n",
    "\n",
    "Each Bag of Words is associated with an index in a list that stores its pre-defined classification.\n",
    "\n",
    "There are two of these classification lists. **'output'** and **'output2'**.\n",
    "\n",
    "+ **'output'** is a list of lists of size 4. '0' is used for each target and '1' for the current target. This is how the classification is represented. Ex: [0, 0, 1, 0] \n",
    "\n",
    "+ In the **'output2'** list, each classification is assigned a number '0', '1', '2', or '3'. Only the number value of the current target is appended.\n",
    "\n",
    "This is necessary because the **StratifiedKFold method** imported from **sklearn** cannot sort target values based on **arrays** (both the **'output'** and **'output2'** lists are converted into numpy arrays before being fed into the StratifiedKFold method), but it can sort single integers.\n",
    "\n",
    "## Examples Below"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('')\n",
    "print(output)\n",
    "print('')\n",
    "print(output2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## End Work Area 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import time\n",
    "from sklearn.metrics import precision_recall_fscore_support\n",
    "\n",
    "# Create variables for calculating model average scores\n",
    "fold_averages_list = []\n",
    "fold_dict = {}\n",
    "\n",
    "\n",
    "# compute sigmoid nonlinearity\n",
    "def sigmoid(x):\n",
    "    output = 1/(1+np.exp(-x))\n",
    "    return output\n",
    "\n",
    "# convert output of sigmoid function to its derivative\n",
    "def sigmoid_output_to_derivative(output):\n",
    "    return output*(1-output)\n",
    " \n",
    "def clean_up_sentence(sentence):\n",
    "    # tokenize the pattern\n",
    "    sentence_words = nltk.word_tokenize(sentence)\n",
    "    # stem each word\n",
    "    sentence_words = [stemmer.stem(word.lower()) for word in sentence_words]\n",
    "    return sentence_words\n",
    "\n",
    "# return bag of words array: 0 or 1 for each word in the bag that exists in the sentence\n",
    "def bow(sentence, words, show_details=False):\n",
    "    # tokenize the pattern\n",
    "    sentence_words = clean_up_sentence(sentence)\n",
    "    # bag of words\n",
    "    bag = [0]*len(words)  \n",
    "    for s in sentence_words:\n",
    "        for i,w in enumerate(words):\n",
    "            if w == s: \n",
    "                bag[i] = 1\n",
    "                if show_details:\n",
    "                    print (\"found in bag: %s\" % w)\n",
    "\n",
    "    return(np.array(bag))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Old 'Think' method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def think(sentence, show_details=False):\n",
    "    x = bow(sentence.lower(), words, show_details)\n",
    "    if show_details:\n",
    "        print (\"sentence:\", sentence, \"\\n bow:\", x)\n",
    "    # input layer is our bag of words\n",
    "    \n",
    "    l0 = x\n",
    "    # matrix multiplication of input and hidden layer\n",
    "    l1 = sigmoid(np.dot(l0, synapse_0))\n",
    "    # output layer\n",
    "    l2 = sigmoid(np.dot(l1, synapse_1))\n",
    "    \n",
    "    return l2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Start Work Area 2\n",
    "\n",
    "The method \"think_classify\" **below**, is temporary since it doesn't score the test set.\n",
    "What is actually required is a modifyed version of the \"train\" method which\n",
    "scores the test set but doesn't modify the models weights in response to the\n",
    "sets scores."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def think_classify(bow, s0, s1, show_details=False):\n",
    "    x = np.array(bow)\n",
    "    l0 = x   \n",
    "    l1 = sigmoid(np.dot(l0, s0))\n",
    "    l2 = sigmoid(np.dot(l1, s1))\n",
    "    \n",
    "    return l2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## End Work Area 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Start Work Area 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ANN and Gradient Descent code from https://iamtrask.github.io//2015/07/27/python-network-part2/\n",
    "def train_validation(X, y, y_alt, cycle):\n",
    "    classify_results = []\n",
    "    classify_results_names = []\n",
    "    correct_names = []\n",
    "    last_mean_error = 1\n",
    "    \n",
    "    # probability threshold\n",
    "    ERROR_THRESHOLD = 0.2\n",
    "    # load our calculated synapse values\n",
    "    synapse_file = 'synapses%d.json' % cycle\n",
    "    with open(synapse_file) as data_file: \n",
    "        synapse = json.load(data_file) \n",
    "        synapse_0 = np.asarray(synapse['synapse0']) \n",
    "        synapse_1 = np.asarray(synapse['synapse1'])\n",
    "        \n",
    "    for j in iter(range(10000+1)):\n",
    "\n",
    "        # Feed forward through layers 0, 1, and 2\n",
    "        layer_0 = X\n",
    "        layer_1 = sigmoid(np.dot(layer_0, synapse_0))\n",
    "        layer_2 = sigmoid(np.dot(layer_1, synapse_1))\n",
    "        \n",
    "        # how much did we miss the target value?\n",
    "        layer_2_error = y - layer_2\n",
    "\n",
    "        if (j% 10000) == 0 and j > 5000:\n",
    "            # if this 10k iteration's error is greater than the last iteration, break out\n",
    "            if np.mean(np.abs(layer_2_error)) < last_mean_error:\n",
    "                print (\"delta after \"+str(j)+\" iterations:\" + str(np.mean(np.abs(layer_2_error))) )\n",
    "                last_mean_error = np.mean(np.abs(layer_2_error))\n",
    "            else:\n",
    "                print (\"break:\", np.mean(np.abs(layer_2_error)), \">\", last_mean_error )\n",
    "                break\n",
    "    \n",
    "    print('')\n",
    "    print('Classification results: ')\n",
    "    \n",
    "    for k in range(len(X)):\n",
    "        r = classify(X[k], synapse_0, synapse_1, ERROR_THRESHOLD)\n",
    "        classify_results.append(r)\n",
    "        \n",
    "    for k in range(len(y_alt)):\n",
    "        correct_names.append(class_dict.get('%s' % y_alt[k]))\n",
    "    \n",
    "    for k in range(len(classify_results)):\n",
    "        try:\n",
    "            classify_results_names.append(classify_results[k][0][0])\n",
    "        except IndexError:\n",
    "            classify_results_names.append('None')\n",
    "    \n",
    "    # Show relevant metrics for multi-class classification\n",
    "    print('\\nConfusion Matrix: \\n', confusion_matrix(correct_names, classify_results_names))\n",
    "    print('\\n', classification_report(correct_names, classify_results_names))\n",
    "    print(f'\\nAccuracy --> {accuracy_score(correct_names, classify_results_names)}')\n",
    "    \n",
    "    # Collect the averages of each fold to calculate the average of the model\n",
    "    scores = precision_recall_fscore_support(correct_names, classify_results_names, average='weighted')\n",
    "    fold_dict = {'Precision':scores[0], 'Recall':scores[1], 'Fscore':scores[2]}\n",
    "    fold_averages_list.append(fold_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def classify(bow, s0, s1, error, show_details=False):\n",
    "    ERROR_THRESHOLD = error\n",
    "    results = think_classify(bow, s0, s1, show_details)\n",
    "    \n",
    "    results = [[i,r] for i,r in enumerate(results) if r>ERROR_THRESHOLD ] \n",
    "    results.sort(key=lambda x: x[1], reverse=True) \n",
    "    return_results =[[class_dict[classes[r[0]]],r[1]] for r in results]\n",
    "    #print (\"%s \\n classification: %s\" % (sentence, return_results))\n",
    "    print('classification: %s' % return_results)\n",
    "    \n",
    "    return return_results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## End Work Area 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ANN and Gradient Descent code from https://iamtrask.github.io//2015/07/27/python-network-part2/\n",
    "def train(X, y, cycle, hidden_neurons=10, alpha=1, epochs=50000, dropout=False, dropout_percent=0.5):\n",
    "\n",
    "    print (\"Training with %s neurons, alpha:%s, dropout:%s %s\" % (hidden_neurons, str(alpha), dropout, dropout_percent if dropout else '') )\n",
    "    print (\"Input matrix: %sx%s    Output matrix: %sx%s\" % (len(X),len(X[0]),1, len(classes)) )\n",
    "    np.random.seed(1)\n",
    "\n",
    "    last_mean_error = 1\n",
    "    # randomly initialize our weights with mean 0\n",
    "    synapse_0 = 2*np.random.random((len(X[0]), hidden_neurons)) - 1\n",
    "    synapse_1 = 2*np.random.random((hidden_neurons, len(classes))) - 1\n",
    "\n",
    "    prev_synapse_0_weight_update = np.zeros_like(synapse_0)\n",
    "    prev_synapse_1_weight_update = np.zeros_like(synapse_1)\n",
    "\n",
    "    synapse_0_direction_count = np.zeros_like(synapse_0)\n",
    "    synapse_1_direction_count = np.zeros_like(synapse_1)\n",
    "        \n",
    "    for j in iter(range(epochs+1)):\n",
    "\n",
    "        # Feed forward through layers 0, 1, and 2\n",
    "        layer_0 = X\n",
    "        layer_1 = sigmoid(np.dot(layer_0, synapse_0))\n",
    "                \n",
    "        if(dropout):\n",
    "            layer_1 *= np.random.binomial([np.ones((len(X),hidden_neurons))],1-dropout_percent)[0] * (1.0/(1-dropout_percent))\n",
    "\n",
    "        layer_2 = sigmoid(np.dot(layer_1, synapse_1))\n",
    "        \n",
    "        # how much did we miss the target value?\n",
    "        layer_2_error = y - layer_2\n",
    "\n",
    "        if (j% 10000) == 0 and j > 5000:\n",
    "            # if this 10k iteration's error is greater than the last iteration, break out\n",
    "            if np.mean(np.abs(layer_2_error)) < last_mean_error:\n",
    "                print (\"delta after \"+str(j)+\" iterations:\" + str(np.mean(np.abs(layer_2_error))) )\n",
    "                last_mean_error = np.mean(np.abs(layer_2_error))\n",
    "            #else:\n",
    "                #print (\"break:\", np.mean(np.abs(layer_2_error)), \">\", last_mean_error )\n",
    "                #break\n",
    "                \n",
    "        # in what direction is the target value?\n",
    "        # were we really sure? if so, don't change too much.\n",
    "        layer_2_delta = layer_2_error * sigmoid_output_to_derivative(layer_2)\n",
    "\n",
    "        # how much did each l1 value contribute to the l2 error (according to the weights)?\n",
    "        layer_1_error = layer_2_delta.dot(synapse_1.T)\n",
    "\n",
    "        # in what direction is the target l1?\n",
    "        # were we really sure? if so, don't change too much.\n",
    "        layer_1_delta = layer_1_error * sigmoid_output_to_derivative(layer_1)\n",
    "        \n",
    "        synapse_1_weight_update = (layer_1.T.dot(layer_2_delta))\n",
    "        synapse_0_weight_update = (layer_0.T.dot(layer_1_delta))\n",
    "        \n",
    "        if(j > 0):\n",
    "            synapse_0_direction_count += np.abs(((synapse_0_weight_update > 0)+0) - ((prev_synapse_0_weight_update > 0) + 0))\n",
    "            synapse_1_direction_count += np.abs(((synapse_1_weight_update > 0)+0) - ((prev_synapse_1_weight_update > 0) + 0))        \n",
    "        \n",
    "        synapse_1 += alpha * synapse_1_weight_update\n",
    "        synapse_0 += alpha * synapse_0_weight_update\n",
    "        \n",
    "        prev_synapse_0_weight_update = synapse_0_weight_update\n",
    "        prev_synapse_1_weight_update = synapse_1_weight_update\n",
    "\n",
    "    now = datetime.datetime.now()\n",
    "\n",
    "    # persist synapses\n",
    "    synapse = {'synapse0': synapse_0.tolist(), 'synapse1': synapse_1.tolist(),\n",
    "               'datetime': now.strftime(\"%Y-%m-%d %H:%M\"),\n",
    "               'words': words,\n",
    "               'classes': classes\n",
    "              }\n",
    "    synapse_file = \"synapses%d.json\" % cycle\n",
    "\n",
    "    with open(synapse_file, 'w') as outfile:\n",
    "        json.dump(synapse, outfile, indent=4, sort_keys=True)\n",
    "    print (\"saved synapses to:\", synapse_file)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Start Work Area 4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "X = np.array(training)\n",
    "y = np.array(output2)\n",
    "y2 = np.array(output)\n",
    "#target_names = ['Performance','Usability','Security','Operability']\n",
    "#dataset = sklearn.datasets.base.Bunch(data=X, target=y, target_names=target_names)\n",
    "skf = StratifiedKFold(n_splits=10, random_state=None)\n",
    "cycle = 0\n",
    "\n",
    "print(datetime.datetime.now())\n",
    "\n",
    "# X is the feature set and y is the target\n",
    "for train_index, val_index in skf.split(X,y): \n",
    "    cycle = cycle + 1\n",
    "    print('***************************')\n",
    "    print(\"CROSS VALIDATION CYCLE: %d\" % cycle)\n",
    "    print('***************************')\n",
    "    print(\"Train:\", train_index)\n",
    "    print(\"Validation:\", val_index)\n",
    "    print('')\n",
    "    X_train, X_test = X[train_index], X[val_index] \n",
    "    y_train, y_test = y2[train_index], y2[val_index]\n",
    "    y_test_alt = y[val_index]\n",
    "    \n",
    "    start_time = time.time()\n",
    "\n",
    "    train(X_train, y_train, cycle, hidden_neurons=20, alpha=0.1, epochs=200000, dropout=True, dropout_percent=0.2)\n",
    "\n",
    "    elapsed_time = time.time() - start_time\n",
    "    print (\"processing time:\", elapsed_time, \"seconds\")\n",
    "    print('')\n",
    "    \n",
    "    print('*********************************')\n",
    "    print(\"VALIDATION RESULTS FOR CYCLE: %d\" % cycle)\n",
    "    print('*********************************')\n",
    "    \n",
    "    start_time = time.time()\n",
    "    \n",
    "    train_validation(X_test, y_test, y_test_alt, cycle)\n",
    "    \n",
    "    elapsed_time = time.time() - start_time\n",
    "    print (\"processing time:\", elapsed_time, \"seconds\")\n",
    "    print('')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Calculating Mean Average of model scores\n",
    "p_list = []\n",
    "r_list = []\n",
    "f_list = []\n",
    "\n",
    "for r in range(len(fold_averages_list)):\n",
    "    p_list.append(fold_averages_list[r].get('Precision'))\n",
    "    r_list.append(fold_averages_list[r].get('Recall'))\n",
    "    f_list.append(fold_averages_list[r].get('Fscore'))\n",
    "\n",
    "def mean(numbers):\n",
    "    return float(sum(numbers)) / max(len(numbers), 1)\n",
    "\n",
    "print(\"\\nModel Mean Precision: %f \" % mean(p_list))\n",
    "print(\"Model Mean Recall: %f \" % mean(r_list))\n",
    "print(\"Model Mean Fscore: %f \" % mean(f_list))\n",
    "\n",
    "print(\"Finish time: \", datetime.datetime.now().time())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## End Work Area 4"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Scratch Pad (Notes, Questions, Ideas, Resolve/d)\n",
    "\n",
    "### (RESOLVED) Solve turning an array of all features (172) into a array of just the four types with the true type represented as a 1 and the remaining three as all 0.\n",
    "\n",
    "I need to find a way to map this **new validation array** consisting of 0,1,2,3 to\n",
    "the **original design** of 4 tensors, 1 for each of the possible answers.\n",
    "\n",
    "### Zone 4 Print Tests\n",
    "\n",
    "    # TESTING PRINT STATEMENTS\n",
    "    '''print('X_train')\n",
    "    print(X_train)\n",
    "    print('')\n",
    "    print('y_train')\n",
    "    print(y_train)\n",
    "    print('')\n",
    "    print('X_test')\n",
    "    print(X_test)\n",
    "    print('')\n",
    "    print('y_test')\n",
    "    print(y_test)\n",
    "    print('')'''\n",
    "    \n",
    "### Zone 3 Print Tests\n",
    "\n",
    "    # TESTING PRINT STATEMENTS\n",
    "    '''print('')\n",
    "    print('layer_0 length: %s' % len(layer_0))\n",
    "    print('')\n",
    "    print('layer_1 length: %s' % len(layer_1))\n",
    "    print('')\n",
    "    print(\"y length: %s\" % len(y))\n",
    "    print('')\n",
    "    print('y itself: %s' % y)\n",
    "    print('')\n",
    "    print(\"y2 length: %s\" % len(y2))\n",
    "    print('')\n",
    "    print('y2 itself: %s' % y2)\n",
    "    print('')\n",
    "    print('layer_2 length: %s' % len(layer_2))'''\n",
    "    \n",
    "### (RESOLVED) Study how classifation occurs in the tutorial model, and mimic it in the new cross validation structure.\n",
    "\n",
    "### (RESOLVED) Try making a dict, [{key: , value: }] to map the output to their proper names, instead of trying to align new lists which aren't generated to be in the same order.\n",
    "\n",
    "### Accuracy Print Tests\n",
    "\n",
    "    print('classify results')\n",
    "    print(len(classify_results))\n",
    "    print(classify_results)\n",
    "    print('')\n",
    "    print('correct names')\n",
    "    print(len(correct_names))\n",
    "    print(correct_names)\n",
    "    print('')\n",
    "    print('classify results names THE LOGIC FOR THIS ONE MATTERS')\n",
    "    print(len(classify_results_names))\n",
    "    print(classify_results_names)\n",
    "    \n",
    "### (RESOLVED) Make sure the correct names list is actually the correct names, and in the correct order!\n",
    "\n",
    "### (Resolve) Make sure networks are using the correct network weghts each cross validation. Not the same from the previous training cycle, but new random near-zero values.\n",
    "\n",
    "### (Resolve) Make sure you know what accuracy is and how it is calculated."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "my-env",
   "language": "python",
   "name": "my-env"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}