LLM Autograding manual graded questions

apps/prairielearn/package.json

@@ -136,6 +136,7 @@
     "oauth-signature": "^1.5.0",
     "object-hash": "^3.0.0",
     "on-finished": "^2.4.1",
+    "openai": "^4.47.1",
     "openid-client": "^5.6.5",
     "parse5": "^7.1.2",
     "passport": "^0.7.0",

apps/prairielearn/src/lib/config.ts

@@ -516,6 +516,8 @@ const ConfigSchema = z.object({
    * Maps a plan name ("basic", "compute", etc.) to a Stripe product ID.
    */
   stripeProductIds: z.record(z.string(), z.string()).default({}),
+  openAiApiKey: z.string().nullable().default(null),
+  openAiOrganization: z.string().nullable().default(null),
 });
 
 export type Config = z.infer<typeof ConfigSchema>;

apps/prairielearn/src/lib/features/index.ts

@@ -6,6 +6,7 @@ const featureNames = [
   // Can only be applied to courses/institutions.
   'process-questions-in-worker',
   'question-sharing',
+  'bot-grading',
   // Can only be applied to institutions.
   'lti13',
   'terms-clickthrough',

apps/prairielearn/src/lib/question-render.js

@@ -184,7 +184,7 @@ async function render(
  * @param  {import('./db-types.js').InstanceQuestion?} instance_question The instance question.
  * @return {Record<string, any>} An object containing the named URLs.
  */
-function buildQuestionUrls(urlPrefix, variant, question, instance_question) {
+export function buildQuestionUrls(urlPrefix, variant, question, instance_question) {
   const urls = {};
 
   if (!instance_question) {
@@ -234,7 +234,7 @@ function buildQuestionUrls(urlPrefix, variant, question, instance_question) {
   return urls;
 }
 
-function buildLocals(
+export function buildLocals(
   variant,
   question,
   instance_question,

apps/prairielearn/src/middlewares/authzCourseOrInstance.js

@@ -115,6 +115,8 @@ export async function authzCourseOrInstance(req, res) {
     res.locals,
   );
 
+  res.locals.bot_grading_enabled = await features.enabledFromLocals('bot-grading', res.locals);
+
   // Check if it is necessary to request a user data override - if not, return
   let overrides = [];
   if (req.cookies.pl_requested_uid) {

apps/prairielearn/src/pages/instructorAssessmentManualGrading/assessmentQuestion/assessmentQuestion.ejs

@@ -340,6 +340,14 @@
       </a>
       <%- include('../../partials/assessmentSyncErrorsAndWarnings'); %>
 
+      <form name="start-bot-grading" method="POST">
+        <input type="hidden" name="__action" value="bot_grade_assessment" />
+        <input type="hidden" name="__csrf_token" value="<%= __csrf_token %>">
+        <div class="mt-4">
+          <button type="submit" class="btn btn-primary">Bot Grade!</button>
+        </div>
+      </form>
+
       <div class="card mb-4">
         <div class="card-header bg-primary text-white">
           <%= assessment.tid %> / Question <%= number_in_alternative_group %>.

apps/prairielearn/src/pages/instructorAssessmentManualGrading/assessmentQuestion/assessmentQuestion.sql

@@ -71,3 +71,42 @@ WHERE
         )
     )
   );
+
+-- BLOCK select_last_submission
+SELECT
+  s.* AS submission
+FROM
+  variants AS v
+  JOIN submissions AS s ON (s.variant_id = v.id)
+WHERE
+  v.instance_question_id = $instance_question_id
+ORDER BY
+  v.date DESC,
+  s.date DESC
+LIMIT
+  1;
+
+-- BLOCK select_last_variant
+SELECT
+  v.* AS variant
+FROM
+  variants AS v
+  JOIN submissions AS s ON (s.variant_id = v.id)
+WHERE
+  v.instance_question_id = $instance_question_id
+ORDER BY
+  v.date DESC,
+  s.date DESC
+LIMIT
+  1;
+
+-- BLOCK select_question_of_variant
+SELECT
+  q.* AS question
+FROM
+  questions AS q
+WHERE
+  q.course_id = $question_course_id
+  AND q.id = $question_id
+LIMIT
+  1;

apps/prairielearn/src/pages/instructorAssessmentManualGrading/assessmentQuestion/assessmentQuestion.ts

@@ -1,12 +1,19 @@
 import * as express from 'express';
 import asyncHandler from 'express-async-handler';
+import { OpenAI } from 'openai';
 import { z } from 'zod';
 
 import * as error from '@prairielearn/error';
-import { loadSqlEquiv, queryAsync, queryRows } from '@prairielearn/postgres';
+import { logger } from '@prairielearn/logger';
+import { loadSqlEquiv, queryAsync, queryRow, queryRows } from '@prairielearn/postgres';
 
-import { InstanceQuestionSchema } from '../../../lib/db-types.js';
+import { config } from '../../../lib/config.js';
+import { InstanceQuestionSchema, SubmissionSchema, VariantSchema } from '../../../lib/db-types.js';
 import * as manualGrading from '../../../lib/manualGrading.js';
+import { buildLocals, buildQuestionUrls } from '../../../lib/question-render.js';
+import { getQuestionCourse } from '../../../lib/question-variant.js';
+import { createServerJob } from '../../../lib/server-jobs.js';
+import * as questionServers from '../../../question-servers/index.js';
 
 const router = express.Router();
 const sql = loadSqlEquiv(import.meta.url);
@@ -136,6 +143,179 @@ router.post(
       } else {
         res.send({});
       }
+    } else if (req.body.__action === 'bot_grade_assessment') {
+      // check if bot grading is enabled
+      if (!res.locals.bot_grading_enabled) {
+        throw new error.HttpStatusError(403, 'Access denied (feature not available)');
+      }
+      console.log('Bot grading the assessment question');
+
+      if (config.openAiApiKey === null || config.openAiOrganization === null) {
+        throw new error.HttpStatusError(501, 'Not implemented (feature not available)');
+      }
+      const openaiconfig = {
+        apiKey: config.openAiApiKey,
+        organization: config.openAiOrganization,
+      };
+
+      const { urlPrefix, assessment, assessment_instance, assessment_question, authz_result } =
+        res.locals;
+      const question = res.locals.question;
+      const question_course = await getQuestionCourse(question, res.locals.course);
+
+      const serverJob = await createServerJob({
+        courseId: res.locals.course ? res.locals.course.id : null,
+        type: 'botGrading',
+        description: 'Use LLM to grade assessment question',
+      });
+
+      serverJob.executeInBackground(async (job) => {
+        const openai = new OpenAI(openaiconfig);
+
+        // get all instance questions
+        const result = await queryRows(
+          sql.select_instance_questions_manual_grading,
+          {
+            assessment_id: res.locals.assessment.id,
+            assessment_question_id: res.locals.assessment_question.id,
+          },
+          InstanceQuestionSchema,
+        );
+
+        let error_count = 0;
+        let output_count = 0;
+        let output: string | null = null;
+
+        // get each instance question
+        for (const instance_question of result) {
+          // get last submission of instance question
+          const submission = await queryRow(
+            sql.select_last_submission,
+            { instance_question_id: instance_question.id },
+            SubmissionSchema,
+          );
+
+          // maybe remove some if statements that can never happen
+          // if nothing submitted
+          if (submission.submitted_answer == null) {
+            continue;
+          }
+          // if no file submitted or too many files submitted
+          if (
+            submission.submitted_answer._files == null ||
+            submission.submitted_answer._files.length !== 1
+          ) {
+            continue;
+          }
+          const student_answer = atob(submission.submitted_answer._files[0].contents);
+
+          // get question prompt
+          const variant = await queryRow(
+            sql.select_last_variant,
+            { instance_question_id: instance_question.id },
+            VariantSchema,
+          );
+
+          // build new locals for the question server
+          const urls = buildQuestionUrls(urlPrefix, variant, question, instance_question);
+          const newLocals = buildLocals(
+            variant,
+            question,
+            instance_question,
+            assessment,
+            assessment_instance,
+            assessment_question,
+            authz_result,
+          );
+          const locals = {};
+          Object.assign(locals, urls);
+          Object.assign(locals, newLocals);
+
+          // get question html
+          const questionModule = questionServers.getModule(question.type);
+          const { courseIssues, data } = await questionModule.render(
+            { question: true, submissions: false, answer: false },
+            variant,
+            question,
+            submission,
+            [submission],
+            question_course,
+            locals,
+          );
+          if (courseIssues.length) {
+            job.info(courseIssues.toString());
+            job.error('Error occurred');
+            job.fail('Errors occurred while bot grading, see output for details');
+          }
+
+          const question_prompt = data.questionHtml.split('<script>', 2)[0];
+
+          // Call OpenAI API
+          const completion = await openai.chat.completions.create({
+            messages: [
+              {
+                role: 'system',
+                content: `You are an instructor for a course, and you are grading assignments. You should always return the grade using a json object of 2 parameters: grade and feedback. The grade should be an integer between 0 and 100. 0 being the lowest and 100 being the highest, and the feedback should be why you give this grade, or how to improve the answer. You can say correct or leave blank when the grade is close to 100. `,
+              },
+              {
+                role: 'user',
+                content: `Question: \n${question_prompt} \nAnswer: \n${student_answer} \nHow would you grade this? Please return the json object.`,
+              },
+            ],
+            model: 'gpt-3.5-turbo',
+          });
+
+          let msg = '';
+          try {
+            if (completion.choices[0].message.content === null) {
+              error_count++;
+              continue;
+            }
+            const gpt_answer = JSON.parse(completion.choices[0].message.content);
+            const update_result = await manualGrading.updateInstanceQuestionScore(
+              res.locals.assessment.id,
+              instance_question.id,
+              submission.id,
+              req.body.modified_at,
+              {
+                score_perc: gpt_answer.grade,
+                feedback: { manual: gpt_answer.feedback },
+                // NEXT STEPS: rubrics
+              },
+              '1',
+            );
+            msg = `Bot grades for ${instance_question.id}: ${gpt_answer.grade}`;
+            if (update_result.modified_at_conflict) {
+              error_count++;
+              msg += `\nERROR modified at conflict for ${instance_question.id}`;
+            }
+          } catch (err) {
+            logger.error('error while regrading', { err });
+            error_count++;
+            msg = `ERROR bot grading for ${instance_question.id}`;
+          }
+          output = (output == null ? '' : `${output}\n`) + msg;
+          output_count++;
+          if (output_count >= 100) {
+            job.info(output);
+            output = null;
+            output_count = 0;
+          }
+        }
+
+        if (output != null) {
+          job.info(output);
+        }
+        if (error_count > 0) {
+          job.error('Number of errors: ' + error_count);
+          job.fail('Errors occurred while bot grading, see output for details');
+        }
+      });
+
+      // for debugging, run your docker container with "docker run -it --rm -p 3000:3000 -e NODEMON=true -v ~/git/PrairieLearn:/PrairieLearn --name mypl prairielearn/prairielearn"
+      // to check out your database, run "docker exec -it mypl psql postgres"
+
+      res.redirect(req.originalUrl);
     } else {
       throw new error.HttpStatusError(400, `unknown __action: ${req.body.__action}`);
     }