diff --git a/api-testing-bot/build.gradle b/api-testing-bot/build.gradle index bb1ba32..17d0f78 100644 --- a/api-testing-bot/build.gradle +++ b/api-testing-bot/build.gradle @@ -34,6 +34,8 @@ dependencies { implementation "org.kohsuke:github-api:1.308" implementation "io.jsonwebtoken:jjwt-impl:0.11.5" implementation "io.jsonwebtoken:jjwt-jackson:0.11.5" + + implementation "com.github.javaparser:javaparser-core:3.24.4" } configurations { diff --git a/api-testing-bot/src/main/java/i5/las2peer/services/apiTestingBot/RESTResources.java b/api-testing-bot/src/main/java/i5/las2peer/services/apiTestingBot/RESTResources.java index 3e61915..07a50de 100644 --- a/api-testing-bot/src/main/java/i5/las2peer/services/apiTestingBot/RESTResources.java +++ b/api-testing-bot/src/main/java/i5/las2peer/services/apiTestingBot/RESTResources.java @@ -56,6 +56,7 @@ public Response modelTest(String body) { StringBuilder responseMessageSB = new StringBuilder(); APITestingBot service = (APITestingBot) Context.get().getService(); + String codexAPIToken = service.getCodexAPIToken(); // setup message handler MessageHandler messageHandler; @@ -68,6 +69,22 @@ public Response modelTest(String body) { handleNextState = messageHandler.handleInit(responseMessageSB, context); } + if(handleNextState && context.getState() == API_TEST_FAMILIARITY_QUESTION) { + handleNextState = messageHandler.handleAPITestFamiliarityQuestion(responseMessageSB); + } + + if(initialState == API_TEST_FAMILIARITY_QUESTION) { + handleNextState = messageHandler.handleAPITestFamiliarityQuestionAnswer(responseMessageSB, context, intent); + } + + if(handleNextState && context.getState() == ENTER_TEST_CASE_DESCRIPTION) { + handleNextState = messageHandler.handleTestCaseDescriptionQuestion(responseMessageSB); + } + + if(initialState == ENTER_TEST_CASE_DESCRIPTION) { + handleNextState = messageHandler.handleTestCaseDescription(responseMessageSB, context, message, codexAPIToken); + } + if (handleNextState && context.getState() == RC_SELECT_PROJECT) { handleNextState = ((RCMessageHandler) messageHandler).handleProjectSelectionQuestion(responseMessageSB, context, channel); } diff --git a/api-testing-bot/src/main/java/i5/las2peer/services/apiTestingBot/chat/GHMessageHandler.java b/api-testing-bot/src/main/java/i5/las2peer/services/apiTestingBot/chat/GHMessageHandler.java index df14854..d8dd3a8 100644 --- a/api-testing-bot/src/main/java/i5/las2peer/services/apiTestingBot/chat/GHMessageHandler.java +++ b/api-testing-bot/src/main/java/i5/las2peer/services/apiTestingBot/chat/GHMessageHandler.java @@ -1,7 +1,12 @@ package i5.las2peer.services.apiTestingBot.chat; +import i5.las2peer.apiTestModel.BodyAssertion; +import i5.las2peer.apiTestModel.RequestAssertion; +import i5.las2peer.apiTestModel.StatusCodeAssertion; +import i5.las2peer.apiTestModel.TestRequest; import i5.las2peer.services.apiTestingBot.context.TestModelingContext; +import static i5.las2peer.services.apiTestingBot.chat.MessageHandlerUtil.handleYesNoQuestion; import static i5.las2peer.services.apiTestingBot.chat.Messages.*; import static i5.las2peer.services.apiTestingBot.context.TestModelingState.*; @@ -10,18 +15,17 @@ */ public class GHMessageHandler extends MessageHandler { - /** - * Reacts to the initial message of the user that starts a test modeling conversation. - * - * @param responseMessageSB StringBuilder - * @param context Current test modeling context - * @return Whether the next state should be handled too. - */ @Override - public boolean handleInit(StringBuilder responseMessageSB, TestModelingContext context) { - responseMessageSB.append(MODEL_TEST_CASE_INTRO); - context.setState(NAME_TEST_CASE); - return true; + public boolean handleAPITestFamiliarityQuestionAnswer(StringBuilder responseMessageSB, TestModelingContext context, String intent) { + return handleYesNoQuestion(responseMessageSB, intent, () -> { + responseMessageSB.append(OK); + context.setState(ENTER_TEST_CASE_DESCRIPTION); + return true; + }, () -> { + responseMessageSB.append(OK); + context.setState(NAME_TEST_CASE); + return true; + }); } /** @@ -99,4 +103,37 @@ public boolean handlePath(StringBuilder responseMessageSB, TestModelingContext c context.setState(BODY_QUESTION); return true; } + + @Override + public String getTestDescription(TestRequest request) { + return getGitHubTestDescription(request); + } + + public static String getGitHubTestDescription(TestRequest request) { + String description = ""; + String url = getRequestUrlWithPathParamValues(request); + description += "**Method & Path:** `" + request.getType() + "` `" + url + "`\n"; + + // request auth info + String auth = request.getAgent() == -1 ? "None" : "User Agent"; + description += "**Authorization:** " + auth + "\n"; + + // request body (if exists) + if(request.getBody() != null && !request.getBody().isEmpty()) { + description += "**Body:**\n" + "```json" + request.getBody() + "```\n"; + } + + // list assertions + description += "**Assertions:**\n"; + for(RequestAssertion assertion : request.getAssertions()) { + description += "- "; + if(assertion instanceof StatusCodeAssertion) { + description += "Expected status code: " + ((StatusCodeAssertion) assertion).getStatusCodeValue(); + } else { + description += ((BodyAssertion) assertion).getOperator().toString(); + } + description += "\n"; + } + return description; + } } diff --git a/api-testing-bot/src/main/java/i5/las2peer/services/apiTestingBot/chat/MessageHandler.java b/api-testing-bot/src/main/java/i5/las2peer/services/apiTestingBot/chat/MessageHandler.java index 8d87bb4..b07ca35 100644 --- a/api-testing-bot/src/main/java/i5/las2peer/services/apiTestingBot/chat/MessageHandler.java +++ b/api-testing-bot/src/main/java/i5/las2peer/services/apiTestingBot/chat/MessageHandler.java @@ -1,14 +1,16 @@ package i5.las2peer.services.apiTestingBot.chat; -import i5.las2peer.apiTestModel.BodyAssertion; -import i5.las2peer.apiTestModel.BodyAssertionOperator; -import i5.las2peer.apiTestModel.RequestAssertion; -import i5.las2peer.apiTestModel.StatusCodeAssertion; +import i5.las2peer.apiTestModel.*; +import i5.las2peer.services.apiTestingBot.codex.CodeToTestModel; +import i5.las2peer.services.apiTestingBot.codex.CodexAPI; +import i5.las2peer.services.apiTestingBot.codex.CodexTestGen; import i5.las2peer.services.apiTestingBot.context.BodyAssertionType; import i5.las2peer.services.apiTestingBot.context.TestModelingContext; +import org.json.simple.JSONObject; import org.json.simple.JSONValue; import org.json.simple.parser.ParseException; +import java.io.IOException; import java.util.List; import static i5.las2peer.services.apiTestingBot.chat.MessageHandlerUtil.*; @@ -25,7 +27,68 @@ public abstract class MessageHandler { * @param context Current test modeling context * @return Whether the next state should be handled too. */ - public abstract boolean handleInit(StringBuilder responseMessageSB, TestModelingContext context); + public boolean handleInit(StringBuilder responseMessageSB, TestModelingContext context) { + responseMessageSB.append(MODEL_TEST_CASE_INTRO); + context.setState(API_TEST_FAMILIARITY_QUESTION); + return true; + } + + public boolean handleAPITestFamiliarityQuestion(StringBuilder responseMessageSB) { + if(!responseMessageSB.isEmpty()) responseMessageSB.append(" "); + responseMessageSB.append(API_TEST_FAMILIARITY_QUESTION_TEXT); + return false; + } + + public abstract boolean handleAPITestFamiliarityQuestionAnswer(StringBuilder responseMessageSB, TestModelingContext context, String intent); + + public boolean handleTestCaseDescriptionQuestion(StringBuilder responseMessageSB) { + if(!responseMessageSB.isEmpty()) responseMessageSB.append(" "); + responseMessageSB.append(ENTER_TEST_CASE_DESCRIPTION_TEXT); + return false; + } + + public boolean handleTestCaseDescription(StringBuilder responseMessageSB, TestModelingContext context, String message, + String codexAPIToken) { + TestRequest generatedRequest = null; + try { + generatedRequest = new CodexTestGen(codexAPIToken).descriptionToTestModel(message); + } catch (CodexAPI.CodexAPIException | IOException e) { + e.printStackTrace(); + return false; + } catch (CodeToTestModel.CodeToTestModelException e) { + e.printStackTrace(); + responseMessageSB.append(ERROR_TEST_CASE_GENERATION); + return false; + } + context.setRequestMethod(generatedRequest.getType()); + context.setRequestPath(generatedRequest.getUrl()); + context.setPathParamValues(generatedRequest.getPathParams()); + context.setTestCaseName("TestCaseName"); + context.setRequestBody(generatedRequest.getBody()); + context.getAssertions().addAll(generatedRequest.getAssertions()); + responseMessageSB.append(getTestDescription(generatedRequest)); + context.setState(FINAL); + return false; + } + + public abstract String getTestDescription(TestRequest request); + + /** + * Returns the request url of the given test request where the path parameters are replaced with their values. + * + * @param request TestRequest + * @return Request url of the given test request where the path parameters are replaced with their values. + */ + protected static String getRequestUrlWithPathParamValues(TestRequest request) { + String url = request.getUrl(); + JSONObject pathParams = request.getPathParams(); + for(Object key : pathParams.keySet()) { + String paramValue = String.valueOf(pathParams.get(key)); + if(paramValue.isEmpty()) paramValue = ""; + url = url.replace("{" + key + "}", paramValue); + } + return url; + } /** * Ask user to enter a name for the test case. diff --git a/api-testing-bot/src/main/java/i5/las2peer/services/apiTestingBot/chat/Messages.java b/api-testing-bot/src/main/java/i5/las2peer/services/apiTestingBot/chat/Messages.java index 967b9d4..dd86659 100644 --- a/api-testing-bot/src/main/java/i5/las2peer/services/apiTestingBot/chat/Messages.java +++ b/api-testing-bot/src/main/java/i5/las2peer/services/apiTestingBot/chat/Messages.java @@ -3,6 +3,8 @@ public class Messages { public static String MODEL_TEST_CASE_INTRO = "Ok, let's model a test case."; + public static String API_TEST_FAMILIARITY_QUESTION_TEXT = "Are you familiar with REST API testing?"; + public static String ENTER_TEST_CASE_DESCRIPTION_TEXT = "How would you describe the test case? Please give me a few sentences about it."; public static String SELECT_PROJECT_FOR_TEST_CASE = "Which project should the test case be added to? Please enter a number:"; public static String SELECT_MICROSERVICE_FOR_TEST_CASE = "Which microservice should the test case be added to? Please enter a number:"; public static String ENTER_TEST_CASE_NAME = "Please enter a name for the test case:"; @@ -38,6 +40,7 @@ public class Messages { public static String ERROR_COULD_NOT_UNDERSTAND = "I could not understand that. Please try again."; public static String ERROR_COULD_NOT_UNDERSTAND_TYPE = "I could not understand that. Please enter a valid type."; public static String ERROR_BODY_NO_VALID_JSON = "Entered body is no valid JSON! Please try again."; + public static String ERROR_TEST_CASE_GENERATION = "Unfortunately I was not able to generate a test case based on your description. Please try again."; public static String TEST_ADD_TO_PROJECT(String projectName) { return "The test will be added to the project \"" + projectName + "\"."; diff --git a/api-testing-bot/src/main/java/i5/las2peer/services/apiTestingBot/chat/RCMessageHandler.java b/api-testing-bot/src/main/java/i5/las2peer/services/apiTestingBot/chat/RCMessageHandler.java index 104a34c..291db9b 100644 --- a/api-testing-bot/src/main/java/i5/las2peer/services/apiTestingBot/chat/RCMessageHandler.java +++ b/api-testing-bot/src/main/java/i5/las2peer/services/apiTestingBot/chat/RCMessageHandler.java @@ -1,5 +1,9 @@ package i5.las2peer.services.apiTestingBot.chat; +import i5.las2peer.apiTestModel.BodyAssertion; +import i5.las2peer.apiTestModel.RequestAssertion; +import i5.las2peer.apiTestModel.StatusCodeAssertion; +import i5.las2peer.apiTestModel.TestRequest; import i5.las2peer.services.apiTestingBot.context.TestModelingContext; import i5.las2peer.services.apiTestingBot.util.ProjectServiceHelper; import io.swagger.v3.oas.models.Operation; @@ -15,8 +19,7 @@ import java.util.List; import java.util.Map; -import static i5.las2peer.services.apiTestingBot.chat.MessageHandlerUtil.getFirstUnsetPathParam; -import static i5.las2peer.services.apiTestingBot.chat.MessageHandlerUtil.handleNumberSelectionQuestion; +import static i5.las2peer.services.apiTestingBot.chat.MessageHandlerUtil.*; import static i5.las2peer.services.apiTestingBot.chat.Messages.*; import static i5.las2peer.services.apiTestingBot.chat.Messages.SELECT_PROJECT_FOR_TEST_CASE; import static i5.las2peer.services.apiTestingBot.context.TestModelingState.*; @@ -32,18 +35,17 @@ public RCMessageHandler(String caeBackendURL) { this.caeBackendURL = caeBackendURL; } - /** - * Reacts to the initial message of the user that starts a test modeling conversation. - * - * @param responseMessageSB StringBuilder - * @param context Current test modeling context - * @return Whether the next state should be handled too. - */ @Override - public boolean handleInit(StringBuilder responseMessageSB, TestModelingContext context) { - responseMessageSB.append(MODEL_TEST_CASE_INTRO); - context.setState(RC_SELECT_PROJECT); - return true; + public boolean handleAPITestFamiliarityQuestionAnswer(StringBuilder responseMessageSB, TestModelingContext context, String intent) { + return handleYesNoQuestion(responseMessageSB, intent, () -> { + responseMessageSB.append(OK); + context.setState(ENTER_TEST_CASE_DESCRIPTION); + return true; + }, () -> { + responseMessageSB.append(OK); + context.setState(RC_SELECT_PROJECT); + return true; + }); } /** @@ -324,4 +326,33 @@ public boolean handlePathParams(StringBuilder responseMessageSB, TestModelingCon return false; } + + @Override + public String getTestDescription(TestRequest request) { + String description = ""; + String url = getRequestUrlWithPathParamValues(request); + description += "Method & Path: `" + request.getType() + "` `" + url + "`\n"; + + // request auth info + String auth = request.getAgent() == -1 ? "None" : "User Agent"; + description += "Authorization: " + auth + "\n"; + + // request body (if exists) + if(request.getBody() != null && !request.getBody().isEmpty()) { + description += "Body:\n" + "`" + request.getBody() + "`\n"; + } + + // list assertions + description += "Assertions:\n"; + for(RequestAssertion assertion : request.getAssertions()) { + description += "- "; + if(assertion instanceof StatusCodeAssertion) { + description += "Expected status code: " + ((StatusCodeAssertion) assertion).getStatusCodeValue(); + } else { + description += ((BodyAssertion) assertion).getOperator().toString(); + } + description += "\n"; + } + return description; + } } diff --git a/api-testing-bot/src/main/java/i5/las2peer/services/apiTestingBot/codex/CodeToTestModel.java b/api-testing-bot/src/main/java/i5/las2peer/services/apiTestingBot/codex/CodeToTestModel.java new file mode 100644 index 0000000..398f9cd --- /dev/null +++ b/api-testing-bot/src/main/java/i5/las2peer/services/apiTestingBot/codex/CodeToTestModel.java @@ -0,0 +1,238 @@ +package i5.las2peer.services.apiTestingBot.codex; + +import com.github.javaparser.JavaParser; +import com.github.javaparser.ast.CompilationUnit; +import com.github.javaparser.ast.Node; +import com.github.javaparser.ast.body.MethodDeclaration; +import com.github.javaparser.ast.body.VariableDeclarator; +import com.github.javaparser.ast.expr.*; +import com.github.javaparser.ast.stmt.ExpressionStmt; +import com.github.javaparser.ast.stmt.Statement; +import com.github.javaparser.ast.stmt.TryStmt; +import i5.las2peer.apiTestModel.*; +import org.json.simple.JSONObject; + +import java.util.ArrayList; +import java.util.List; +import java.util.regex.MatchResult; +import java.util.regex.Pattern; + +import static i5.las2peer.services.apiTestingBot.codex.CodexTestGen.BASE_PROMPT_CLASS_NAME; +import static i5.las2peer.services.apiTestingBot.codex.CodexTestGen.BASE_PROMPT_METHOD_NAME; + +public class CodeToTestModel { + + public class CodeToTestModelException extends Exception { + public CodeToTestModelException(String message) { + super(message); + } + } + + // try to create test model from code + public TestRequest convert(String code) throws CodeToTestModelException { + TryStmt tryStmt = getTryStmtOfTestMethod(code); + if(tryStmt == null) return null; + + List statements = tryStmt.getTryBlock().getStatements().stream() + .filter(statement -> statement instanceof ExpressionStmt) + .map(ExpressionStmt.class::cast).toList(); + + List variableDeclarationExprs = statements.stream() + .filter(exp -> exp.getExpression().isVariableDeclarationExpr()) + .map(exp -> exp.getExpression().asVariableDeclarationExpr()).toList(); + + String requestMethod = getRequestMethod(variableDeclarationExprs); + String requestPath = getRequestPath(variableDeclarationExprs); + JSONObject pathParameters = getPathParams(variableDeclarationExprs, requestPath); + String body = getBody(variableDeclarationExprs); + + List methodCallExprs = statements.stream() + .filter(exp -> exp.getExpression().isMethodCallExpr()) + .map(exp -> exp.getExpression().asMethodCallExpr()).toList(); + + List assertThatExprs = methodCallExprs.stream() + .filter(exp -> exp.getName().asString().equals("assertThat")).toList(); + + List assertions = getAssertions(assertThatExprs); + + return new TestRequest(requestMethod, requestPath, pathParameters, -1, body, assertions); + } + + private List getAssertions(List assertThatExprs) throws CodeToTestModelException { + List assertions = new ArrayList<>(); + + for(MethodCallExpr assertThatExp : assertThatExprs) { + if(assertThatExp.getArguments().size() < 2) continue; + + Expression firstArg = assertThatExp.getArguments().get(0); + Expression secondArg = assertThatExp.getArguments().get(1); + + if(firstArg.toString().equals("statusCode")) { + RequestAssertion statusCodeAssertion = getStatusCodeAssertion(secondArg); + if(statusCodeAssertion != null) assertions.add(statusCodeAssertion); + } else if(firstArg.toString().equals("response")) { + RequestAssertion bodyAssertion = getBodyAssertion(secondArg); + if(bodyAssertion != null) assertions.add(bodyAssertion); + } + } + return assertions; + } + + private static RequestAssertion getStatusCodeAssertion(Expression arg) { + if(arg.toString().startsWith("is(")) { + int statusCode = Integer.valueOf(arg.toString().split("\\(")[1].split("\\)")[0]); + return new StatusCodeAssertion(0, statusCode); + } + return null; + } + + private RequestAssertion getBodyAssertion(Expression arg) throws CodeToTestModelException { + Expression current = arg; + BodyAssertionOperator operator = null; + while(current != null) { + if(!current.isMethodCallExpr()) break; + List childNodes = current.getChildNodes(); + + BodyAssertionOperator followingOperator = null; + + String methodName = current.asMethodCallExpr().getName().asString(); + if(methodName.equals("isA")) { + // create "has type" assertion + Expression e = ((Expression) childNodes.get(1)); + if(e.isClassExpr()) { + String className = e.asClassExpr().getType().asString(); + int inputType = classNameToInputType(className); + followingOperator = new BodyAssertionOperator(0, inputType); + } + current = null; + } else if(methodName.equals("asJSONObject") || methodName.equals("asJSONObjectList")) { + // continue with first argument of asJSONObject or asJSONObjectList + if(childNodes.size() > 1) current = (Expression) childNodes.get(1); + else current = null; + } else if(methodName.equals("everyItem")) { + if(childNodes.size() == 2) { + followingOperator = new BodyAssertionOperator(3, 0); + current = (Expression) childNodes.get(1); + } else { + throw new CodeToTestModelException("Error: everyItem has more/less than 1 argument"); + } + } else if(methodName.equals("hasField")) { + Node hasFieldFirstArg = childNodes.get(1); + + if(hasFieldFirstArg instanceof StringLiteralExpr) { + followingOperator = new BodyAssertionOperator(1, 1, ((StringLiteralExpr) hasFieldFirstArg).getValue(), null); + // check if there is a second argument + if(childNodes.size() > 2) { + Node hasFieldSecondArg = childNodes.get(2); + current = (Expression) hasFieldSecondArg; + } else { + current = null; + } + } else { + throw new CodeToTestModelException("Error, hasField first arg is not StringLiteralExpr"); + } + } else { + throw new CodeToTestModelException("methodName " + methodName + " unsupported here!"); + } + + if(operator == null) { + operator = followingOperator; + } else { + BodyAssertionOperator last = operator; + while(last.hasFollowingOperator()) last = last.getFollowingOperator(); + last.setFollowedByOperator(followingOperator); + } + } + + if(operator != null) return new BodyAssertion(operator); + return null; + } + + private static int classNameToInputType(String className) { + switch (className) { + case "JSONObject": + return 2; + case "JSONArray": + return 3; + case "String": + return 4; + case "Number": + return 5; + case "Boolean": + return 6; + default: + return -1; + } + } + + private static String getRequestMethod(List variableDeclarationExprs) { + List nodes = getSendRequestArgs(variableDeclarationExprs); + return ((StringLiteralExpr) nodes.get(2)).getValue(); + } + + private static String getRequestPath(List variableDeclarationExprs) { + List nodes = getSendRequestArgs(variableDeclarationExprs); + return ((BinaryExpr) nodes.get(3)).getRight().asStringLiteralExpr().getValue(); + } + + private static JSONObject getPathParams(List variableDeclarationExprs, String requestPath) { + List nodes = getSendRequestArgs(variableDeclarationExprs); + JSONObject pathParameters = new JSONObject(); + + Pattern p = Pattern.compile("\\{([^}]*)}"); + int i = 7; + for(Object match : p.matcher(requestPath).results().toArray()) { + MatchResult result = (MatchResult) match; + String paramName = requestPath.substring(result.start() + 1, result.end() - 1); + if (i < nodes.size()) { + Node node = nodes.get(i); + String paramValue = "?"; + if(node instanceof StringLiteralExpr) { + paramValue = ((StringLiteralExpr) nodes.get(i)).getValue(); + } else if(node instanceof IntegerLiteralExpr) { + paramValue = ((IntegerLiteralExpr) nodes.get(i)).getValue(); + } + pathParameters.put(paramName, paramValue); + i++; + } else { + break; + } + } + return pathParameters; + } + + private static String getBody(List variableDeclarationExprs) { + List nodes = getSendRequestArgs(variableDeclarationExprs); + String body = ""; + Expression bodyExp = (Expression) nodes.get(4); + if(bodyExp instanceof NullLiteralExpr) { + // body is null + } else if(bodyExp instanceof StringLiteralExpr) { + body = ((StringLiteralExpr) bodyExp).getValue(); + } else if(bodyExp instanceof TextBlockLiteralExpr) { + body = ((TextBlockLiteralExpr) bodyExp).getValue(); + } + return body; + } + + private static List getSendRequestArgs(List variableDeclarationExprs) { + for(VariableDeclarationExpr varDeclExp : variableDeclarationExprs) { + if (varDeclExp.getElementType().toString().equals("ClientResponse")) { + VariableDeclarator d = varDeclExp.getVariables().get(0); + Expression e = d.getInitializer().get(); + return e.getChildNodes(); + } + } + return null; + } + + private static TryStmt getTryStmtOfTestMethod(String code) { + CompilationUnit unit = new JavaParser().parse(code).getResult().get(); + MethodDeclaration methodDeclaration = unit.getClassByName(BASE_PROMPT_CLASS_NAME).get() + .getMethodsByName(BASE_PROMPT_METHOD_NAME).get(0); + for(Statement statement : methodDeclaration.getBody().get().getStatements()) { + if(statement instanceof TryStmt) return (TryStmt) statement; + } + return null; + } +} diff --git a/api-testing-bot/src/main/java/i5/las2peer/services/apiTestingBot/codex/CodexAPI.java b/api-testing-bot/src/main/java/i5/las2peer/services/apiTestingBot/codex/CodexAPI.java new file mode 100644 index 0000000..ebf1b73 --- /dev/null +++ b/api-testing-bot/src/main/java/i5/las2peer/services/apiTestingBot/codex/CodexAPI.java @@ -0,0 +1,54 @@ +package i5.las2peer.services.apiTestingBot.codex; + +import kong.unirest.ContentType; +import kong.unirest.HttpResponse; +import kong.unirest.Unirest; +import org.json.simple.JSONArray; +import org.json.simple.JSONObject; +import org.json.simple.JSONValue; + +public class CodexAPI { + + public class CodexAPIException extends Exception { + public CodexAPIException(String message) { + super(message); + } + } + + private static final String COMPLETIONS_ENDPOINT = "https://api.openai.com/v1/completions"; + private static final String MODEL_NAME = "code-davinci-002"; + private static final double TEMPERATURE = 0.2; + private static final int MAX_TOKENS = 100; + + private String apiToken; + + public CodexAPI(String apiToken) { + this.apiToken = apiToken; + } + + public JSONArray insert(String prompt, String suffix, String stop) throws CodexAPIException { + JSONObject body = new JSONObject(); + body.put("model", MODEL_NAME); + body.put("prompt", prompt); + body.put("suffix", suffix); + body.put("temperature", TEMPERATURE); + body.put("max_tokens", MAX_TOKENS); + body.put("n", 1); + body.put("stop", stop); + + HttpResponse res = Unirest.post(COMPLETIONS_ENDPOINT) + .basicAuth("", apiToken) + .body(body.toJSONString()) + .contentType(ContentType.APPLICATION_JSON.toString()) + .asString(); + + if(res.isSuccess()) { + JSONObject jsonRes = (JSONObject) JSONValue.parse(res.getBody()); + JSONArray choices = (JSONArray) jsonRes.get("choices"); + return choices; + } else { + throw new CodexAPIException("An error occurred while using the Codex API. Status code: " + res.getStatus() + + ", message: " + res.getBody()); + } + } +} diff --git a/api-testing-bot/src/main/java/i5/las2peer/services/apiTestingBot/codex/CodexTestGen.java b/api-testing-bot/src/main/java/i5/las2peer/services/apiTestingBot/codex/CodexTestGen.java new file mode 100644 index 0000000..89ea0f8 --- /dev/null +++ b/api-testing-bot/src/main/java/i5/las2peer/services/apiTestingBot/codex/CodexTestGen.java @@ -0,0 +1,167 @@ +package i5.las2peer.services.apiTestingBot.codex; + +import com.github.javaparser.JavaParser; +import com.github.javaparser.ast.CompilationUnit; +import com.github.javaparser.ast.body.MethodDeclaration; +import com.github.javaparser.ast.comments.LineComment; +import com.github.javaparser.ast.stmt.Statement; +import com.github.javaparser.ast.stmt.TryStmt; +import i5.las2peer.apiTestModel.BodyAssertion; +import i5.las2peer.apiTestModel.BodyAssertionOperator; +import i5.las2peer.apiTestModel.RequestAssertion; +import i5.las2peer.apiTestModel.TestRequest; +import org.json.simple.JSONArray; +import org.json.simple.JSONObject; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; + +public class CodexTestGen { + + public static final String BASE_PROMPT_CLASS_NAME = "Test"; + public static final String BASE_PROMPT_METHOD_NAME = "test"; + + private String codexAPIToken; + + public CodexTestGen(String codexAPIToken) { + this.codexAPIToken = codexAPIToken; + } + + public TestRequest descriptionToTestModel(String testCaseDescription) throws CodexAPI.CodexAPIException, IOException, CodeToTestModel.CodeToTestModelException { + String testCode = descriptionToCode(testCaseDescription); + return new CodeToTestModel().convert(testCode); + } + + private String descriptionToCode(String testCaseDescription) throws CodexAPI.CodexAPIException, IOException, CodeToTestModel.CodeToTestModelException { + InputStream in = getClass().getResourceAsStream("/basePrompt.java"); + String code = new String(in.readAllBytes(), StandardCharsets.UTF_8); + + // put test case description into method comment + code = code.replace("[INPUT]", testCaseDescription); + + // insert rest of c.sendRequest(...) + code = insert(code, ","); + code = code.replace("[insert2]", "[insert]"); + code = insert(code, ";"); + + // insert status code assertion + code = addInsertTag(code); + code = code.replace("[insert]", "assertThat(statusCode, [insert];"); + code = insert(code, ";"); + + // insert other assertions + int iterations = 5; + for(int i = 0; i < iterations; i++) { + String updatedContent = addInsertTag(new String(code)); + updatedContent = updatedContent.replace("[insert]", "assertThat(response, [insert];"); + updatedContent = insert(updatedContent, ";"); + + // get body assertions from current request model + List bodyAssertions = getBodyAssertionsFromCode(updatedContent); + if(bodyAssertions.isEmpty()) continue; + + // get newest assertion + BodyAssertion latest = bodyAssertions.get(bodyAssertions.size()-1); + if(containsIrrelevantHasFieldAssertion(latest, testCaseDescription)) { + // stop code generation here + break; + } + + code = updatedContent; + } + + return code; + } + + private static boolean containsIrrelevantHasFieldAssertion(BodyAssertion bodyAssertion, String testCaseDescription) { + BodyAssertionOperator operator = bodyAssertion.getOperator(); + while(operator != null) { + if(operator.getOperatorId() == 1) { + // this is a "has field" operator + // check if the field name was mentioned in the test case description given by the user + String fieldName = operator.getInputValue(); + if(!(testCaseDescription.contains(" " + fieldName + " ") + || testCaseDescription.contains(" " + fieldName) + || testCaseDescription.contains("\"" + fieldName) + || testCaseDescription.contains("'" + fieldName))) { + // might be irrelevant + return true; + } + } + operator = operator.getFollowingOperator(); + } + return false; + } + + private static List getBodyAssertionsFromCode(String code) throws CodeToTestModel.CodeToTestModelException { + List bodyAssertions = new ArrayList<>(); + + List assertions = new CodeToTestModel().convert(code).getAssertions(); + for(RequestAssertion assertion : assertions) { + if(assertion instanceof BodyAssertion) { + BodyAssertion b = (BodyAssertion) assertion; + bodyAssertions.add(b); + } + } + + return bodyAssertions; + } + + /** + * Appends a new [insert] tag at the end of the try-block of the test method. + * + * @param code Java code + * @return Given code, with a new [insert] tag at the end of the try-block of the test method. + */ + private static String addInsertTag(String code) { + // parse java code + CompilationUnit unit = new JavaParser().parse(code).getResult().get(); + + // get test method + MethodDeclaration methodDeclaration = unit.getClassByName(BASE_PROMPT_CLASS_NAME).get() + .getMethodsByName(BASE_PROMPT_METHOD_NAME).get(0); + + // add new [insert] tag at the end of the try-block + for(Statement statement : methodDeclaration.getBody().get().getStatements()) { + if(statement instanceof TryStmt) { + TryStmt tryStmt = (TryStmt) statement; + // we need to append the [insert] tag as a comment first + tryStmt.getTryBlock().addOrphanComment(new LineComment("[insert]")); + break; + } + } + code = unit.toString(); + code = code.replace("// [insert]", "[insert]\n"); + return code; + } + + /** + * Uses OpenAI's Codex model to complete the given code (at the [insert] tag). + * + * @param code Code containing [insert] at the place where new code should be generated and inserted. + * @param stop Character at which the code generation should stop. + * @return Given code, where [insert] got replaced with code generated by the Codex model. + * @throws CodexAPI.CodexAPIException If the API request was not successful. + */ + private String insert(String code, String stop) throws CodexAPI.CodexAPIException { + // split code into two parts - before and after [insert] + String prompt = code.split("\\[insert]")[0]; + String suffix = code.split("\\[insert]")[1]; + + // call OpenAI API + JSONArray choices = new CodexAPI(codexAPIToken).insert(prompt, suffix, stop); + + // get first choice + JSONObject choice = (JSONObject) choices.get(0); + String text = (String) choice.get("text"); + + // replace [insert] with generated code + code = code.replace("[insert]", text); + + return code; + } + +} diff --git a/api-testing-bot/src/main/java/i5/las2peer/services/apiTestingBot/context/TestModelingState.java b/api-testing-bot/src/main/java/i5/las2peer/services/apiTestingBot/context/TestModelingState.java index 1a6af11..0f9bca5 100644 --- a/api-testing-bot/src/main/java/i5/las2peer/services/apiTestingBot/context/TestModelingState.java +++ b/api-testing-bot/src/main/java/i5/las2peer/services/apiTestingBot/context/TestModelingState.java @@ -6,6 +6,10 @@ public enum TestModelingState { */ INIT, + API_TEST_FAMILIARITY_QUESTION, + + ENTER_TEST_CASE_DESCRIPTION, + /** * This state handles the selection of a CAE project (which must be linked to the current channel). * In this state, the available projects are listed. diff --git a/api-testing-bot/src/main/java/i5/las2peer/services/apiTestingBot/util/PRTestGenHelper.java b/api-testing-bot/src/main/java/i5/las2peer/services/apiTestingBot/util/PRTestGenHelper.java index 9b2e6f1..6f1496f 100644 --- a/api-testing-bot/src/main/java/i5/las2peer/services/apiTestingBot/util/PRTestGenHelper.java +++ b/api-testing-bot/src/main/java/i5/las2peer/services/apiTestingBot/util/PRTestGenHelper.java @@ -4,6 +4,7 @@ import i5.las2peer.api.logging.MonitoringEvent; import i5.las2peer.apiTestModel.*; import i5.las2peer.services.apiTestingBot.APITestingBot; +import i5.las2peer.services.apiTestingBot.chat.GHMessageHandler; import org.apache.commons.io.IOUtils; import org.json.simple.JSONArray; import org.json.simple.JSONObject; @@ -179,29 +180,7 @@ private static String getTestCasePresentationComment(TestCase testCase, String t // show request method and path TestRequest request = testCase.getRequests().get(0); - String url = getRequestUrlWithPathParamValues(request); - message += "**Method & Path:** `" + request.getType() + "` `" + url + "`\n"; - - // request auth info - String auth = request.getAgent() == -1 ? "None" : "User Agent"; - message += "**Authorization:** " + auth + "\n"; - - // request body (if exists) - if(request.getBody() != null && !request.getBody().isEmpty()) { - message += "**Body:**\n" + "```json" + request.getBody() + "```\n"; - } - - // list assertions - message += "**Assertions:**\n"; - for(RequestAssertion assertion : request.getAssertions()) { - message += "- "; - if(assertion instanceof StatusCodeAssertion) { - message += "Expected status code: " + ((StatusCodeAssertion) assertion).getStatusCodeValue(); - } else { - message += ((BodyAssertion) assertion).getOperator().toString(); - } - message += "\n"; - } + message += GHMessageHandler.getGitHubTestDescription(request); // append overview of available bot commands message += BOT_COMMAND_OVERVIEW; @@ -209,23 +188,6 @@ private static String getTestCasePresentationComment(TestCase testCase, String t return message; } - /** - * Returns the request url of the given test request where the path parameters are replaced with their values. - * - * @param request TestRequest - * @return Request url of the given test request where the path parameters are replaced with their values. - */ - private static String getRequestUrlWithPathParamValues(TestRequest request) { - String url = request.getUrl(); - JSONObject pathParams = request.getPathParams(); - for(Object key : pathParams.keySet()) { - String paramValue = String.valueOf(pathParams.get(key)); - if(paramValue.isEmpty()) paramValue = ""; - url = url.replace("{" + key + "}", paramValue); - } - return url; - } - /** * Checks if the workflow run with the given id contains an OpenAPI doc artifact. * diff --git a/api-testing-bot/src/main/resources/basePrompt.java b/api-testing-bot/src/main/resources/basePrompt.java new file mode 100644 index 0000000..9cb511f --- /dev/null +++ b/api-testing-bot/src/main/resources/basePrompt.java @@ -0,0 +1,140 @@ +import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.isA; +import static org.hamcrest.MatcherAssert.assertThat; + +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.Assert; + +import org.json.simple.JSONObject; +import org.json.simple.JSONValue; +import org.json.simple.JSONArray; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.networknt.schema.JsonSchema; +import com.networknt.schema.JsonSchemaFactory; +import com.networknt.schema.SpecVersion; +import com.networknt.schema.ValidationMessage; + +import org.hamcrest.Description; +import org.hamcrest.FeatureMatcher; +import org.hamcrest.Matcher; +import org.hamcrest.TypeSafeDiagnosingMatcher; + +import java.util.List; +import java.util.Set; + +public class Test { + + public static FeatureMatcher hasField(String featureName, Matcher subMatcher) { + return new FeatureMatcher<>(subMatcher, "should have field\"" + featureName + "\"", featureName) { + @Override + protected Object featureValueOf(JSONObject jsonObject) { + return jsonObject.get(featureName); + } + }; + } + + public static FeatureMatcher> asJSONObjectList(Matcher> subMatcher) { + return new FeatureMatcher<>(subMatcher, "as a JSONObjectList", "") { + @Override + protected List featureValueOf(Object o) { + JSONArray arr = (JSONArray) o; + return List.of(arr.toArray()); + } + }; + } + + public static TypeSafeDiagnosingMatcher hasField(String fieldName) { + return new TypeSafeDiagnosingMatcher<>() { + @Override + protected boolean matchesSafely(JSONObject jsonObject, Description mismatchDescription) { + if (!jsonObject.containsKey(fieldName)) + mismatchDescription.appendText("but does not have field " + fieldName); + return jsonObject.containsKey(fieldName); + } + + @Override + public void describeTo(Description description) { + description.appendText("should have field " + fieldName); + } + }; + } + + public static FeatureMatcher asJSONObject(Matcher subMatcher) { + return new FeatureMatcher<>(subMatcher, "as a JSONObject", "") { + @Override + protected JSONObject featureValueOf(Object o) { + return (JSONObject) o; + } + }; + } + + public ClientResponse sendRequest(String method, String uri, String content, String contentType, String accept, Map headers, Object... pathParameters) throws Exception { + return super.sendRequest(method, basePath + uri, content, contentType, accept, headers); + } + + /** + * JUnit test: Send post request to /example and check that status code is 201 and response has field "exampleText". + */ + @Test + public void exampleTest() { + MiniClientCoverage c = new MiniClientCoverage(mainPath); + c.setConnectorEndpoint(connector.getHttpEndpoint()); + + try { + ClientResponse result = c.sendRequest("POST", mainPath + "/example", "application/json", "*/*", null); + int statusCode = result.getHttpCode(); + Object response = JSONValue.parse(result.getResponse().trim()); + + assertThat(statusCode, is(201)); + assertThat(response, asJSONObject(hasField("exampleText", isA(String.class)))); + } catch (Exception e) { + e.printStackTrace(); + fail("Exception: " + e); + } + } + + /** + * JUnit test: Send post request to /example/3 with body {"num": 5, "value": "Hi"} and check that status code is 201 and response has type json object. + */ + @Test + public void exampleTest() { + MiniClientCoverage c = new MiniClientCoverage(mainPath); + c.setConnectorEndpoint(connector.getHttpEndpoint()); + + try { + ClientResponse result = c.sendRequest("POST", mainPath + "/example/{id}", """{"num": 5, "value": "Hi"}""", "application/json", "*/*", 3); + int statusCode = result.getHttpCode(); + Object response = JSONValue.parse(result.getResponse().trim()); + + assertThat(statusCode, is(201)); + assertThat(response, isA(JSONObject.class)); + } catch (Exception e) { + e.printStackTrace(); + fail("Exception: " + e); + } + } + + /* JUnit test: [INPUT] */ + @Test + public void test() { + MiniClientCoverage c = new MiniClientCoverage(mainPath); + c.setConnectorEndpoint(connector.getHttpEndpoint()); + + try { + ClientResponse result = c.sendRequest([insert], mainPath + [insert2]; + int statusCode = result.getHttpCode(); + Object response = JSONValue.parse(result.getResponse().trim()); + } catch (Exception e) { + e.printStackTrace(); + fail("Exception: " + e); + } + } +} \ No newline at end of file