Brute Ratel Parser Alpha & Custom Parser Foundation (#143)

Co-authored-by: Austin Golding <austin.golding@pnnl.gov>
Co-authored-by: Courtney Carpenter <ccarpenter28@gmail.com>
Co-authored-by: Sebastian Ang <dajian.ang@pnnl.gov>

.eslintrc.json

@@ -33,7 +33,7 @@
 			{
 				"prefer": "type-imports",
 				"fixStyle": "separate-type-imports",
-				"disallowTypeAnnotations": true
+				"disallowTypeAnnotations": false
 			}
 		],
 		"import/no-anonymous-default-export": ["error"]

.moon/tasks.yml

@@ -44,6 +44,18 @@ fileGroups:
 # This setting requires a map, where the key is a unique name for the task,
 # and the value is an object of task parameters.
 tasks:
+  xstate-typegen:
+    command:
+      - xstate
+      - typegen
+      - src/**/*.machine.ts?(x)
+    outputs:
+      - 'src/**/*.machine.typegen.{ts,tsx}'
+    inputs:
+      - 'src/**/*.machine.{ts,tsx}'
+    platform: node
+    options:
+      runInCI: false
   test-jest:
     command: 'jest . --coverage --reporters="jest-junit" --passWithNoTests'
     inputs:
@@ -69,7 +81,7 @@ tasks:
     options:
       runInCI: false
   lint-global:
-    command: 'eslint --config $workspaceRoot/.eslintrc.json --ignore-path $workspaceRoot/.gitignore src --fix --max-warnings 0'
+    command: 'eslint --config $workspaceRoot/.eslintrc.json --ignore-path $workspaceRoot/.gitignore src --fix'
     inputs:
       - '@globs(sources)'
       - '@globs(tests)'

.moon/workspace.yml

@@ -14,8 +14,12 @@ $schema: 'https://moonrepo.dev/schemas/workspace.json'
 # path to the project folder as the map value. File paths are relative from the workspace root,
 # and cannot reference projects located outside the workspace boundary.
 projects:
-  - 'applications/*'
-  - 'packages/*'
+  globs:
+    - 'applications/*'
+    - 'packages/*'
+    - 'parsers/*'
+  sources:
+    landing-page: 'docs/landing-page'
 
 # Configures the version control system to utilize within the workspace. A VCS
 # is required for determining touched (added, modified, etc) files, calculating file hashes,

Dockerfile

@@ -6,13 +6,13 @@ ENV CYPRESS_INSTALL_BINARY=0
 RUN npm install -g pkg
 RUN yarn install --immutable --inline-builds
 RUN curl -fsSL https://moonrepo.dev/install/moon.sh | bash
-RUN moon run server:build client:build cs-parser:build
+RUN moon run server:build client:build cobalt-strike-parser:build
 RUN pkg applications/server/package.json -t node16-mac-x64 -o release/mac/RedEye
 RUN pkg applications/server/package.json -t node16-linux-x64 -o release/linux/RedEye
 RUN pkg applications/server/package.json -t node16-windows-x64 -o release/windows/RedEye
-RUN pkg packages/cs-parser/package.json -t node16-mac-x64 -o release/mac/parsers/cs-parser
-RUN pkg packages/cs-parser/package.json -t node16-linux-x64 -o release/linux/parsers/cs-parser
-RUN pkg packages/cs-parser/package.json -t node16-windows-x64 -o release/windows/parsers/cs-parser
+RUN pkg packages/cobalt-strike-parser/package.json -t node16-mac-x64 -o release/mac/parsers/cobalt-strike-parser
+RUN pkg packages/cobalt-strike-parser/package.json -t node16-linux-x64 -o release/linux/parsers/cobalt-strike-parser
+RUN pkg packages/cobalt-strike-parser/package.json -t node16-windows-x64 -o release/windows/parsers/cobalt-strike-parser
 RUN tar -zcvf release.tar.gz ./release/
 RUN mkdir outputs
 RUN cp release.tar.gz outputs/release.tar.gz

README.md

@@ -18,19 +18,19 @@ Red Team: [![Red Team](https://img.shields.io/endpoint?url=https://cloud.cypress
 
 Blue Team: [![Blue Team](https://img.shields.io/endpoint?url=https://cloud.cypress.io/badge/simple/46ahz3&style=flat&logo=cypress)](https://cloud.cypress.io/projects/46ahz3/runs)
 
-## [User Guide](docs/UserGuide.md)
+## [User Guide](https://github.com/cisagov/RedEye/blob/develop/docs/UserGuide.md)
 
-Follow along with the [User Guide](docs/UserGuide.md) to learn about RedEye's feature set.
+Follow along with the [User Guide](https://github.com/cisagov/RedEye/blob/develop/docs/UserGuide.md) to learn about RedEye's feature set.
 
 ## Quick start
 
 1. **Download** the latest RedEye binaries for your OS[\*](#platform-support) from the [Releases](https://github.com/cisagov/RedEye/releases) page.
-2. **Pick a mode** and **Run the server**
-   - [ **Red Team mode**](#red-team) enables the full feature set: upload C2 logs, explore data, and create presentations. To start the server in Red Team mode, run the following in a terminal. _You must provide a password to run in RedTeam mode._
+   2. **Pick a mode** and **Run the server**
+     - [ **Red Team mode**](#red-team) enables the full feature set: upload C2 logs, explore data, and create presentations. To start the server in Red Team mode, run the following in a terminal. _You must provide a password to run in RedTeam mode._
      ```
-     AUTHENTICATION_PASSWORD=<your_password> ./RedEye --redTeam
+       ./RedEye --redTeam --password <your_password>
      ```
-   - [**Blue Team mode**](#blue-team) (default) enables a simplified, read-only UI for reviewing campaigns exported by a Red Team. To start the server in Blue Team mode. Double-click on the 'RedEye' executable or run `./RedEye` from the command line.
+     - [**Blue Team mode**](#blue-team) (default) enables a simplified, read-only UI for reviewing campaigns exported by a Red Team. To start the server in Blue Team mode. Double-click on the 'RedEye' executable or run `./RedEye` from the command line.
 3. **Use the web app** in a browser at http://127.0.0.1:4000. The RedEye binary runs as a server in a terminal window and will automatically open the web app UI your default browser. You must close the terminal window to quit the RedEye server.
 
 _**MacOS Issue** - When running RedEye for the first time, you may get a "not verified" error. You must go to "System Preferences" > "Security & Privacy" > "General" and click "Open Anyway." More info on the [Apple support page](https://support.apple.com/guide/mac-help/open-a-mac-app-from-an-unidentified-developer-mh40616/)._
@@ -46,13 +46,13 @@ _Note: Both Red and Blue Team modes can be started from the same RedEye applicat
 The downloaded binary comes in two parts:
 
 - The `RedEye` application binary
-- The `parsers` folder containing the `cs-parser` Cobalt Strike log parser binary
+- The `parsers` folder containing parser binaries (e.g. `cobalt-strike-parser` Cobalt Strike log parser binary)
 
 There are three options to run RedEye in Red Team mode:
 
 1. Run the downloaded binary, passing in the `--redTeam` and password options:
    ```
-   AUTHENTICATION_PASSWORD=<your_password> ./RedEye --redTeam
+   ./RedEye --redTeam --password <your_password>
    ```
 2. Clone, install, and run the project directly (covered in the [Local Build](#local-build) section).
 3. Docker Compose
@@ -100,23 +100,23 @@ RedEye runs as a server and can be setup to serve the UI on a network..
 Type `./Redeye -h` to view the options
 
 ```
--d, --developmentMode [boolean]  put the database and server in development mode
--r, --redTeam [boolean]          run the server in red team mode
--p, --port [number]              the port the server should be exposed at
--t, --childProcesses [number]    max # of child processes the parser can use
--h, --help                       display help for command
+-d, --developmentMode [boolean]  put the database and server in development mode
+-r, --redTeam [boolean]          run the server in red team mode
+--port [number]                  the port the server should be exposed at
+-p, --password [string]          the password for user authentication
+--parsers [string...]            A list of parsers to use or a flag to use all parsers in the parsers folder
+-t, --childProcesses [number]    max # of child processes the parser can use
+-h, --help                       display help for command
 ```
 
-you can also configure the sever parameters in an `.env` file that sits next to the `RedEye` binary
+you can also configure the server parameters in a `config.json` file that sits next to the `RedEye` binary
+```json
 
-<!-- ***{is this true?}*** -->
-
-```env
-AUTHENTICATION_PASSWORD=937038570
-AUTHENTICATION_SECRET=supertopsecretdonttellanyone
-DATABASE_MODE=DEV_PERSIST
-SERVER_BLUE_TEAM=false
-SERVER_PRODUCTION=false
+{
+	"password": "937038570",
+	"redTeam": true,
+	"parsers": ["cobalt-strike-parser", "brute-ratel-parser"]
+}
 ```
 
 ## Local Build

applications/client/.eslintrc.yaml

@@ -18,6 +18,8 @@ extends:
   - airbnb
   - airbnb-typescript
   - prettier
+globals:
+  globalThis: false
 rules:
   react-hooks/exhaustive-deps: 0 # Disabled for mobx
   react/no-unknown-property: 0 # Disabled for css and cy-test

applications/client/moon.yml

@@ -17,9 +17,15 @@ tasks:
     command: mk-gql --outDir=./src/store/graphql ../server/schema.graphql
     options:
       runInCI: false
+  lint-graphql:
+    command: 'eslint src/store/graphql --fix'
+    options:
+      runInCI: false
   generate-graphql:
     command: 'prettier --write src/store/graphql/*.ts'
     deps:
       - ~:graphql
+      - ~:lint-graphql
     options:
       runInCI: false
+      runDepsInParallel: false

applications/client/src/components/Mitre/process-enterprise-attack.js

@@ -69,11 +69,11 @@ Object.keys(mitreAttackDictionary)
 			: mitreAttackDictionary[id];
 	});
 
-console.log(`Parsed ${Object.entries(alphabeticalMitreAttackDictionary).length} MITRE ATT&CK ids`);
+globalThis.console.log(`Parsed ${Object.entries(alphabeticalMitreAttackDictionary).length} MITRE ATT&CK ids`);
 
 // it helps to manually run prettier on this after its generated
 const mitreAttackDictionaryPathTs = path.join(__dirname, 'mitreAttackDictionary.ts');
 const tsFileContents = `export const mitreAttackDictionary = ${JSON.stringify(alphabeticalMitreAttackDictionary)}`;
 fs.writeFile(mitreAttackDictionaryPathTs, tsFileContents, (err) => {
-	if (err) console.error(err);
+	if (err) globalThis.console.error(err);
 });

applications/client/src/store/graphql/BeaconMetaModel.base.ts

@@ -33,10 +33,14 @@ export class BeaconMetaModelBase extends Model({
 	ip: prop<string | null>().withSetter(),
 	/** Process Identifier the beacon is running on */
 	pid: prop<number | null>().withSetter(),
+	/** The IP of the host at the time of the metadata line */
+	port: prop<number | null>().withSetter(),
+	/** Process Identifier the beacon is running on */
+	process: prop<string | null>().withSetter(),
 	/** The shape of the beacon */
 	shape: prop<Shapes | null>().withSetter(),
 	/** The log line from which the BeaconMeta was extracted */
-	source: prop<Ref<LogEntryModel>>().withSetter(),
+	source: prop<Ref<LogEntryModel> | null>().withSetter(),
 	/** The start time of the beacon */
 	startTime: prop<any | null>().withSetter(),
 	/** The communication type used by the beacon */
@@ -65,6 +69,12 @@ export class BeaconMetaModelSelector extends QueryBuilder {
 	get pid() {
 		return this.__attr(`pid`);
 	}
+	get port() {
+		return this.__attr(`port`);
+	}
+	get process() {
+		return this.__attr(`process`);
+	}
 	get shape() {
 		return this.__attr(`shape`);
 	}
@@ -85,4 +95,5 @@ export function selectFromBeaconMeta() {
 	return new BeaconMetaModelSelector();
 }
 
-export const beaconMetaModelPrimitives = selectFromBeaconMeta().color.endTime.ip.pid.shape.startTime.type.username;
+export const beaconMetaModelPrimitives =
+	selectFromBeaconMeta().color.endTime.ip.pid.port.process.shape.startTime.type.username;

applications/client/src/store/graphql/BeaconTypeEnum.ts

@@ -9,10 +9,10 @@ import { types, prop, tProp, Model, Ref } from 'mobx-keystone';
  */
 
 export enum BeaconType {
-	DNS = 'DNS',
-	HTTP = 'HTTP',
-	HTTPS = 'HTTPS',
-	SMB = 'SMB',
+	dns = 'dns',
+	http = 'http',
+	https = 'https',
+	smb = 'smb',
 }
 
 /**

applications/client/src/store/graphql/CampaignModel.base.ts

@@ -5,9 +5,11 @@
 
 import { types, prop, tProp, Model, Ref, idProp } from 'mobx-keystone';
 import { QueryBuilder } from 'mk-gql';
+import type { CampaignParserModel } from './CampaignParserModel';
 import type { GlobalOperatorModel } from './GlobalOperatorModel';
 import type { ParsingStatus } from './ParsingStatusEnum';
 
+import { CampaignParserModelSelector, campaignParserModelPrimitives } from './CampaignParserModel';
 import { GlobalOperatorModelSelector, globalOperatorModelPrimitives } from './GlobalOperatorModel';
 
 /* The TypeScript type that explicits the refs to other models in order to prevent a circular refs issue */
@@ -33,6 +35,7 @@ export class CampaignModelBase extends Model({
 	lastOpenedBy: prop<Ref<GlobalOperatorModel> | null>().withSetter(),
 	migrationError: prop<boolean>().withSetter(),
 	name: prop<string>().withSetter(),
+	parsers: prop<CampaignParserModel[] | null>(() => []).withSetter(),
 	parsingStatus: prop<ParsingStatus>().withSetter(),
 	serverCount: prop<number>().withSetter(),
 }) {
@@ -91,6 +94,14 @@ export class CampaignModelSelector extends QueryBuilder {
 	) {
 		return this.__child(`lastOpenedBy`, GlobalOperatorModelSelector, builder);
 	}
+	parsers(
+		builder?:
+			| string
+			| CampaignParserModelSelector
+			| ((selector: CampaignParserModelSelector) => CampaignParserModelSelector)
+	) {
+		return this.__child(`parsers`, CampaignParserModelSelector, builder);
+	}
 }
 export function selectFromCampaign() {
 	return new CampaignModelSelector();

applications/client/src/store/graphql/CampaignModel.ts

@@ -9,6 +9,7 @@ export { campaignModelPrimitives, CampaignModelSelector, selectFromCampaign } fr
 
 export interface Servers {
 	name: string;
+	displayName: string;
 	fileData: FormData | undefined;
 	fileCount: number;
 	isParsingFiles: boolean;

applications/client/src/store/graphql/CampaignParserModel.base.ts

@@ -0,0 +1,31 @@
+/* This is a mk-gql generated file, don't modify it manually */
+/* eslint-disable */
+/* tslint:disable */
+// @ts-nocheck
+
+import { types, prop, tProp, Model, Ref, idProp } from 'mobx-keystone';
+import { QueryBuilder } from 'mk-gql';
+
+/**
+ * CampaignParserBase
+ * auto generated base class for the model CampaignParserModel.
+ */
+export class CampaignParserModelBase extends Model({
+	__typename: tProp('CampaignParser'),
+	parserName: prop<string | null>().withSetter(),
+	path: prop<string | null>().withSetter(),
+}) {}
+
+export class CampaignParserModelSelector extends QueryBuilder {
+	get parserName() {
+		return this.__attr(`parserName`);
+	}
+	get path() {
+		return this.__attr(`path`);
+	}
+}
+export function selectFromCampaignParser() {
+	return new CampaignParserModelSelector();
+}
+
+export const campaignParserModelPrimitives = selectFromCampaignParser().parserName.path;

applications/client/src/store/graphql/CampaignParserModel.ts

@@ -0,0 +1,15 @@
+import { ExtendedModel, model } from 'mobx-keystone';
+import { CampaignParserModelBase } from './CampaignParserModel.base';
+
+/* A graphql query fragment builders for CampaignParserModel */
+export {
+	selectFromCampaignParser,
+	campaignParserModelPrimitives,
+	CampaignParserModelSelector,
+} from './CampaignParserModel.base';
+
+/**
+ * CampaignParserModel
+ */
+@model('CampaignParser')
+export class CampaignParserModel extends ExtendedModel(CampaignParserModelBase, {}) {}

applications/client/src/store/graphql/FileDisplayModel.base.ts

@@ -0,0 +1,27 @@
+/* This is a mk-gql generated file, don't modify it manually */
+/* eslint-disable */
+/* tslint:disable */
+// @ts-nocheck
+
+import { types, prop, tProp, Model, Ref, idProp } from 'mobx-keystone';
+import { QueryBuilder } from 'mk-gql';
+
+/**
+ * FileDisplayBase
+ * auto generated base class for the model FileDisplayModel.
+ */
+export class FileDisplayModelBase extends Model({
+	__typename: tProp('FileDisplay'),
+	editable: prop<boolean>().withSetter(),
+}) {}
+
+export class FileDisplayModelSelector extends QueryBuilder {
+	get editable() {
+		return this.__attr(`editable`);
+	}
+}
+export function selectFromFileDisplay() {
+	return new FileDisplayModelSelector();
+}
+
+export const fileDisplayModelPrimitives = selectFromFileDisplay().editable;

applications/client/src/store/graphql/FileDisplayModel.ts

@@ -0,0 +1,11 @@
+import { ExtendedModel, model } from 'mobx-keystone';
+import { FileDisplayModelBase } from './FileDisplayModel.base';
+
+/* A graphql query fragment builders for FileDisplayModel */
+export { selectFromFileDisplay, fileDisplayModelPrimitives, FileDisplayModelSelector } from './FileDisplayModel.base';
+
+/**
+ * FileDisplayModel
+ */
+@model('FileDisplay')
+export class FileDisplayModel extends ExtendedModel(FileDisplayModelBase, {}) {}

applications/client/src/store/graphql/FileModel.base.ts

@@ -17,7 +17,6 @@ export class FileModelBase extends Model({
 	fileFlag: prop<FileFlag>().withSetter(),
 	fileName: prop<string>().withSetter(),
 	id: prop<string>().withSetter(),
-	ip: prop<string | null>().withSetter(),
 	location: prop<string>().withSetter(),
 	/** Generated automatically when using the upload command, the MD5 message-digest algorithm is a widely used hash function producing a 128-bit hash value. */
 	md5: prop<string | null>().withSetter(),
@@ -40,9 +39,6 @@ export class FileModelSelector extends QueryBuilder {
 	get id() {
 		return this.__attr(`id`);
 	}
-	get ip() {
-		return this.__attr(`ip`);
-	}
 	get location() {
 		return this.__attr(`location`);
 	}
@@ -54,4 +50,4 @@ export function selectFromFile() {
 	return new FileModelSelector();
 }
 
-export const fileModelPrimitives = selectFromFile().dateTime.fileFlag.fileName.ip.location.md5;
+export const fileModelPrimitives = selectFromFile().dateTime.fileFlag.fileName.location.md5;