Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion codex-rs/analytics/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ codex-plugin = { workspace = true }
codex-protocol = { workspace = true }
os_info = { workspace = true }
serde = { workspace = true, features = ["derive"] }
serde_json = { workspace = true }
sha1 = { workspace = true }
tokio = { workspace = true, features = [
"macros",
Expand All @@ -29,4 +30,3 @@ tracing = { workspace = true, features = ["log"] }

[dev-dependencies]
pretty_assertions = { workspace = true }
serde_json = { workspace = true }
7 changes: 7 additions & 0 deletions codex-rs/analytics/src/client.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
use crate::events::AppServerRpcTransport;
use crate::events::GuardianReviewEventParams;
use crate::events::TrackEventRequest;
use crate::events::TrackEventsRequest;
use crate::events::current_runtime_metadata;
Expand Down Expand Up @@ -151,6 +152,12 @@ impl AnalyticsEventsClient {
));
}

pub fn track_guardian_review(&self, input: GuardianReviewEventParams) {
self.record_fact(AnalyticsFact::Custom(CustomAnalyticsFact::GuardianReview(
Box::new(input),
)));
}

pub fn track_app_mentioned(&self, tracking: TrackEventsContext, mentions: Vec<AppInvocation>) {
if mentions.is_empty() {
return;
Expand Down
177 changes: 177 additions & 0 deletions codex-rs/analytics/src/events.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,9 @@ use crate::facts::SubAgentThreadStartedInput;
use crate::facts::TrackEventsContext;
use codex_login::default_client::originator;
use codex_plugin::PluginTelemetryMetadata;
use codex_protocol::approvals::NetworkApprovalProtocol;
use codex_protocol::models::PermissionProfile;
use codex_protocol::models::SandboxPermissions;
use codex_protocol::protocol::SessionSource;
use codex_protocol::protocol::SubAgentSource;
use serde::Serialize;
Expand Down Expand Up @@ -36,6 +39,7 @@ pub(crate) struct TrackEventsRequest {
pub(crate) enum TrackEventRequest {
SkillInvocation(SkillInvocationEventRequest),
ThreadInitialized(ThreadInitializedEvent),
GuardianReview(Box<GuardianReviewEventRequest>),
AppMentioned(CodexAppMentionedEventRequest),
AppUsed(CodexAppUsedEventRequest),
Compaction(Box<CodexCompactionEventRequest>),
Expand Down Expand Up @@ -101,6 +105,179 @@ pub(crate) struct ThreadInitializedEvent {
pub(crate) event_params: ThreadInitializedEventParams,
}

#[derive(Serialize)]
pub(crate) struct GuardianReviewEventRequest {
pub(crate) event_type: &'static str,
pub(crate) event_params: GuardianReviewEventPayload,
}

#[derive(Clone, Copy, Debug, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum GuardianReviewDecision {
Approved,
Denied,
Aborted,
}

#[derive(Clone, Copy, Debug, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum GuardianReviewTerminalStatus {
Approved,
Denied,
Aborted,
TimedOut,
FailedClosed,
}

#[derive(Clone, Copy, Debug, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum GuardianReviewFailureReason {
Timeout,
Cancelled,
PromptBuildError,
SessionError,
ParseError,
}

#[derive(Clone, Copy, Debug, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum GuardianReviewSessionKind {
TrunkNew,
TrunkReused,
EphemeralForked,
}

#[derive(Clone, Copy, Debug, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum GuardianReviewRiskLevel {
Low,
Medium,
High,
Critical,
}

#[derive(Clone, Copy, Debug, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum GuardianReviewUserAuthorization {
Unknown,
Low,
Medium,
High,
}

#[derive(Clone, Copy, Debug, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum GuardianReviewOutcome {
Allow,
Deny,
}

#[derive(Clone, Copy, Debug, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum GuardianApprovalRequestSource {
/// Approval requested directly by the main Codex turn.
MainTurn,
/// Approval requested by a delegated subagent and routed through the parent
/// session for guardian review.
DelegatedSubagent,
}

#[derive(Clone, Debug, Serialize)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum GuardianReviewedAction {
Shell {
command: Vec<String>,
command_display: String,
cwd: String,
sandbox_permissions: SandboxPermissions,
additional_permissions: Option<PermissionProfile>,
justification: Option<String>,
},
UnifiedExec {
command: Vec<String>,
command_display: String,
cwd: String,
sandbox_permissions: SandboxPermissions,
additional_permissions: Option<PermissionProfile>,
justification: Option<String>,
tty: bool,
},
Execve {
source: GuardianCommandSource,
program: String,
argv: Vec<String>,
cwd: String,
additional_permissions: Option<PermissionProfile>,
},
ApplyPatch {
cwd: String,
files: Vec<String>,
},
NetworkAccess {
target: String,
host: String,
protocol: NetworkApprovalProtocol,
port: u16,
},
McpToolCall {
server: String,
tool_name: String,
connector_id: Option<String>,
connector_name: Option<String>,
tool_title: Option<String>,
},
}

#[derive(Clone, Copy, Debug, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum GuardianCommandSource {
Shell,
UnifiedExec,
}

#[derive(Clone, Serialize)]
pub struct GuardianReviewEventParams {
pub thread_id: String,
pub turn_id: String,
pub review_id: String,
pub target_item_id: String,
Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

is this approval_id?

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

maps to these:

/// Stable identifier for this review.
pub review_id: String,
/// Identifier for the reviewed item or tool call when one exists.
///
/// In most cases, one review maps to one target item. The exceptions are
/// - execve reviews, where a single command may contain multiple execve
/// calls to review (only possible when using the shell_zsh_fork feature)
/// - network policy reviews, where there is no target item
///
/// A network call is triggered by a CommandExecution item, so having a
/// target_item_id set to the CommandExecution item would be misleading
/// because the review is about the network call, not the command execution.
/// Therefore, target_item_id is set to None for network policy reviews.
pub target_item_id: Option<String>,

pub retry_reason: Option<String>,
pub approval_request_source: GuardianApprovalRequestSource,
pub reviewed_action: GuardianReviewedAction,
pub reviewed_action_truncated: bool,
pub decision: GuardianReviewDecision,
pub terminal_status: GuardianReviewTerminalStatus,
pub failure_reason: Option<GuardianReviewFailureReason>,
pub risk_level: Option<GuardianReviewRiskLevel>,
pub user_authorization: Option<GuardianReviewUserAuthorization>,
pub outcome: Option<GuardianReviewOutcome>,
pub rationale: Option<String>,
pub guardian_thread_id: Option<String>,
pub guardian_session_kind: Option<GuardianReviewSessionKind>,
pub guardian_model: Option<String>,
pub guardian_reasoning_effort: Option<String>,
pub had_prior_review_context: Option<bool>,
pub review_timeout_ms: u64,
pub tool_call_count: u64,
pub time_to_first_token_ms: Option<u64>,
pub completion_latency_ms: Option<u64>,
pub started_at: u64,
pub completed_at: Option<u64>,
pub input_tokens: Option<i64>,
pub cached_input_tokens: Option<i64>,
pub output_tokens: Option<i64>,
pub reasoning_output_tokens: Option<i64>,
pub total_tokens: Option<i64>,
}

#[derive(Serialize)]
pub(crate) struct GuardianReviewEventPayload {
pub(crate) app_server_client: CodexAppServerClientMetadata,
pub(crate) runtime: CodexRuntimeMetadata,
#[serde(flatten)]
pub(crate) guardian_review: GuardianReviewEventParams,
}

#[derive(Serialize)]
pub(crate) struct CodexAppMetadata {
pub(crate) connector_id: Option<String>,
Expand Down
2 changes: 2 additions & 0 deletions codex-rs/analytics/src/facts.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
use crate::events::AppServerRpcTransport;
use crate::events::CodexRuntimeMetadata;
use crate::events::GuardianReviewEventParams;
use codex_app_server_protocol::ClientRequest;
use codex_app_server_protocol::ClientResponse;
use codex_app_server_protocol::InitializeParams;
Expand Down Expand Up @@ -154,6 +155,7 @@ pub(crate) enum AnalyticsFact {
pub(crate) enum CustomAnalyticsFact {
SubAgentThreadStarted(SubAgentThreadStartedInput),
Compaction(Box<CodexCompactionEvent>),
GuardianReview(Box<GuardianReviewEventParams>),
SkillInvoked(SkillInvokedInput),
AppMentioned(AppMentionedInput),
AppUsed(AppUsedInput),
Expand Down
11 changes: 11 additions & 0 deletions codex-rs/analytics/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,17 @@ mod reducer;

pub use client::AnalyticsEventsClient;
pub use events::AppServerRpcTransport;
pub use events::GuardianApprovalRequestSource;
pub use events::GuardianCommandSource;
pub use events::GuardianReviewDecision;
pub use events::GuardianReviewEventParams;
pub use events::GuardianReviewFailureReason;
pub use events::GuardianReviewOutcome;
pub use events::GuardianReviewRiskLevel;
pub use events::GuardianReviewSessionKind;
pub use events::GuardianReviewTerminalStatus;
pub use events::GuardianReviewUserAuthorization;
pub use events::GuardianReviewedAction;
pub use facts::AppInvocation;
pub use facts::CodexCompactionEvent;
pub use facts::CompactionImplementation;
Expand Down
42 changes: 42 additions & 0 deletions codex-rs/analytics/src/reducer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,9 @@ use crate::events::CodexCompactionEventRequest;
use crate::events::CodexPluginEventRequest;
use crate::events::CodexPluginUsedEventRequest;
use crate::events::CodexRuntimeMetadata;
use crate::events::GuardianReviewEventParams;
use crate::events::GuardianReviewEventPayload;
use crate::events::GuardianReviewEventRequest;
use crate::events::SkillInvocationEventParams;
use crate::events::SkillInvocationEventRequest;
use crate::events::ThreadInitializationMode;
Expand Down Expand Up @@ -120,6 +123,9 @@ impl AnalyticsReducer {
CustomAnalyticsFact::Compaction(input) => {
self.ingest_compaction(*input, out);
}
CustomAnalyticsFact::GuardianReview(input) => {
self.ingest_guardian_review(*input, out);
}
CustomAnalyticsFact::SkillInvoked(input) => {
self.ingest_skill_invoked(input, out).await;
}
Expand Down Expand Up @@ -174,6 +180,42 @@ impl AnalyticsReducer {
));
}

fn ingest_guardian_review(
&mut self,
input: GuardianReviewEventParams,
out: &mut Vec<TrackEventRequest>,
) {
let Some(connection_id) = self.thread_connections.get(&input.thread_id) else {
tracing::warn!(
thread_id = %input.thread_id,
turn_id = %input.turn_id,
review_id = %input.review_id,
"dropping guardian analytics event: missing thread connection metadata"
);
return;
};
let Some(connection_state) = self.connections.get(connection_id) else {
tracing::warn!(
thread_id = %input.thread_id,
turn_id = %input.turn_id,
review_id = %input.review_id,
connection_id,
"dropping guardian analytics event: missing connection metadata"
);
return;
};
out.push(TrackEventRequest::GuardianReview(Box::new(
GuardianReviewEventRequest {
event_type: "codex_guardian_review",
event_params: GuardianReviewEventPayload {
app_server_client: connection_state.app_server_client.clone(),
runtime: connection_state.runtime.clone(),
guardian_review: input,
},
},
)));
}

async fn ingest_skill_invoked(
&mut self,
input: SkillInvokedInput,
Expand Down
2 changes: 1 addition & 1 deletion codex-rs/core/src/guardian/review_session.rs
Original file line number Diff line number Diff line change
Expand Up @@ -414,7 +414,7 @@ impl GuardianReviewSessionManager {
let snapshot = state.last_committed_fork_snapshot.as_ref()?;
match &snapshot.initial_history {
InitialHistory::Forked(items) => Some(items.clone()),
InitialHistory::New | InitialHistory::Resumed(_) => None,
_ => None,
}
}

Expand Down
Loading