Skip to content

Commit

Permalink
feat: Add workflow logs selector support. Fixes #6910 (#7067)
Browse files Browse the repository at this point in the history
Signed-off-by: maybaby <berlinsaint@126.com>
  • Loading branch information
whybeyoung committed Oct 31, 2021
1 parent b05eaa8 commit 5eab921
Show file tree
Hide file tree
Showing 12 changed files with 301 additions and 101 deletions.
10 changes: 10 additions & 0 deletions api/openapi-spec/swagger.json
Original file line number Diff line number Diff line change
Expand Up @@ -3635,6 +3635,11 @@
"type": "string",
"name": "grep",
"in": "query"
},
{
"type": "string",
"name": "selector",
"in": "query"
}
],
"responses": {
Expand Down Expand Up @@ -4065,6 +4070,11 @@
"type": "string",
"name": "grep",
"in": "query"
},
{
"type": "string",
"name": "selector",
"in": "query"
}
],
"responses": {
Expand Down
11 changes: 9 additions & 2 deletions cmd/argo/commands/logs.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ func NewLogsCommand() *cobra.Command {
sinceTime string
tailLines int64
grep string
selector string
)
logOptions := &corev1.PodLogOptions{}
command := &cobra.Command{
Expand All @@ -37,6 +38,10 @@ func NewLogsCommand() *cobra.Command {
argo logs my-wf --follow
# Print the logs of a workflows with a selector:
argo logs my-wf -l app=sth
# Print the logs of single container in a pod
argo logs my-wf my-pod -c my-container
Expand Down Expand Up @@ -92,7 +97,7 @@ func NewLogsCommand() *cobra.Command {
serviceClient := apiClient.NewWorkflowServiceClient()
namespace := client.Namespace()

logWorkflow(ctx, serviceClient, namespace, workflow, podName, grep, logOptions)
logWorkflow(ctx, serviceClient, namespace, workflow, podName, grep, selector, logOptions)
},
}
command.Flags().StringVarP(&logOptions.Container, "container", "c", "main", "Print the logs of this container")
Expand All @@ -102,18 +107,20 @@ func NewLogsCommand() *cobra.Command {
command.Flags().StringVar(&sinceTime, "since-time", "", "Only return logs after a specific date (RFC3339). Defaults to all logs. Only one of since-time / since may be used.")
command.Flags().Int64Var(&tailLines, "tail", -1, "If set, the number of lines from the end of the logs to show. If not specified, logs are shown from the creation of the container or sinceSeconds or sinceTime")
command.Flags().StringVar(&grep, "grep", "", "grep for lines")
command.Flags().StringVarP(&selector, "selector", "l", "", "log selector for some pod")
command.Flags().BoolVar(&logOptions.Timestamps, "timestamps", false, "Include timestamps on each line in the log output")
command.Flags().BoolVar(&noColor, "no-color", false, "Disable colorized output")
return command
}

func logWorkflow(ctx context.Context, serviceClient workflowpkg.WorkflowServiceClient, namespace, workflow, podName, grep string, logOptions *corev1.PodLogOptions) {
func logWorkflow(ctx context.Context, serviceClient workflowpkg.WorkflowServiceClient, namespace, workflow, podName, grep, selector string, logOptions *corev1.PodLogOptions) {
// logs
stream, err := serviceClient.WorkflowLogs(ctx, &workflowpkg.WorkflowLogRequest{
Name: workflow,
Namespace: namespace,
PodName: podName,
LogOptions: logOptions,
Selector: selector,
Grep: grep,
})
errors.CheckError(err)
Expand Down
2 changes: 1 addition & 1 deletion cmd/argo/commands/submit.go
Original file line number Diff line number Diff line change
Expand Up @@ -258,7 +258,7 @@ func unmarshalWorkflows(wfBytes []byte, strict bool) []wfv1.Workflow {
func waitWatchOrLog(ctx context.Context, serviceClient workflowpkg.WorkflowServiceClient, namespace string, workflowNames []string, cliSubmitOpts cliSubmitOpts) {
if cliSubmitOpts.log {
for _, workflow := range workflowNames {
logWorkflow(ctx, serviceClient, namespace, workflow, "", "", &corev1.PodLogOptions{
logWorkflow(ctx, serviceClient, namespace, workflow, "", "", "", &corev1.PodLogOptions{
Container: common.MainContainerName,
Follow: true,
Previous: false,
Expand Down
5 changes: 5 additions & 0 deletions docs/cli/argo_logs.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,10 @@ argo logs WORKFLOW [POD] [flags]
argo logs my-wf --follow
# Print the logs of a workflows with a selector:
argo logs my-wf -l app=sth
# Print the logs of single container in a pod
argo logs my-wf my-pod -c my-container
Expand All @@ -43,6 +47,7 @@ argo logs WORKFLOW [POD] [flags]
-h, --help help for logs
--no-color Disable colorized output
-p, --previous Specify if the previously terminated container logs should be returned.
-l, --selector string log selector for some pod
--since duration Only return logs newer than a relative duration like 5s, 2m, or 3h. Defaults to all logs. Only one of since-time / since may be used.
--since-time string Only return logs after a specific date (RFC3339). Defaults to all logs. Only one of since-time / since may be used.
--tail int If set, the number of lines from the end of the logs to show. If not specified, logs are shown from the creation of the container or sinceSeconds or sinceTime (default -1)
Expand Down
28 changes: 28 additions & 0 deletions docs/fields.md
Original file line number Diff line number Diff line change
Expand Up @@ -180,6 +180,8 @@ Workflow is the definition of a workflow resource

- [`k8s-jobs.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/k8s-jobs.yaml)

- [`k8s-kubeflow-jobs.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/k8s-kubeflow-jobs.yaml)

- [`k8s-orchestration.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/k8s-orchestration.yaml)

- [`k8s-owner-reference.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/k8s-owner-reference.yaml)
Expand Down Expand Up @@ -595,6 +597,8 @@ WorkflowSpec is the specification of a Workflow.

- [`k8s-jobs.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/k8s-jobs.yaml)

- [`k8s-kubeflow-jobs.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/k8s-kubeflow-jobs.yaml)

- [`k8s-orchestration.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/k8s-orchestration.yaml)

- [`k8s-owner-reference.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/k8s-owner-reference.yaml)
Expand Down Expand Up @@ -1028,6 +1032,8 @@ CronWorkflowSpec is the specification of a CronWorkflow

- [`k8s-jobs.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/k8s-jobs.yaml)

- [`k8s-kubeflow-jobs.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/k8s-kubeflow-jobs.yaml)

- [`k8s-orchestration.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/k8s-orchestration.yaml)

- [`k8s-owner-reference.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/k8s-owner-reference.yaml)
Expand Down Expand Up @@ -1417,6 +1423,8 @@ WorkflowTemplateSpec is a spec of WorkflowTemplate.

- [`k8s-jobs.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/k8s-jobs.yaml)

- [`k8s-kubeflow-jobs.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/k8s-kubeflow-jobs.yaml)

- [`k8s-orchestration.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/k8s-orchestration.yaml)

- [`k8s-owner-reference.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/k8s-owner-reference.yaml)
Expand Down Expand Up @@ -2217,6 +2225,8 @@ Outputs hold parameters, artifacts, and results from a step

- [`k8s-jobs.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/k8s-jobs.yaml)

- [`k8s-kubeflow-jobs.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/k8s-kubeflow-jobs.yaml)

- [`k8s-orchestration.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/k8s-orchestration.yaml)

- [`k8s-wait-wf.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/k8s-wait-wf.yaml)
Expand Down Expand Up @@ -2467,6 +2477,8 @@ Parameter indicate a passed string parameter to a service template with an optio

- [`k8s-jobs.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/k8s-jobs.yaml)

- [`k8s-kubeflow-jobs.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/k8s-kubeflow-jobs.yaml)

- [`k8s-orchestration.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/k8s-orchestration.yaml)

- [`k8s-wait-wf.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/k8s-wait-wf.yaml)
Expand Down Expand Up @@ -3110,6 +3122,8 @@ ResourceTemplate is a template subtype to manipulate kubernetes resources

- [`k8s-jobs.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/k8s-jobs.yaml)

- [`k8s-kubeflow-jobs.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/k8s-kubeflow-jobs.yaml)

- [`k8s-orchestration.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/k8s-orchestration.yaml)

- [`k8s-owner-reference.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/k8s-owner-reference.yaml)
Expand Down Expand Up @@ -3803,6 +3817,8 @@ ValueFrom describes a location in which to obtain the value to a parameter

- [`k8s-jobs.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/k8s-jobs.yaml)

- [`k8s-kubeflow-jobs.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/k8s-kubeflow-jobs.yaml)

- [`k8s-orchestration.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/k8s-orchestration.yaml)

- [`k8s-wait-wf.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/k8s-wait-wf.yaml)
Expand Down Expand Up @@ -3947,6 +3963,8 @@ MetricLabel is a single label for a prometheus metric

- [`http-hello-world.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/http-hello-world.yaml)

- [`k8s-kubeflow-jobs.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/k8s-kubeflow-jobs.yaml)

- [`k8s-owner-reference.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/k8s-owner-reference.yaml)

- [`k8s-patch-basic.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/k8s-patch-basic.yaml)
Expand Down Expand Up @@ -4000,6 +4018,8 @@ _No description available_

- [`k8s-jobs.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/k8s-jobs.yaml)

- [`k8s-kubeflow-jobs.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/k8s-kubeflow-jobs.yaml)

- [`k8s-orchestration.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/k8s-orchestration.yaml)

- [`pod-spec-patch-wf-tmpl.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/pod-spec-patch-wf-tmpl.yaml)
Expand Down Expand Up @@ -4644,6 +4664,8 @@ _No description available_

- [`k8s-jobs.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/k8s-jobs.yaml)

- [`k8s-kubeflow-jobs.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/k8s-kubeflow-jobs.yaml)

- [`k8s-orchestration.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/k8s-orchestration.yaml)

- [`k8s-wait-wf.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/k8s-wait-wf.yaml)
Expand Down Expand Up @@ -4869,6 +4891,8 @@ ObjectMeta is metadata that all persisted resources must have, which includes al

- [`k8s-jobs.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/k8s-jobs.yaml)

- [`k8s-kubeflow-jobs.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/k8s-kubeflow-jobs.yaml)

- [`k8s-orchestration.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/k8s-orchestration.yaml)

- [`k8s-owner-reference.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/k8s-owner-reference.yaml)
Expand Down Expand Up @@ -6180,6 +6204,8 @@ PersistentVolumeClaimSpec describes the common attributes of storage devices and

- [`k8s-jobs.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/k8s-jobs.yaml)

- [`k8s-kubeflow-jobs.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/k8s-kubeflow-jobs.yaml)

- [`k8s-orchestration.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/k8s-orchestration.yaml)

- [`k8s-owner-reference.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/k8s-owner-reference.yaml)
Expand Down Expand Up @@ -6803,6 +6829,8 @@ EnvVarSource represents a source for the value of an EnvVar.

- [`k8s-jobs.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/k8s-jobs.yaml)

- [`k8s-kubeflow-jobs.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/k8s-kubeflow-jobs.yaml)

- [`k8s-orchestration.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/k8s-orchestration.yaml)

- [`k8s-wait-wf.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/k8s-wait-wf.yaml)
Expand Down
66 changes: 66 additions & 0 deletions examples/k8s-kubeflow-jobs.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
# This example demonstrates the 'resource' template type, which provides a
# convenient way to create/update/delete any type of kubernetes resources
# in a workflow. The resource template type accepts any k8s manifest
# (including CRDs) and can perform any kubectl action against it (e.g. create,
# apply, delete, patch).
apiVersion: argoproj.io/v1alpha1
kind: Workflow
metadata:
generateName: k8s-kubeflow-jobs-
spec:
entrypoint: tf-jobtmpl
templates:
- name: tf-jobtmpl
resource:
action: create
# successCondition and failureCondition are optional expressions which are
# evaluated upon every update of the resource. If failureCondition is ever
# evaluated to true, the step is considered failed. Likewise, if successCondition
# is ever evaluated to true the step is considered successful. It uses kubernetes
# label selection syntax and can be applied against any field of the resource
# (not just labels). Multiple AND conditions can be represented by comma
# delimited expressions. For more details, see:
# https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/
successCondition: status.succeeded = 2
failureCondition: status.failed > 0
# You can also create Any other distributed k8s resource here. This is an example
# that using KubeFlow TFJob. See more comment in the below. ↓
manifest: |
apiVersion: kubeflow.org/v1
kind: TFJob
metadata:
name: tfjob-examples
spec:
tfReplicaSpecs:
Worker:
replicas: 2
restartPolicy: Never
template:
metadata:
# This label is important, so that you can use `argo logs -c tensorflow` to see the pod's logs.
# workflow.name is global varaibles that argo support.
# And the TFJob controller crd will automatically add worker pod with some labels ,such as
# job-role, replica-index. then you can use `argo logs -c tensorflow -l replica-index=0` to see
# work-0's podlogs.
# To see detail, please refer to: https://github.com/argoproj/argo-workflows/pull/7067
labels:
workflows.argoproj.io/workflow: {{workflow.name}}
spec:
containers:
- name: tensorflow
image: "your image here"
# Resource templates can have output parameters extracted from fields of the
# resource. Two techniques are provided: jsonpath and a jq filter.
outputs:
parameters:
# job-name is extracted using a jsonPath expression and is equivalent to:
# `kubectl get job <jobname> -o jsonpath='{.metadata.name}'`
- name: job-name
valueFrom:
jsonPath: '{.metadata.name}'
# job-obj is extracted using a jq filter and is equivalent to:
# `kubectl get job <jobname> -o json | jq -c '.'
# which returns the entire job object in json format
- name: job-obj
valueFrom:
jqFilter: '.'

0 comments on commit 5eab921

Please sign in to comment.