From 2db41dfe438d2d169fff2967be59941cd7955e16 Mon Sep 17 00:00:00 2001 From: Yuji Ito Date: Sat, 4 Feb 2023 23:48:50 +0900 Subject: [PATCH] wip Signed-off-by: Yuji Ito --- agent/cri/driver.go | 101 ++++++++ agent/cri/types.go | 190 +++++++++++++++ cmd/agent/main.go | 2 + src/cri.ts | 575 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 868 insertions(+) create mode 100644 agent/cri/driver.go create mode 100644 agent/cri/types.go create mode 100644 src/cri.ts diff --git a/agent/cri/driver.go b/agent/cri/driver.go new file mode 100644 index 0000000..61d9c3b --- /dev/null +++ b/agent/cri/driver.go @@ -0,0 +1,101 @@ +package cri + +import ( + "encoding/json" + "strings" + + "github.com/llamerada-jp/oinari/agent/crosslink" +) + +const ( + crosslinkPath = "cri" +) + +type criImpl struct { + cl crosslink.Crosslink +} + +func NewCRI(cl crosslink.Crosslink) CRI { + return &criImpl{ + cl: cl, + } +} + +func criCallHelper[REQ any, RES any](ci *criImpl, path string, request *REQ) (*RES, error) { + type ResErr struct { + res *RES + err error + } + + reqJson, err := json.Marshal(request) + if err != nil { + return nil, err + } + + ch := make(chan *ResErr) + + ci.cl.Call(string(reqJson), map[string]string{ + crosslink.TAG_PATH: strings.Join([]string{crosslinkPath, path}, "/"), + }, func(result string, err error) { + if err != nil { + ch <- &ResErr{nil, err} + return + } + + var res RES + err = json.Unmarshal([]byte(result), &res) + if err != nil { + ch <- &ResErr{nil, err} + return + } + + ch <- &ResErr{&res, nil} + }) + + resErr := <-ch + return resErr.res, resErr.err +} + +func (ci *criImpl) RunPodSandbox(request *RunPodSandboxRequest) (*RunPodSandboxResponse, error) { + return criCallHelper[RunPodSandboxRequest, RunPodSandboxResponse](ci, "runPodSandbox", request) +} + +func (ci *criImpl) StopPodSandbox(request *StopPodSandboxRequest) (*StopPodSandboxResponse, error) { + return criCallHelper[StopPodSandboxRequest, StopPodSandboxResponse](ci, "stopPodSandbox", request) +} + +func (ci *criImpl) RemovePodSandbox(request *RemovePodSandboxRequest) (*RemovePodSandboxResponse, error) { + return criCallHelper[RemovePodSandboxRequest, RemovePodSandboxResponse](ci, "removePodSandbox", request) +} + +func (ci *criImpl) PodSandboxStatus(request *PodSandboxStatusRequest) (*PodSandboxStatusResponse, error) { + return criCallHelper[PodSandboxStatusRequest, PodSandboxStatusResponse](ci, "podSandboxStatus", request) +} + +func (ci *criImpl) CreateContainer(request *CreateContainerRequest) (*CreateContainerResponse, error) { + return criCallHelper[CreateContainerRequest, CreateContainerResponse](ci, "createContainer", request) +} + +func (ci *criImpl) StartContainer(request *StartContainerRequest) (*StartContainerResponse, error) { + return criCallHelper[StartContainerRequest, StartContainerResponse](ci, "startContainer", request) +} + +func (ci *criImpl) StopContainer(request *StopContainerRequest) (*StopContainerResponse, error) { + return criCallHelper[StopContainerRequest, StopContainerResponse](ci, "stopContainer", request) +} + +func (ci *criImpl) RemoveContainer(request *RemoveContainerRequest) (*RemoveContainerResponse, error) { + return criCallHelper[RemoveContainerRequest, RemoveContainerResponse](ci, "removeContainer", request) +} + +func (ci *criImpl) ListImages(request *ListImagesRequest) (*ListImagesResponse, error) { + return criCallHelper[ListImagesRequest, ListImagesResponse](ci, "listImages", request) +} + +func (ci *criImpl) PullImage(request *PullImageRequest) (*PullImageResponse, error) { + return criCallHelper[PullImageRequest, PullImageResponse](ci, "pullImage", request) +} + +func (ci *criImpl) RemoveImage(request *RemoveImageRequest) (*RemoveImageResponse, error) { + return criCallHelper[RemoveImageRequest, RemoveImageResponse](ci, "removeImage", request) +} diff --git a/agent/cri/types.go b/agent/cri/types.go new file mode 100644 index 0000000..3f32f3e --- /dev/null +++ b/agent/cri/types.go @@ -0,0 +1,190 @@ +package cri + +/** + * This interface is partial mimic of Kubernetes cri-api. And there are some differences + * caused by Oinari using WASM on the web browsers. Oinari implements the interface + * using crosslink internally without gRPC because it is difficult to implement it + * using gRPC between Go(WASM) and TypeScript via web worker. + * ref: https://github.com/kubernetes/cri-api + */ +type CRI interface { + // apis for sandbox + RunPodSandbox(*RunPodSandboxRequest) (*RunPodSandboxResponse, error) + StopPodSandbox(*StopPodSandboxRequest) (*StopPodSandboxResponse, error) + RemovePodSandbox(*RemovePodSandboxRequest) (*RemovePodSandboxResponse, error) + PodSandboxStatus(*PodSandboxStatusRequest) (*PodSandboxStatusResponse, error) + + // apis for container + CreateContainer(*CreateContainerRequest) (*CreateContainerResponse, error) + StartContainer(*StartContainerRequest) (*StartContainerResponse, error) + StopContainer(*StopContainerRequest) (*StopContainerResponse, error) + RemoveContainer(*RemoveContainerRequest) (*RemoveContainerResponse, error) + + // apis for image + ListImages(*ListImagesRequest) (*ListImagesResponse, error) + PullImage(*PullImageRequest) (*PullImageResponse, error) + RemoveImage(*RemoveImageRequest) (*RemoveImageResponse, error) +} + +type RunPodSandboxRequest struct { + Config PodSandboxConfig `json:"config"` +} + +type PodSandboxConfig struct { + Metadata PodSandboxMetadata `json:"metadata"` +} + +type PodSandboxMetadata struct { + Name string `json:"name"` + // UID is equal to Pod UID in the Pod ObjectMeta. This value must be global unique in the Oinari system. + UID string `json:"uid"` + Namespace string `json:"namespace"` +} + +type RunPodSandboxResponse struct { + // PodSandboxID is not equal to UID of PodSandboxMetadata. This value used in node local and unique in only the node. + PodSandboxID string `json:"pod_sandbox_id"` +} + +type StopPodSandboxRequest struct { + PodSandboxID string `json:"pod_sandbox_id"` +} + +type StopPodSandboxResponse struct { + // empty +} + +type RemovePodSandboxRequest struct { + PodSandboxID string `json:"pod_sandbox_id"` +} + +type RemovePodSandboxResponse struct { + // empty +} + +type PodSandboxStatusRequest struct { + PodSandboxID string `json:"pod_sandbox_id"` +} + +type PodSandboxStatusResponse struct { + Status PodSandboxStatus `json:"status"` + ContainersStatuses []ContainerStatus `json:"containers_statuses "` + Timestamp string `json:"timestamp"` +} + +type PodSandboxStatus struct { + ID string `json:"id"` + Metadata PodSandboxMetadata `json:"metadata"` + State PodSandboxState `json:"state"` + CreatedAt string `json:"created_at"` +} + +type PodSandboxState int + +const ( + SandboxReady PodSandboxState = iota + SandboxNotReady +) + +type CreateContainerRequest struct { + PodSandboxID string `json:"pod_sandbox_id"` + Config ContainerConfig `json:"config"` + SandboxConfig PodSandboxConfig `json:"sandbox_config"` +} + +type ContainerConfig struct { + Metadata ContainerMetadata `json:"metadata"` + Image ImageSpec `json:"image"` +} + +type ContainerMetadata struct { + Name string `json:"name"` +} + +type CreateContainerResponse struct { + ContainerID string `json:"container_id"` +} + +type StartContainerRequest struct { + ContainerID string `json:"container_id"` +} + +type StartContainerResponse struct { + // empty +} + +type StopContainerRequest struct { + ContainerID string `json:"container_id"` +} + +type StopContainerResponse struct { + // empty +} + +type RemoveContainerRequest struct { + ContainerID string `json:"container_id"` +} + +type RemoveContainerResponse struct { + // empty +} + +type ContainerStatus struct { + ID string `json:"id"` + Metadata ContainerMetadata `json:"metadata"` + State ContainerState `json:"state"` + CreatedAt string `json:"created_at"` + StartedAt string `json:"started_at"` + FinishedAt string `json:"finished_at"` + ExitCode int32 `json:"exit_code"` + Image ImageSpec `json:"image"` + ImageRef string `json:"image_ref"` +} + +type ContainerState int + +const ( + ContainerCreated ContainerState = iota + ContainerRunning + ContainerExited + ContainerUnknown +) + +type ListImagesRequest struct { + Filter ImageFilter `json:"filter"` +} + +type ImageFilter struct { + Image ImageSpec `json:"image"` +} + +type ImageSpec struct { + Image string `json:"image"` +} + +type ListImagesResponse struct { + Images []Image `json:"images"` +} + +type Image struct { + ID string `json:"id"` + Spec ImageSpec `json:"spec"` + // this field meaning the runtime environment of wasm, like 'go:1.19' + Runtime string `json:"runtime"` +} + +type PullImageRequest struct { + Image ImageSpec `json:"image"` +} + +type PullImageResponse struct { + ImageRef string `json:"image_ref"` +} + +type RemoveImageRequest struct { + Image ImageSpec `json:"image"` +} + +type RemoveImageResponse struct { + // nothing +} diff --git a/cmd/agent/main.go b/cmd/agent/main.go index b7d2d0a..19da860 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -24,6 +24,7 @@ import ( "github.com/llamerada-jp/colonio/go/colonio" "github.com/llamerada-jp/oinari/agent/core" + "github.com/llamerada-jp/oinari/agent/cri" "github.com/llamerada-jp/oinari/agent/crosslink" "github.com/llamerada-jp/oinari/agent/global" "github.com/llamerada-jp/oinari/agent/local" @@ -131,6 +132,7 @@ func main() { gcd := global.NewCommandDriver(col) lcd := local.NewCommandDriver(cl) + cri := cri.NewCRI(cl) seh := newSystemEventHandler(col) sys := core.NewSystem(col, seh, gcd, lcd) diff --git a/src/cri.ts b/src/cri.ts new file mode 100644 index 0000000..f47c475 --- /dev/null +++ b/src/cri.ts @@ -0,0 +1,575 @@ +import * as CL from "./crosslink"; + +const crosslinkPath: string = "cri"; +const podSandboxIDMax: number = Math.floor(Math.pow(2, 30)); +const containerIDMax: number = Math.floor(Math.pow(2, 30)); +const imageRefIDMax: number = Math.floor(Math.pow(2, 30)); + +// key means PodSandboxID +let sandboxes: Map = new Map(); +// key means ContainerID +let containers: Map = new Map(); +// key means image url (not image ref id) +let imageByURL: Map = new Map(); +let images: Array> = new Array(); + +function getTimestamp(): string { + return new Date().toISOString(); +} + +enum ImageState { + Created = 0, + Downloading = 1, + Downloaded = 2, + Error = 3, +} + +class ImageInstance { + state: ImageState + // image ref id + id: string + url: string + runtime: string + image: ArrayBuffer | undefined + + constructor(id: string, url: string) { + this.state = ImageState.Created; + this.id = id; + this.url = url; + this.runtime = ""; + this.reload(); + } + + reload() { + console.assert(this.state !== ImageState.Downloading, "duplicate download"); + + this.state = ImageState.Downloading; + + fetch(this.url).then((response: Response) => { + if (!response.ok) { + throw new Error(response.statusText); + } + return response.json(); + + }).then((p) => { + let param = p as { + image: string + runtime: string + }; + + // TODO check runtime + this.runtime = param.runtime; + + return fetch(param.image); + + }).then((response: Response) => { + if (!response.ok || response.status !== 200) { + throw new Error(response.statusText); + } + + return response.arrayBuffer(); + + }).then((buffer) => { + if (buffer == null) { + throw new Error("array buffer shouldn't be null"); + } + this.state = ImageState.Downloaded; + this.image = buffer; + + }).catch((reason) => { + this.state = ImageState.Error; + console.error(reason); + }); + } +} + +class Container { + id: string + // PodSandboxID + sandbox_id: string + name: string + image: ImageInstance + created_at: string + started_at: string | undefined + finished_at: string | undefined + exit_code: number | undefined + + constructor(id: string, sandbox_id: string, name: string, image: ImageInstance) { + this.id = id; + this.sandbox_id = sandbox_id; + this.name = name; + this.image = image; + this.created_at = getTimestamp(); + } + + getState(): ContainerState { + if (this.finished_at != null) { + return ContainerState.ContainerExited; + } + if (this.started_at != null) { + return ContainerState.ContainerRunning; + } + return ContainerState.ContainerCreated; + } + + start() { + console.warn("fix it"); + } + + stop() { + console.warn("fix it"); + } +} + +class Sandbox { + name: string + uid: string + namespace: string + containers: Map + created_at: string + + constructor(name: string, uid: string, namespace: string) { + this.name = name; + this.uid = uid; + this.namespace = namespace; + this.containers = new Map(); + this.created_at = getTimestamp(); + } + + stop() { + for (const [_, container] of this.containers) { + container.stop(); + } + this.containers.clear(); + } + + createContainer(name: string, image: ImageInstance): string { + let id: string = (Math.floor(Math.random() * containerIDMax)).toString(16); + while (containers.has(id)) { + id = (Math.floor(Math.random() * containerIDMax)).toString(16); + } + + let container = new Container(id, this.uid, name, image); + containers.set(id, container); + this.containers.set(id, container); + + return id; + } + + removeContainer(id: string) { + containers.delete(id); + } +} + +export function initCRI(rootMpx: CL.MultiPlexer): void { + initHandler(rootMpx); +} + +function initHandler(rootMpx: CL.MultiPlexer) { + let mpx = new CL.MultiPlexer(); + rootMpx.setHandler(crosslinkPath, mpx); + + mpx.setObjHandlerFunc("runPodSandbox", (data: any, _: Map, writer: CL.ResponseObjWriter): void => { + let res = runPodSandbox(data as RunPodSandboxRequest); + writer.replySuccess(res); + }); + + mpx.setObjHandlerFunc("stopPodSandbox", (data: any, _: Map, writer: CL.ResponseObjWriter): void => { + let res = stopPodSandbox(data as StopPodSandboxRequest); + writer.replySuccess(res); + }); + + mpx.setObjHandlerFunc("removePodSandbox", (data: any, _: Map, writer: CL.ResponseObjWriter): void => { + let res = removePodSandbox(data as RemovePodSandboxRequest); + writer.replySuccess(res); + }); + + mpx.setObjHandlerFunc("podSandboxStatus", (data: any, _: Map, writer: CL.ResponseObjWriter): void => { + let res = podSandboxStatus(data as PodSandboxStatusRequest); + writer.replySuccess(res); + }); + + mpx.setObjHandlerFunc("createContainer", (data: any, _: Map, writer: CL.ResponseObjWriter): void => { + let res = createContainer(data as CreateContainerRequest); + writer.replySuccess(res); + }); + + mpx.setObjHandlerFunc("startContainer", (data: any, _: Map, writer: CL.ResponseObjWriter): void => { + let res = startContainer(data as StartContainerRequest); + writer.replySuccess(res); + }); + + mpx.setObjHandlerFunc("stopContainer", (data: any, _: Map, writer: CL.ResponseObjWriter): void => { + let res = stopContainer(data as StopContainerRequest); + writer.replySuccess(res); + }); + + mpx.setObjHandlerFunc("removeContainer", (data: any, _: Map, writer: CL.ResponseObjWriter): void => { + let res = removeContainer(data as RemoveContainerRequest); + writer.replySuccess(res); + }); + + mpx.setObjHandlerFunc("listImages", (data: any, _: Map, writer: CL.ResponseObjWriter): void => { + let res = listImages(data as ListImagesRequest); + writer.replySuccess(res); + }); + + mpx.setObjHandlerFunc("pullImage", (data: any, _: Map, writer: CL.ResponseObjWriter): void => { + let res = pullImage(data as PullImageRequest); + writer.replySuccess(res); + }); + + mpx.setObjHandlerFunc("removeImage", (data: any, _: Map, writer: CL.ResponseObjWriter): void => { + let res = removeImage(data as RemoveImageRequest); + writer.replySuccess(res); + }); +} + +interface RunPodSandboxRequest { + config: PodSandboxConfig +} + +interface PodSandboxConfig { + metadata: PodSandboxMetadata +} + +interface PodSandboxMetadata { + name: string + uid: string + namespace: string +} + +interface RunPodSandboxResponse { + pod_sandbox_id: string +} + + +interface StopPodSandboxRequest { + pod_sandbox_id: string +} + +interface StopPodSandboxResponse { + // empty +} + +interface RemovePodSandboxRequest { + pod_sandbox_id: string +} + +interface RemovePodSandboxResponse { + // empty +} + +interface PodSandboxStatusRequest { + pod_sandbox_id: string +} + +interface PodSandboxStatusResponse { + status: PodSandboxStatus + containers_statuses: ContainerStatus[] + timestamp: string +} + +interface PodSandboxStatus { + id: string + metadata: PodSandboxMetadata + state: PodSandboxState + created_at: string +} + +enum PodSandboxState { + SandboxReady = 0, + SandboxNotReady = 1, +} + +interface CreateContainerRequest { + pod_sandbox_id: string + config: ContainerConfig + // sandbox_config: PodSandboxConfig +} + +interface ContainerConfig { + metadata: ContainerMetadata + image: ImageSpec +} + +interface ContainerMetadata { + name: string +} + +interface CreateContainerResponse { + container_id: string +} + +interface StartContainerRequest { + container_id: string +} + +interface StartContainerResponse { + // empty +} + +interface StopContainerRequest { + container_id: string +} + +interface StopContainerResponse { + // empty +} + +interface RemoveContainerRequest { + container_id: string +} + +interface RemoveContainerResponse { + // empty +} + +interface ContainerStatus { + id: string + metadata: ContainerMetadata + state: ContainerState + created_at: string + started_at: string + finished_at: string + exit_code: number + image: ImageSpec + image_ref: string +} + +enum ContainerState { + ContainerCreated = 0, + ContainerRunning = 1, + ContainerExited = 2, + ContainerUnknown = 3, +} + +interface ListImagesRequest { + filter?: ImageFilter +} + +interface ImageFilter { + image: ImageSpec +} + +interface ImageSpec { + image: string +} + +interface ListImagesResponse { + images: Image[] +} + +interface Image { + id: string + spec: ImageSpec + // this field meaning the runtime environment of wasm, like 'go:1.19' + runtime: string +} + +interface PullImageRequest { + image: ImageSpec +} + +interface PullImageResponse { + image_ref: string +} + +interface RemoveImageRequest { + image: ImageSpec +} + +interface RemoveImageResponse { + // nothing +} + +function runPodSandbox(request: RunPodSandboxRequest): RunPodSandboxResponse { + let id: string = (Math.floor(Math.random() * podSandboxIDMax)).toString(16); + while (sandboxes.has(id)) { + id = (Math.floor(Math.random() * podSandboxIDMax)).toString(16); + } + + let meta: PodSandboxMetadata = request.config.metadata; + sandboxes.set(id, new Sandbox(meta.name, meta.uid, meta.namespace)); + + return { pod_sandbox_id: id }; +} + +function stopPodSandbox(request: StopPodSandboxRequest): StopPodSandboxResponse { + let sandbox = sandboxes.get(request.pod_sandbox_id); + + if (sandbox != null) { + sandbox.stop(); + } + + return {}; +} + +function removePodSandbox(request: RemovePodSandboxRequest): RemovePodSandboxResponse { + let sandbox = sandboxes.get(request.pod_sandbox_id); + + if (sandbox != null) { + sandbox.stop(); + sandboxes.delete(request.pod_sandbox_id); + } + + return {}; +} + +function podSandboxStatus(request: PodSandboxStatusRequest): PodSandboxStatusResponse { + let sandbox = sandboxes.get(request.pod_sandbox_id); + + if (sandbox == null) { + throw new Error("sandbox not found"); + } + + let containers_statuses: ContainerStatus[] = new Array(); + for (const [_, container] of sandbox.containers) { + containers_statuses.push({ + id: container.id, + metadata: { + name: container.name, + }, + state: container.getState(), + created_at: container.created_at, + started_at: container.started_at || "", + finished_at: container.finished_at || "", + exit_code: container.exit_code || 0, + image: { + image: container.image.url, + }, + image_ref: container.image.id, + }); + } + + return { + status: { + id: sandbox.uid, + metadata: { + name: sandbox.name, + namespace: sandbox.namespace, + uid: sandbox.uid, + }, + state: PodSandboxState.SandboxReady, + created_at: sandbox.created_at, + }, + containers_statuses: containers_statuses, + timestamp: getTimestamp(), + }; +} + +function createContainer(request: CreateContainerRequest): CreateContainerResponse { + let sandbox = sandboxes.get(request.pod_sandbox_id); + if (sandbox == null) { + throw new Error("sandbox not found"); + } + + let image = imageByURL.get(request.config.image.image); + if (image == null) { + throw new Error("image not found"); + } + + let id = sandbox.createContainer(request.config.metadata.name, image); + return { container_id: id }; +} + +function startContainer(request: StartContainerRequest): StartContainerResponse { + let container = containers.get(request.container_id); + if (container == null) { + throw new Error("container not found"); + } + container.start(); + return {}; +} + +function stopContainer(request: StopContainerRequest): StopContainerResponse { + let container = containers.get(request.container_id); + if (container == null) { + throw new Error("container not found"); + } + container.stop(); + return {}; +} + +function removeContainer(request: RemoveContainerRequest): RemoveContainerResponse { + let container = containers.get(request.container_id); + if (container == null) { + return {}; + } + + container.stop(); + + let sandbox = sandboxes.get(container.sandbox_id); + if (sandbox != null) { + sandbox.removeContainer(container.id); + } + + return {}; +} + +function listImages(request: ListImagesRequest): ListImagesResponse { + let buf: Array = new Array(); + if (request.filter != null) { + let image = imageByURL.get(request.filter.image.image); + buf.push(image); + + } else { + for (const it of images) { + buf.push(it.deref()); + } + } + + let resImages = new Array(); + for (const it of buf) { + if (it == null || it.state != ImageState.Downloaded) { + continue; + } + + resImages.push({ + id: it.id, + spec: { + image: it.url + }, + runtime: it.runtime, + }); + } + + return { images: resImages }; +} + +function pullImage(request: PullImageRequest): PullImageResponse { + let idSet: Set = new Set(); + for (const it of images) { + let instance = it.deref(); + if (instance == null) { + continue; + } + idSet.add(instance.id); + } + + let id: string = (Math.floor(Math.random() * imageRefIDMax)).toString(16); + while (idSet.has(id)) { + id = (Math.floor(Math.random() * imageRefIDMax)).toString(16); + } + + let image = new ImageInstance(id, request.image.image); + imageByURL.set(image.url, image); + images.push(new WeakRef(image)); + + return { image_ref: id }; +} + +function removeImage(request: RemoveImageRequest): RemoveImageResponse { + let url = request.image.image; + imageByURL.delete(url); + + images = images.filter((it): boolean => { + let instance = it.deref(); + if (instance == null) { + return false; + } + return instance.url !== url; + }); + + return {}; +} \ No newline at end of file