From 9dace51ca3dca141e647015573eb0d736626d025 Mon Sep 17 00:00:00 2001 From: zhouchencheng Date: Sat, 27 Jul 2019 23:08:44 +0800 Subject: [PATCH] feature: supernode support download remote file by skipping secure verify or setting root ca certs. Signed-off-by: zhouchencheng --- apis/swagger.yml | 2443 +++++++++-------- apis/types/task_register_request.go | 27 + cmd/dfget/app/root.go | 4 + dfdaemon/config/config.go | 27 +- dfdaemon/downloader/dfget/dfget.go | 17 +- dfget/config/config.go | 6 + dfget/core/regist/register.go | 12 + dfget/types/register_request.go | 2 + pkg/httputils/http_util.go | 63 - supernode/daemon/mgr/cdn/cache_detector.go | 10 +- supernode/daemon/mgr/cdn/downloader.go | 16 +- supernode/daemon/mgr/cdn/downloader_test.go | 3 +- supernode/daemon/mgr/cdn/manager.go | 7 +- supernode/daemon/mgr/task/manager.go | 10 +- supernode/daemon/mgr/task/manager_test.go | 18 +- supernode/daemon/mgr/task/manager_util.go | 6 +- .../daemon/mgr/task/manager_util_test.go | 17 +- .../mock/mock_origin_http_client.go | 109 + supernode/httpclient/origin_http_client.go | 183 ++ supernode/server/0.3_bridge.go | 1 + supernode/server/server.go | 9 +- 21 files changed, 1645 insertions(+), 1345 deletions(-) create mode 100644 supernode/httpclient/mock/mock_origin_http_client.go create mode 100644 supernode/httpclient/origin_http_client.go diff --git a/apis/swagger.yml b/apis/swagger.yml index 3b1548bb3..c66b26bf6 100644 --- a/apis/swagger.yml +++ b/apis/swagger.yml @@ -1,1216 +1,1227 @@ ---- -swagger: "2.0" -schemes: - - "http" - - "https" -produces: - - "application/json" - - "text/plain" -consumes: - - "application/json" - - "text/plain" -info: - title: "Dragonfly SuperNode API" - version: "0.1" - description: | - API is an HTTP API served by Dragonfly's SuperNode. It is the API dfget or Harbor uses to communicate - with the supernode. -tags: - # primary objects - - name: "Peer" - x-displayName: "Peers" - description: "Create and manage peer nodes in peer networks." - - name: "Task" - x-displayName: "Tasks" - description: "create and manage image/file distribution task in supernode." - - name: "Piece" - x-displayName: "Pieces" - description: "create and manage image/file pieces in supernode." - - name: "PreheatTask" - x-displayName: "PreheatTasks" - description: "Create and manage image or file preheat task in supernode." - -paths: - /_ping: - get: - summary: "Ping" - description: "This is a dummy endpoint you can use to test if the server is accessible." - responses: - 200: - description: "no error" - schema: - type: "string" - example: "OK" - 500: - $ref: "#/responses/500ErrorResponse" - - /version: - get: - summary: "Get version and build information" - description: | - Get version and build information, including GoVersion, OS, - Arch, Version, BuildDate, and GitCommit. - responses: - 200: - description: "no error" - schema: - $ref: "#/definitions/DragonflyVersion" - 500: - $ref: "#/responses/500ErrorResponse" - - /metrics: - get: - summary: "Get Prometheus metrics" - description: "Get Prometheus metrics" - responses: - 200: - description: "no error" - schema: - type: "string" - example: "go_goroutines 1" - - /peer/registry: - post: - summary: "registry a task" - description: | - Create a peer-to-peer downloading task in supernode. - parameters: - - name: "body" - in: "body" - description: "request body which contains task creation information" - schema: - $ref: "#/definitions/TaskRegisterRequest" - responses: - 200: - description: "no error" - schema: - $ref: "#/definitions/ResultInfo" - 400: - description: "bad parameter" - schema: - $ref: '#/definitions/Error' - 500: - $ref: "#/responses/500ErrorResponse" - - /peer/task: - get: - summary: "Get pieces in task" - description: | - When dfget starts to download pieces of a task, it should get fixed - number of pieces in a task and the use pieces information to download - the pirces. The request piece number is set in query. - produces: - - "application/json" - parameters: - - name: taskId - in: query - required: true - description: "ID of task" - type: string - - name: srcCid - in: query - type: "string" - required: true - description: - When dfget needs to get pieces of specific task, it must mark which peer it plays role of. - - name: dstCid - in: query - type: "string" - description: | - the uploader cid - - name: status - type: "string" - in: query - description: | - dfgetTaskStatus indicates whether the dfgetTask is running. - enum: ["STARTED", "RUNNING", "FINISHED"] - - name: result - in: query - type: "string" - description: | - pieceResult It indicates whether the dfgetTask successfully download the piece. - It's only useful when `status` is `RUNNING`. - enum: ["FAILED", "SUCCESS", "INVALID", "SEMISUC"] - - name: range - type: "string" - in: query - description: | - the range of specific piece in the task, example "0-45565". - responses: - 200: - description: "no error" - schema: - $ref: "#/definitions/ResultInfo" - 404: - description: "no such task" - schema: - $ref: "#/responses/404ErrorResponse" - 500: - $ref: "#/responses/500ErrorResponse" - - /peer/piece/suc: - get: - summary: "report a piece has been success" - description: | - Update some information of piece. When peer A finishes to download - piece B, A must send request to supernode to update piece B's info - to mark that peer A has the complete piece B. Then when other peers - request to download this piece B, supernode could schedule peer A - to those peers. - produces: - - "application/json" - parameters: - - name: taskId - in: query - required: true - description: "ID of task" - type: string - - name: pieceRange - in: query - required: true - description: | - the range of specific piece in the task, example "0-45565". - type: string - - name: cid - in: query - type: string - required: true - description: | - the downloader clientID - - name: dstCid - in: query - type: string - description: | - the uploader peerID - responses: - 200: - description: "no error" - schema: - $ref: "#/definitions/ResultInfo" - 404: - $ref: "#/responses/404ErrorResponse" - 500: - $ref: "#/responses/500ErrorResponse" - - /peer/service/down: - get: - summary: "report a peer service will offline" - produces: - - "application/json" - parameters: - - name: taskId - in: query - required: true - description: "ID of task" - type: string - - name: cid - in: query - type: string - required: true - description: | - the downloader clientID - responses: - 200: - description: "no error" - schema: - $ref: "#/definitions/ResultInfo" - 404: - $ref: "#/responses/404ErrorResponse" - 500: - $ref: "#/responses/500ErrorResponse" - - /peers: - post: - summary: "register dfget in Supernode as a peer node" - description: "dfget sends request to register in Supernode as a peer node" - parameters: - - name: "body" - in: "body" - description: "request body which contains peer registrar information." - schema: - $ref: "#/definitions/PeerCreateRequest" - responses: - 201: - description: "no error" - schema: - $ref: "#/definitions/PeerCreateResponse" - 400: - description: "bad parameter" - schema: - $ref: '#/definitions/Error' - 500: - $ref: "#/responses/500ErrorResponse" - - get: - summary: "get all peers" - description: "dfget sends request to register in Supernode as a peer node" - parameters: - - name: pageNum - in: query - type: integer - default: 0 - - name: pageSize - in: query - required: true - type: integer - - name: sortKey - in: query - description: | - "The keyword used to sort. You can provide multiple keys, if two peers have the same first key, sort by the second key, and so on" - type: "array" - items: - type: "string" - - name: sortDirect - in: query - description: "Determine the direction of sorting rules" - type: string - default: "ASC" - enum: ["ASC", "DESC"] - responses: - 201: - description: "no error" - schema: - type: "array" - items: - $ref: "#/definitions/PeerInfo" - 400: - description: "bad parameter" - schema: - $ref: '#/definitions/Error' - 500: - $ref: "#/responses/500ErrorResponse" - - /peers/{id}: - get: - summary: "get a peer in supernode" - description: "return low-level information of a peer in supernode." - produces: - - "application/json" - parameters: - - name: id - in: path - required: true - description: ID of peer - type: string - responses: - 200: - description: "no error" - schema: - $ref: "#/definitions/PeerInfo" - 404: - $ref: "#/responses/404ErrorResponse" - 500: - $ref: "#/responses/500ErrorResponse" - - delete: - summary: "delete a peer in supernode" - description: | - dfget stops playing a role as a peer in peer network constructed by supernode. - When dfget lasts in five minutes without downloading or uploading task, the uploader of dfget - automatically sends a DELETE /peers/{id} request to supernode. - parameters: - - name: id - in: path - required: true - description: "ID of peer" - type: string - responses: - 204: - description: "no error" - 404: - description: "no such peer" - schema: - $ref: '#/responses/404ErrorResponse' - 500: - $ref: "#/responses/500ErrorResponse" - - /tasks: - post: - summary: "create a task" - description: | - Create a peer-to-peer downloading task in supernode. - parameters: - - name: "body" - in: "body" - description: "request body which contains task creation information" - schema: - $ref: "#/definitions/TaskCreateRequest" - responses: - 201: - description: "no error" - schema: - $ref: "#/definitions/TaskCreateResponse" - 400: - description: "bad parameter" - schema: - $ref: '#/definitions/Error' - 500: - $ref: "#/responses/500ErrorResponse" - - /tasks/{id}: - get: - summary: "get a task" - description: | - return low-level information of a task in supernode. - produces: - - "application/json" - parameters: - - name: id - in: path - required: true - description: "ID of task" - type: string - responses: - 200: - description: "no error" - schema: - $ref: "#/definitions/TaskInfo" - 404: - $ref: "#/responses/404ErrorResponse" - 500: - $ref: "#/responses/500ErrorResponse" - - put: - summary: "update a task" - description: | - Update information of a task. - This endpoint is mainly for operation usage. When the peer network or peer - meet some load issues, operation team can update a task directly, such as pause - a downloading task to ease the situation. - consumes: - - "application/json" - produces: - - "application/json" - parameters: - - name: id - in: path - required: true - description: "ID of task" - type: string - - name: "TaskUpdateRequest" - in: "body" - description: | - request body which contains task update information" - schema: - $ref: "#/definitions/TaskUpdateRequest" - responses: - 200: - description: "no error" - 404: - $ref: "#/responses/404ErrorResponse" - 500: - $ref: "#/responses/500ErrorResponse" - - delete: - summary: "delete a task" - description: | - delete a peer-to-peer task in supernode. - This endpoint is mainly for operation usage. When the peer network or peer - meet some load issues, operation team can delete a task directly to ease - the situation. - parameters: - - name: id - in: path - required: true - description: "ID of task" - type: string - responses: - 204: - description: "no error" - 404: - description: "no such task" - schema: - $ref: '#/responses/404ErrorResponse' - 500: - $ref: "#/responses/500ErrorResponse" - - /tasks/{id}/pieces: - get: - summary: "Get pieces in task" - description: | - When dfget starts to download pieces of a task, it should get fixed - number of pieces in a task and the use pieces information to download - the pirces. The request piece number is set in query. - produces: - - "application/json" - parameters: - - name: id - in: path - required: true - description: "ID of task" - type: string - - name: num - in: query - type: "integer" - format: int64 - required: false - description: | - Request number of pieces of task. If request number is larger than the total pieces in supernode, - supernode returns the total pieces of task. If not set, supernode will set 4 by default. - - name: clientID - in: query - type: "string" - required: true - description: - When dfget needs to get pieces of specific task, it must mark which peer it plays role of. - - name: PiecePullRequest - in: body - required: true - description: | - request body which contains the information of pieces that have been downloaded or being downloaded. - schema: - $ref: "#/definitions/PiecePullRequest" - responses: - 200: - description: "no error" - schema: - type: "array" - items: - $ref: "#/definitions/PieceInfo" - 404: - description: "no such task" - schema: - $ref: "#/responses/404ErrorResponse" - 500: - $ref: "#/responses/500ErrorResponse" - - /tasks/{id}/pieces/{pieceRange}: - put: - summary: "Update a piece" - description: | - Update some information of piece. When peer A finishes to download - piece B, A must send request to supernode to update piece B's info - to mark that peer A has the complete piece B. Then when other peers - request to download this piece B, supernode could schedule peer A - to those peers. - consumes: - - "application/json" - produces: - - "application/json" - parameters: - - name: id - in: path - required: true - description: "ID of task" - type: string - - name: pieceRange - in: path - required: true - description: | - the range of specific piece in the task, example "0-45565". - type: string - - name: "PieceUpdateRequest" - in: "body" - description: | - request body which contains task update information. - schema: - $ref: "#/definitions/PieceUpdateRequest" - responses: - 200: - description: "no error" - 404: - $ref: "#/responses/404ErrorResponse" - 500: - $ref: "#/responses/500ErrorResponse" - - /preheats: - post: - summary: "Create a Preheat Task" - description: | - Create a preheat task in supernode to first download image/file which is ready. - Preheat action will shorten the period for dfget to get what it wants. In details, - after preheat action finishes downloading image/file to supernode, dfget can send - request to setup a peer-to-peer network immediately. - parameters: - - name: "PreheatCreateRequest" - in: "body" - description: "request body which contains preheat task creation information" - schema: - $ref: "#/definitions/PreheatCreateRequest" - responses: - 200: - description: "no error" - schema: - $ref: "#/definitions/PreheatCreateResponse" - 400: - description: "bad parameter" - schema: - $ref: '#/definitions/Error' - 500: - $ref: "#/responses/500ErrorResponse" - - get: - summary: "List Preheat Tasks" - description: | - List preheat tasks in supernode of Dragonfly. This API can list all the existing preheat tasks - in supernode. Note, when a preheat is finished after PreheatGCThreshold, it will be GCed, then - this preheat will not be gotten by preheat tasks list API. - responses: - 200: - description: "no error" - schema: - type: "array" - items: - $ref: "#/definitions/PreheatInfo" - 400: - description: "bad parameter" - schema: - $ref: '#/definitions/Error' - 500: - $ref: "#/responses/500ErrorResponse" - - /preheats/{id}: - get: - summary: "Get a preheat task" - description: | - get detailed information of a preheat task in supernode. - produces: - - "application/json" - parameters: - - name: id - in: path - required: true - description: "ID of preheat task" - type: string - responses: - 200: - description: "no error" - schema: - $ref: "#/definitions/PreheatInfo" - 404: - description: "no such preheat task" - schema: - $ref: "#/responses/404ErrorResponse" - 500: - $ref: "#/responses/500ErrorResponse" - - -definitions: - Error: - type: "object" - properties: - message: - type: string - - DragonflyVersion: - type: "object" - description: | - Version and build information of Dragonfly components. - properties: - Version: - type: "string" - description: "Version of Dragonfly components" - Revision: - type: "string" - description: "Git commit when building Dragonfly components" - BuildDate: - type: "string" - description: "Build Date of Dragonfly components" - GoVersion: - type: "string" - description: "Golang runtime version" - OS: - type: "string" - description: "Dragonfly components's operating system" - Arch: - type: "string" - description: "Dragonfly components's architecture target" - - ResultInfo: - type: "object" - description: | - The returned information from supernode. - properties: - code: - type: "integer" - format: "int32" - description: "the result code" - msg: - type: "string" - description: "the result msg" - data: - type: "object" - description: "the result data" - - TaskRegisterRequest: - type: "object" - description: "" - properties: - IP: - type: "string" - description: "IP address which peer client carries" - format: "ipv4" - superNodeIp: - type: "string" - description: "The address of supernode that the client can connect to" - hostName: - type: "string" - description: "host name of peer client node." - minLength: 1 - port: - type: "integer" - description: | - when registering, dfget will setup one uploader process. - This one acts as a server for peer pulling tasks. - This port is which this server listens on. - format: "int32" - minimum: 15000 - maximum: 65000 - version: - type: "string" - description: "version number of dfget binary." - cID: - type: "string" - description: | - CID means the client ID. It maps to the specific dfget process. - When user wishes to download an image/file, user would start a dfget process to do this. - This dfget is treated a client and carries a client ID. - Thus, multiple dfget processes on the same peer have different CIDs. - rawURL: - type: "string" - description: | - The is the resource's URL which user uses dfget to download. The location of URL can be anywhere, LAN or WAN. - For image distribution, this is image layer's URL in image registry. - The resource url is provided by command line parameter. - taskURL: - type: "string" - description: | - taskURL is generated from rawURL. rawURL may contains some queries or parameter, dfget will filter some queries via - --filter parameter of dfget. The usage of it is that different rawURL may generate the same taskID. - md5: - type: "string" - description: | - md5 checksum for the resource to distribute. dfget catches this parameter from dfget's CLI - and passes it to supernode. When supernode finishes downloading file/image from the source location, - it will validate the source file with this md5 value to check whether this is a valid file. - identifier: - type: "string" - description: | - special attribute of remote source file. This field is used with taskURL to generate new taskID to - identify different downloading task of remote source file. For example, if user A and user B uses - the same taskURL and taskID to download file, A and B will share the same peer network to distribute files. - If user A additionally adds an identifier with taskURL, while user B still carries only taskURL, then A's - generated taskID is different from B, and the result is that two users use different peer networks. - path: - type: "string" - description: | - path is used in one peer A for uploading functionality. When peer B hopes - to get piece C from peer A, B must provide a URL for piece C. - Then when creating a task in supernode, peer A must provide this URL in request. - headers: - type: "array" - description: | - extra HTTP headers sent to the rawURL. - This field is carried with the request to supernode. - Supernode will extract these HTTP headers, and set them in HTTP downloading requests - from source server as user's wish. - items: - type: "string" - dfdaemon: - type: "boolean" - description: | - tells whether it is a call from dfdaemon. dfdaemon is a long running - process which works for container engines. It translates the image - pulling request into raw requests into those dfget recognizes. - callSystem: - type: "string" - description: | - This attribute represents where the dfget requests come from. Dfget will pass - this field to supernode and supernode can do some checking and filtering via - black/white list mechanism to guarantee security, or some other purposes like debugging. - minLength: 1 - - PeerCreateRequest: - type: "object" - description: | - PeerCreateRequest is used to create a peer instance in supernode. - Usually, when dfget is going to register in supernode as a peer, - it will send PeerCreateRequest to supernode. - properties: - IP: - type: "string" - description: "IP address which peer client carries" - format: "ipv4" - hostName: - type: "string" - description: "host name of peer client node, as a valid RFC 1123 hostname." - format: "hostname" - minLength: 1 - port: - type: "integer" - description: | - when registering, dfget will setup one uploader process. - This one acts as a server for peer pulling tasks. - This port is which this server listens on. - format: "int32" - minimum: 15000 - maximum: 65000 - version: - type: "string" - description: "version number of dfget binary." - - PeerCreateResponse: - type: "object" - description: "ID of created peer." - properties: - ID: - type: "string" - description: | - Peer ID of the node which dfget locates on. - Every peer has a unique ID among peer network. - It is generated via host's hostname and IP address. - - PeerInfo: - type: "object" - description: | - The detailed information of a peer in supernode. - properties: - ID: - type: "string" - description: "ID of peer" - IP: - type: "string" - description: | - IP address which peer client carries. - (TODO) make IP field contain more information, for example - WAN/LAN IP address for supernode to recognize. - format: "ipv4" - hostName: - type: "string" - description: "host name of peer client node, as a valid RFC 1123 hostname." - format: "hostname" - minLength: 1 - port: - type: "integer" - description: | - when registering, dfget will setup one uploader process. - This one acts as a server for peer pulling tasks. - This port is which this server listens on. - minimum: 15000 - maximum: 65000 - format: "int32" - version: - type: "string" - description: "version number of dfget binary" - created: - type : "string" - format : "date-time" - description: "the time to join the P2P network" - - TaskCreateRequest: - type: "object" - description: "" - properties: - cID: - type: "string" - description: | - CID means the client ID. It maps to the specific dfget process. - When user wishes to download an image/file, user would start a dfget process to do this. - This dfget is treated a client and carries a client ID. - Thus, multiple dfget processes on the same peer have different CIDs. - rawURL: - type: "string" - description: | - The is the resource's URL which user uses dfget to download. The location of URL can be anywhere, LAN or WAN. - For image distribution, this is image layer's URL in image registry. - The resource url is provided by command line parameter. - taskURL: - type: "string" - description: | - taskURL is generated from rawURL. rawURL may contains some queries or parameter, dfget will filter some queries via - --filter parameter of dfget. The usage of it is that different rawURL may generate the same taskID. - md5: - type: "string" - description: | - md5 checksum for the resource to distribute. dfget catches this parameter from dfget's CLI - and passes it to supernode. When supernode finishes downloading file/image from the source location, - it will validate the source file with this md5 value to check whether this is a valid file. - identifier: - type: "string" - description: | - special attribute of remote source file. This field is used with taskURL to generate new taskID to - identify different downloading task of remote source file. For example, if user A and user B uses - the same taskURL and taskID to download file, A and B will share the same peer network to distribute files. - If user A additionally adds an identifier with taskURL, while user B still carries only taskURL, then A's - generated taskID is different from B, and the result is that two users use different peer networks. - path: - type: "string" - description: | - path is used in one peer A for uploading functionality. When peer B hopes - to get piece C from peer A, B must provide a URL for piece C. - Then when creating a task in supernode, peer A must provide this URL in request. - headers: - type: "object" - description: | - extra HTTP headers sent to the rawURL. - This field is carried with the request to supernode. - Supernode will extract these HTTP headers, and set them in HTTP downloading requests - from source server as user's wish. - additionalProperties: - type: "string" - dfdaemon: - type: "boolean" - description: | - tells whether it is a call from dfdaemon. dfdaemon is a long running - process which works for container engines. It translates the image - pulling request into raw requests into those dfget recognizes. - callSystem: - type: "string" - description: | - This attribute represents where the dfget requests come from. Dfget will pass - this field to supernode and supernode can do some checking and filtering via - black/white list mechanism to guarantee security, or some other purposes like debugging. - minLength: 1 - filter: - type: "array" - description: | - filter is used to filter request queries in URL. - For example, when a user wants to start to download a task which has a remote URL of - a.b.com/fileA?user=xxx&auth=yyy, user can add a filter parameter ["user", "auth"] - to filter the url to a.b.com/fileA. Then this parameter can potentially avoid repeatable - downloads, if there is already a task a.b.com/fileA. - items: - type: "string" - peerID: - type: "string" - description: | - PeerID is used to uniquely identifies a peer which will be used to create a dfgetTask. - The value must be the value in the response after registering a peer. - supernodeIP: - type: "string" - description: "IP address of supernode which the peer connects to" - - - TaskCreateResponse: - type: "object" - description: "response get from task creation request." - properties: - ID: - type: "string" - description: "ID of the created task." - fileLength: - type: "integer" - description: | - The length of the file dfget requests to download in bytes. - format: int64 - pieceSize: - type: "integer" - description: | - The size of pieces which is calculated as per the following strategy - 1. If file's total size is less than 200MB, then the piece size is 4MB by default. - 2. Otherwise, it equals to the smaller value between totalSize/100MB + 2 MB and 15MB. - format: int32 - - TaskInfo: - type: "object" - description: "detailed information about task in supernode." - properties: - ID: - type: "string" - description: "ID of the task." - fileLength: - type: "integer" - description: | - The length of the file dfget requests to download in bytes - which including the header and the trailer of each piece. - format: "int64" - httpFileLength: - type: "integer" - description: | - The length of the source file in bytes. - format: "int64" - pieceSize: - type: "integer" - description: | - The size of pieces which is calculated as per the following strategy - 1. If file's total size is less than 200MB, then the piece size is 4MB by default. - 2. Otherwise, it equals to the smaller value between totalSize/100MB + 2 MB and 15MB. - format: "int32" - pieceTotal: - type: "integer" - description: "" - format: "int32" - cdnStatus: - type: "string" - description: | - The status of the created task related to CDN functionality. - enum: ["WAITING", "RUNNING", "FAILED", "SUCCESS", "SOURCE_ERROR"] - rawURL: - type: "string" - description: | - The is the resource's URL which user uses dfget to download. The location of URL can be anywhere, LAN or WAN. - For image distribution, this is image layer's URL in image registry. - The resource url is provided by command line parameter. - taskURL: - type: "string" - description: | - taskURL is generated from rawURL. rawURL may contains some queries or parameter, dfget will filter some queries via - --filter parameter of dfget. The usage of it is that different rawURL may generate the same taskID. - md5: - type: "string" - description: | - md5 checksum for the resource to distribute. dfget catches this parameter from dfget's CLI - and passes it to supernode. When supernode finishes downloading file/image from the source location, - it will validate the source file with this md5 value to check whether this is a valid file. - realMd5: - type: "string" - description: | - when supernode finishes downloading file/image from the source location, - the md5 sum of the source file will be calculated as the value of the realMd5. - And it will be used to compare with md5 value to check whether this is a valid file. - identifier: - type: "string" - description: | - special attribute of remote source file. This field is used with taskURL to generate new taskID to - identify different downloading task of remote source file. For example, if user A and user B uses - the same taskURL and taskID to download file, A and B will share the same peer network to distribute files. - If user A additionally adds an identifier with taskURL, while user B still carries only taskURL, then A's - generated taskID is different from B, and the result is that two users use different peer networks. - headers: - type: "object" - description: | - extra HTTP headers sent to the rawURL. - This field is carried with the request to supernode. - Supernode will extract these HTTP headers, and set them in HTTP downloading requests - from source server as user's wish. - additionalProperties: - type: "string" - - TaskUpdateRequest: - type: "object" - description: "request used to update task attributes." - properties: - peerID: - type: "string" - description: "ID of the peer which has finished to download the whole task." - - PieceInfo: - type: "object" - description: "Peer's detailed information in supernode." - properties: - pID: - type: "string" - description: "the peerID that dfget task should download from" - pieceRange: - type: "string" - description: | - the range of specific piece in the task, example "0-45565". - pieceSize: - type: "integer" - description: | - The size of pieces which is calculated as per the following strategy - 1. If file's total size is less than 200MB, then the piece size is 4MB by default. - 2. Otherwise, it equals to the smaller value between totalSize/100MB + 2 MB and 15MB. - format: int32 - pieceMD5: - type: "string" - description: | - the MD5 information of piece which is generated by supernode when doing CDN cache. - This value will be returned to dfget in order to validate the piece's completeness. - peerIP: - type: string - description: | - When dfget needs to download a piece from another peer. Supernode will return a PieceInfo - that contains a peerIP. This peerIP represents the IP of this dfget's target peer. - peerPort: - type: "integer" - format: "int32" - description: | - When dfget needs to download a piece from another peer. Supernode will return a PieceInfo - that contains a peerPort. This peerPort represents the port of this dfget's target peer's uploader. - path: - type: "string" - description: | - The URL path to download the specific piece from the target peer's uploader. - - PieceUpdateRequest: - type: "object" - description: "request used to update piece attributes." - properties: - clientID: - type: "string" - description: | - the downloader clientID - dstPID: - type: "string" - description: | - the uploader peerID - pieceStatus: - type: "string" - description: | - pieceStatus indicates whether the peer task successfully download the piece. - enum: ["FAILED", "SUCCESS", "INVALID", "SEMISUC"] - - PiecePullRequest: - type: "object" - description: "request used to pull pieces that have not been downloaded." - properties: - dstPID: - type: "string" - description: | - the uploader peerID - dfgetTaskStatus: - type: "string" - description: | - dfgetTaskStatus indicates whether the dfgetTask is running. - enum: ["STARTED", "RUNNING", "FINISHED"] - pieceResult: - type: "string" - description: | - pieceResult It indicates whether the dfgetTask successfully download the piece. - It's only useful when `status` is `RUNNING`. - enum: ["FAILED", "SUCCESS", "INVALID", "SEMISUC"] - pieceRange: - type: "string" - description: | - the range of specific piece in the task, example "0-45565". - - PreheatInfo: - type: "object" - description: | - return detailed information of a preheat task in supernode. An image preheat task may contain multiple downloading - task because that an image may have more than one layer. - properties: - ID: - type: "string" - description: | - ID of preheat task. - status: - type: "string" - description: | - The status of preheat task. - WAITING -----> RUNNING -----> SUCCESS - |--> FAILED - The initial status of a created preheat task is WAITING. - It's finished when a preheat task's status is FAILED or SUCCESS. - A finished preheat task's information can be queried within 24 hours. - enum: ["WAITING", "RUNNING", "FAILED", "SUCCESS"] - startTime: - type: "string" - format: "date-time" - description: "the preheat task start time" - finishTime: - type: "string" - format: "date-time" - description: "the preheat task finish time" - - PreheatCreateRequest: - type: "object" - description: | - Request option of creating a preheat task in supernode. - properties: - type: - type: "string" - description: | - this must be image or file - url: - type: "string" - description: "the image or file location" - filter: - type: "string" - description: | - URL may contains some changeful query parameters such as authentication parameters. Dragonfly will - filter these parameter via 'filter'. The usage of it is that different URL may generate the same - download taskID. - identifier: - type: "string" - description: | - This field is used for generating new downloading taskID to identify different downloading task of remote URL. - headers: - type: "object" - description: | - If there is any authentication step of the remote server, the headers should contains authenticated information. - Dragonfly will sent request taking the headers to remote server. - additionalProperties: - type: "string" - - PreheatCreateResponse: - type: "object" - description: | - Response of a preheat creation request. - properties: - ID: - type: "string" - - DfGetTask: - type: "object" - description: | - A download process initiated by dfget or other clients. - properties: - taskId: - type: "string" - pieceSize: - type: "integer" - description: | - The size of pieces which is calculated as per the following strategy - 1. If file's total size is less than 200MB, then the piece size is 4MB by default. - 2. Otherwise, it equals to the smaller value between totalSize/100MB + 2 MB and 15MB. - format: int32 - cID: - type: "string" - description: | - CID means the client ID. It maps to the specific dfget process. - When user wishes to download an image/file, user would start a dfget process to do this. - This dfget is treated a client and carries a client ID. - Thus, multiple dfget processes on the same peer have different CIDs. - path: - type: "string" - description: | - path is used in one peer A for uploading functionality. When peer B hopes - to get piece C from peer A, B must provide a URL for piece C. - Then when creating a task in supernode, peer A must provide this URL in request. - status: - type: "string" - description: | - The status of Dfget download process. - enum: ["WAITING", "RUNNING", "FAILED", "SUCCESS",] - peerID: - type: "string" - description: | - PeerID uniquely identifies a peer, and the cID uniquely identifies a - download task belonging to a peer. One peer can initiate multiple download tasks, - which means that one peer corresponds to multiple cIDs. - supernodeIP: - type: "string" - description: "IP address of supernode which the peer connects to" - dfdaemon: - type: "boolean" - description: | - tells whether it is a call from dfdaemon. dfdaemon is a long running - process which works for container engines. It translates the image - pulling request into raw requests into those dfget recganises. - callSystem: - type: "string" - description: | - This attribute represents where the dfget requests come from. Dfget will pass - this field to supernode and supernode can do some checking and filtering via - black/white list mechanism to guarantee security, or some other purposes like debugging. - minLength: 1 - - ErrorResponse: - type: "object" - description: | - It contains a code that identify which error occurred for client processing and a detailed error message to read. - properties: - code: - type: "integer" - description: | - the code of this error, it's convenient for client to process with certain error. - message: - type: "string" - description: "detailed error message" - - -responses: - 401ErrorResponse: - description: An unexpected 401 error occurred. - schema: - $ref: "#/definitions/Error" - 404ErrorResponse: - description: An unexpected 404 error occurred. - schema: - $ref: "#/definitions/Error" - 500ErrorResponse: - description: An unexpected server error occurred. - schema: - $ref: "#/definitions/Error" +--- +swagger: "2.0" +schemes: + - "http" + - "https" +produces: + - "application/json" + - "text/plain" +consumes: + - "application/json" + - "text/plain" +info: + title: "Dragonfly SuperNode API" + version: "0.1" + description: | + API is an HTTP API served by Dragonfly's SuperNode. It is the API dfget or Harbor uses to communicate + with the supernode. +tags: + # primary objects + - name: "Peer" + x-displayName: "Peers" + description: "Create and manage peer nodes in peer networks." + - name: "Task" + x-displayName: "Tasks" + description: "create and manage image/file distribution task in supernode." + - name: "Piece" + x-displayName: "Pieces" + description: "create and manage image/file pieces in supernode." + - name: "PreheatTask" + x-displayName: "PreheatTasks" + description: "Create and manage image or file preheat task in supernode." + +paths: + /_ping: + get: + summary: "Ping" + description: "This is a dummy endpoint you can use to test if the server is accessible." + responses: + 200: + description: "no error" + schema: + type: "string" + example: "OK" + 500: + $ref: "#/responses/500ErrorResponse" + + /version: + get: + summary: "Get version and build information" + description: | + Get version and build information, including GoVersion, OS, + Arch, Version, BuildDate, and GitCommit. + responses: + 200: + description: "no error" + schema: + $ref: "#/definitions/DragonflyVersion" + 500: + $ref: "#/responses/500ErrorResponse" + + /metrics: + get: + summary: "Get Prometheus metrics" + description: "Get Prometheus metrics" + responses: + 200: + description: "no error" + schema: + type: "string" + example: "go_goroutines 1" + + /peer/registry: + post: + summary: "registry a task" + description: | + Create a peer-to-peer downloading task in supernode. + parameters: + - name: "body" + in: "body" + description: "request body which contains task creation information" + schema: + $ref: "#/definitions/TaskRegisterRequest" + responses: + 200: + description: "no error" + schema: + $ref: "#/definitions/ResultInfo" + 400: + description: "bad parameter" + schema: + $ref: '#/definitions/Error' + 500: + $ref: "#/responses/500ErrorResponse" + + /peer/task: + get: + summary: "Get pieces in task" + description: | + When dfget starts to download pieces of a task, it should get fixed + number of pieces in a task and the use pieces information to download + the pirces. The request piece number is set in query. + produces: + - "application/json" + parameters: + - name: taskId + in: query + required: true + description: "ID of task" + type: string + - name: srcCid + in: query + type: "string" + required: true + description: + When dfget needs to get pieces of specific task, it must mark which peer it plays role of. + - name: dstCid + in: query + type: "string" + description: | + the uploader cid + - name: status + type: "string" + in: query + description: | + dfgetTaskStatus indicates whether the dfgetTask is running. + enum: ["STARTED", "RUNNING", "FINISHED"] + - name: result + in: query + type: "string" + description: | + pieceResult It indicates whether the dfgetTask successfully download the piece. + It's only useful when `status` is `RUNNING`. + enum: ["FAILED", "SUCCESS", "INVALID", "SEMISUC"] + - name: range + type: "string" + in: query + description: | + the range of specific piece in the task, example "0-45565". + responses: + 200: + description: "no error" + schema: + $ref: "#/definitions/ResultInfo" + 404: + description: "no such task" + schema: + $ref: "#/responses/404ErrorResponse" + 500: + $ref: "#/responses/500ErrorResponse" + + /peer/piece/suc: + get: + summary: "report a piece has been success" + description: | + Update some information of piece. When peer A finishes to download + piece B, A must send request to supernode to update piece B's info + to mark that peer A has the complete piece B. Then when other peers + request to download this piece B, supernode could schedule peer A + to those peers. + produces: + - "application/json" + parameters: + - name: taskId + in: query + required: true + description: "ID of task" + type: string + - name: pieceRange + in: query + required: true + description: | + the range of specific piece in the task, example "0-45565". + type: string + - name: cid + in: query + type: string + required: true + description: | + the downloader clientID + - name: dstCid + in: query + type: string + description: | + the uploader peerID + responses: + 200: + description: "no error" + schema: + $ref: "#/definitions/ResultInfo" + 404: + $ref: "#/responses/404ErrorResponse" + 500: + $ref: "#/responses/500ErrorResponse" + + /peer/service/down: + get: + summary: "report a peer service will offline" + produces: + - "application/json" + parameters: + - name: taskId + in: query + required: true + description: "ID of task" + type: string + - name: cid + in: query + type: string + required: true + description: | + the downloader clientID + responses: + 200: + description: "no error" + schema: + $ref: "#/definitions/ResultInfo" + 404: + $ref: "#/responses/404ErrorResponse" + 500: + $ref: "#/responses/500ErrorResponse" + + /peers: + post: + summary: "register dfget in Supernode as a peer node" + description: "dfget sends request to register in Supernode as a peer node" + parameters: + - name: "body" + in: "body" + description: "request body which contains peer registrar information." + schema: + $ref: "#/definitions/PeerCreateRequest" + responses: + 201: + description: "no error" + schema: + $ref: "#/definitions/PeerCreateResponse" + 400: + description: "bad parameter" + schema: + $ref: '#/definitions/Error' + 500: + $ref: "#/responses/500ErrorResponse" + + get: + summary: "get all peers" + description: "dfget sends request to register in Supernode as a peer node" + parameters: + - name: pageNum + in: query + type: integer + default: 0 + - name: pageSize + in: query + required: true + type: integer + - name: sortKey + in: query + description: | + "The keyword used to sort. You can provide multiple keys, if two peers have the same first key, sort by the second key, and so on" + type: "array" + items: + type: "string" + - name: sortDirect + in: query + description: "Determine the direction of sorting rules" + type: string + default: "ASC" + enum: ["ASC", "DESC"] + responses: + 201: + description: "no error" + schema: + type: "array" + items: + $ref: "#/definitions/PeerInfo" + 400: + description: "bad parameter" + schema: + $ref: '#/definitions/Error' + 500: + $ref: "#/responses/500ErrorResponse" + + /peers/{id}: + get: + summary: "get a peer in supernode" + description: "return low-level information of a peer in supernode." + produces: + - "application/json" + parameters: + - name: id + in: path + required: true + description: ID of peer + type: string + responses: + 200: + description: "no error" + schema: + $ref: "#/definitions/PeerInfo" + 404: + $ref: "#/responses/404ErrorResponse" + 500: + $ref: "#/responses/500ErrorResponse" + + delete: + summary: "delete a peer in supernode" + description: | + dfget stops playing a role as a peer in peer network constructed by supernode. + When dfget lasts in five minutes without downloading or uploading task, the uploader of dfget + automatically sends a DELETE /peers/{id} request to supernode. + parameters: + - name: id + in: path + required: true + description: "ID of peer" + type: string + responses: + 204: + description: "no error" + 404: + description: "no such peer" + schema: + $ref: '#/responses/404ErrorResponse' + 500: + $ref: "#/responses/500ErrorResponse" + + /tasks: + post: + summary: "create a task" + description: | + Create a peer-to-peer downloading task in supernode. + parameters: + - name: "body" + in: "body" + description: "request body which contains task creation information" + schema: + $ref: "#/definitions/TaskCreateRequest" + responses: + 201: + description: "no error" + schema: + $ref: "#/definitions/TaskCreateResponse" + 400: + description: "bad parameter" + schema: + $ref: '#/definitions/Error' + 500: + $ref: "#/responses/500ErrorResponse" + + /tasks/{id}: + get: + summary: "get a task" + description: | + return low-level information of a task in supernode. + produces: + - "application/json" + parameters: + - name: id + in: path + required: true + description: "ID of task" + type: string + responses: + 200: + description: "no error" + schema: + $ref: "#/definitions/TaskInfo" + 404: + $ref: "#/responses/404ErrorResponse" + 500: + $ref: "#/responses/500ErrorResponse" + + put: + summary: "update a task" + description: | + Update information of a task. + This endpoint is mainly for operation usage. When the peer network or peer + meet some load issues, operation team can update a task directly, such as pause + a downloading task to ease the situation. + consumes: + - "application/json" + produces: + - "application/json" + parameters: + - name: id + in: path + required: true + description: "ID of task" + type: string + - name: "TaskUpdateRequest" + in: "body" + description: | + request body which contains task update information" + schema: + $ref: "#/definitions/TaskUpdateRequest" + responses: + 200: + description: "no error" + 404: + $ref: "#/responses/404ErrorResponse" + 500: + $ref: "#/responses/500ErrorResponse" + + delete: + summary: "delete a task" + description: | + delete a peer-to-peer task in supernode. + This endpoint is mainly for operation usage. When the peer network or peer + meet some load issues, operation team can delete a task directly to ease + the situation. + parameters: + - name: id + in: path + required: true + description: "ID of task" + type: string + responses: + 204: + description: "no error" + 404: + description: "no such task" + schema: + $ref: '#/responses/404ErrorResponse' + 500: + $ref: "#/responses/500ErrorResponse" + + /tasks/{id}/pieces: + get: + summary: "Get pieces in task" + description: | + When dfget starts to download pieces of a task, it should get fixed + number of pieces in a task and the use pieces information to download + the pirces. The request piece number is set in query. + produces: + - "application/json" + parameters: + - name: id + in: path + required: true + description: "ID of task" + type: string + - name: num + in: query + type: "integer" + format: int64 + required: false + description: | + Request number of pieces of task. If request number is larger than the total pieces in supernode, + supernode returns the total pieces of task. If not set, supernode will set 4 by default. + - name: clientID + in: query + type: "string" + required: true + description: + When dfget needs to get pieces of specific task, it must mark which peer it plays role of. + - name: PiecePullRequest + in: body + required: true + description: | + request body which contains the information of pieces that have been downloaded or being downloaded. + schema: + $ref: "#/definitions/PiecePullRequest" + responses: + 200: + description: "no error" + schema: + type: "array" + items: + $ref: "#/definitions/PieceInfo" + 404: + description: "no such task" + schema: + $ref: "#/responses/404ErrorResponse" + 500: + $ref: "#/responses/500ErrorResponse" + + /tasks/{id}/pieces/{pieceRange}: + put: + summary: "Update a piece" + description: | + Update some information of piece. When peer A finishes to download + piece B, A must send request to supernode to update piece B's info + to mark that peer A has the complete piece B. Then when other peers + request to download this piece B, supernode could schedule peer A + to those peers. + consumes: + - "application/json" + produces: + - "application/json" + parameters: + - name: id + in: path + required: true + description: "ID of task" + type: string + - name: pieceRange + in: path + required: true + description: | + the range of specific piece in the task, example "0-45565". + type: string + - name: "PieceUpdateRequest" + in: "body" + description: | + request body which contains task update information. + schema: + $ref: "#/definitions/PieceUpdateRequest" + responses: + 200: + description: "no error" + 404: + $ref: "#/responses/404ErrorResponse" + 500: + $ref: "#/responses/500ErrorResponse" + + /preheats: + post: + summary: "Create a Preheat Task" + description: | + Create a preheat task in supernode to first download image/file which is ready. + Preheat action will shorten the period for dfget to get what it wants. In details, + after preheat action finishes downloading image/file to supernode, dfget can send + request to setup a peer-to-peer network immediately. + parameters: + - name: "PreheatCreateRequest" + in: "body" + description: "request body which contains preheat task creation information" + schema: + $ref: "#/definitions/PreheatCreateRequest" + responses: + 200: + description: "no error" + schema: + $ref: "#/definitions/PreheatCreateResponse" + 400: + description: "bad parameter" + schema: + $ref: '#/definitions/Error' + 500: + $ref: "#/responses/500ErrorResponse" + + get: + summary: "List Preheat Tasks" + description: | + List preheat tasks in supernode of Dragonfly. This API can list all the existing preheat tasks + in supernode. Note, when a preheat is finished after PreheatGCThreshold, it will be GCed, then + this preheat will not be gotten by preheat tasks list API. + responses: + 200: + description: "no error" + schema: + type: "array" + items: + $ref: "#/definitions/PreheatInfo" + 400: + description: "bad parameter" + schema: + $ref: '#/definitions/Error' + 500: + $ref: "#/responses/500ErrorResponse" + + /preheats/{id}: + get: + summary: "Get a preheat task" + description: | + get detailed information of a preheat task in supernode. + produces: + - "application/json" + parameters: + - name: id + in: path + required: true + description: "ID of preheat task" + type: string + responses: + 200: + description: "no error" + schema: + $ref: "#/definitions/PreheatInfo" + 404: + description: "no such preheat task" + schema: + $ref: "#/responses/404ErrorResponse" + 500: + $ref: "#/responses/500ErrorResponse" + + +definitions: + Error: + type: "object" + properties: + message: + type: string + + DragonflyVersion: + type: "object" + description: | + Version and build information of Dragonfly components. + properties: + Version: + type: "string" + description: "Version of Dragonfly components" + Revision: + type: "string" + description: "Git commit when building Dragonfly components" + BuildDate: + type: "string" + description: "Build Date of Dragonfly components" + GoVersion: + type: "string" + description: "Golang runtime version" + OS: + type: "string" + description: "Dragonfly components's operating system" + Arch: + type: "string" + description: "Dragonfly components's architecture target" + + ResultInfo: + type: "object" + description: | + The returned information from supernode. + properties: + code: + type: "integer" + format: "int32" + description: "the result code" + msg: + type: "string" + description: "the result msg" + data: + type: "object" + description: "the result data" + + TaskRegisterRequest: + type: "object" + description: "" + properties: + IP: + type: "string" + description: "IP address which peer client carries" + format: "ipv4" + superNodeIp: + type: "string" + description: "The address of supernode that the client can connect to" + hostName: + type: "string" + description: "host name of peer client node." + minLength: 1 + port: + type: "integer" + description: | + when registering, dfget will setup one uploader process. + This one acts as a server for peer pulling tasks. + This port is which this server listens on. + format: "int32" + minimum: 15000 + maximum: 65000 + version: + type: "string" + description: "version number of dfget binary." + cID: + type: "string" + description: | + CID means the client ID. It maps to the specific dfget process. + When user wishes to download an image/file, user would start a dfget process to do this. + This dfget is treated a client and carries a client ID. + Thus, multiple dfget processes on the same peer have different CIDs. + rawURL: + type: "string" + description: | + The is the resource's URL which user uses dfget to download. The location of URL can be anywhere, LAN or WAN. + For image distribution, this is image layer's URL in image registry. + The resource url is provided by command line parameter. + taskURL: + type: "string" + description: | + taskURL is generated from rawURL. rawURL may contains some queries or parameter, dfget will filter some queries via + --filter parameter of dfget. The usage of it is that different rawURL may generate the same taskID. + md5: + type: "string" + description: | + md5 checksum for the resource to distribute. dfget catches this parameter from dfget's CLI + and passes it to supernode. When supernode finishes downloading file/image from the source location, + it will validate the source file with this md5 value to check whether this is a valid file. + identifier: + type: "string" + description: | + special attribute of remote source file. This field is used with taskURL to generate new taskID to + identify different downloading task of remote source file. For example, if user A and user B uses + the same taskURL and taskID to download file, A and B will share the same peer network to distribute files. + If user A additionally adds an identifier with taskURL, while user B still carries only taskURL, then A's + generated taskID is different from B, and the result is that two users use different peer networks. + path: + type: "string" + description: | + path is used in one peer A for uploading functionality. When peer B hopes + to get piece C from peer A, B must provide a URL for piece C. + Then when creating a task in supernode, peer A must provide this URL in request. + headers: + type: "array" + description: | + extra HTTP headers sent to the rawURL. + This field is carried with the request to supernode. + Supernode will extract these HTTP headers, and set them in HTTP downloading requests + from source server as user's wish. + items: + type: "string" + dfdaemon: + type: "boolean" + description: | + tells whether it is a call from dfdaemon. dfdaemon is a long running + process which works for container engines. It translates the image + pulling request into raw requests into those dfget recognizes. + insecure: + type: "boolean" + description: | + tells whether skip secure verify when supernode download the remote source file. + rootCAs: + type: "array" + description: | + The root ca cert from client used to download the remote source file. + items: + type: "string" + format: byte + callSystem: + type: "string" + description: | + This attribute represents where the dfget requests come from. Dfget will pass + this field to supernode and supernode can do some checking and filtering via + black/white list mechanism to guarantee security, or some other purposes like debugging. + minLength: 1 + + PeerCreateRequest: + type: "object" + description: | + PeerCreateRequest is used to create a peer instance in supernode. + Usually, when dfget is going to register in supernode as a peer, + it will send PeerCreateRequest to supernode. + properties: + IP: + type: "string" + description: "IP address which peer client carries" + format: "ipv4" + hostName: + type: "string" + description: "host name of peer client node, as a valid RFC 1123 hostname." + format: "hostname" + minLength: 1 + port: + type: "integer" + description: | + when registering, dfget will setup one uploader process. + This one acts as a server for peer pulling tasks. + This port is which this server listens on. + format: "int32" + minimum: 15000 + maximum: 65000 + version: + type: "string" + description: "version number of dfget binary." + + PeerCreateResponse: + type: "object" + description: "ID of created peer." + properties: + ID: + type: "string" + description: | + Peer ID of the node which dfget locates on. + Every peer has a unique ID among peer network. + It is generated via host's hostname and IP address. + + PeerInfo: + type: "object" + description: | + The detailed information of a peer in supernode. + properties: + ID: + type: "string" + description: "ID of peer" + IP: + type: "string" + description: | + IP address which peer client carries. + (TODO) make IP field contain more information, for example + WAN/LAN IP address for supernode to recognize. + format: "ipv4" + hostName: + type: "string" + description: "host name of peer client node, as a valid RFC 1123 hostname." + format: "hostname" + minLength: 1 + port: + type: "integer" + description: | + when registering, dfget will setup one uploader process. + This one acts as a server for peer pulling tasks. + This port is which this server listens on. + minimum: 15000 + maximum: 65000 + format: "int32" + version: + type: "string" + description: "version number of dfget binary" + created: + type : "string" + format : "date-time" + description: "the time to join the P2P network" + + TaskCreateRequest: + type: "object" + description: "" + properties: + cID: + type: "string" + description: | + CID means the client ID. It maps to the specific dfget process. + When user wishes to download an image/file, user would start a dfget process to do this. + This dfget is treated a client and carries a client ID. + Thus, multiple dfget processes on the same peer have different CIDs. + rawURL: + type: "string" + description: | + The is the resource's URL which user uses dfget to download. The location of URL can be anywhere, LAN or WAN. + For image distribution, this is image layer's URL in image registry. + The resource url is provided by command line parameter. + taskURL: + type: "string" + description: | + taskURL is generated from rawURL. rawURL may contains some queries or parameter, dfget will filter some queries via + --filter parameter of dfget. The usage of it is that different rawURL may generate the same taskID. + md5: + type: "string" + description: | + md5 checksum for the resource to distribute. dfget catches this parameter from dfget's CLI + and passes it to supernode. When supernode finishes downloading file/image from the source location, + it will validate the source file with this md5 value to check whether this is a valid file. + identifier: + type: "string" + description: | + special attribute of remote source file. This field is used with taskURL to generate new taskID to + identify different downloading task of remote source file. For example, if user A and user B uses + the same taskURL and taskID to download file, A and B will share the same peer network to distribute files. + If user A additionally adds an identifier with taskURL, while user B still carries only taskURL, then A's + generated taskID is different from B, and the result is that two users use different peer networks. + path: + type: "string" + description: | + path is used in one peer A for uploading functionality. When peer B hopes + to get piece C from peer A, B must provide a URL for piece C. + Then when creating a task in supernode, peer A must provide this URL in request. + headers: + type: "object" + description: | + extra HTTP headers sent to the rawURL. + This field is carried with the request to supernode. + Supernode will extract these HTTP headers, and set them in HTTP downloading requests + from source server as user's wish. + additionalProperties: + type: "string" + dfdaemon: + type: "boolean" + description: | + tells whether it is a call from dfdaemon. dfdaemon is a long running + process which works for container engines. It translates the image + pulling request into raw requests into those dfget recognizes. + callSystem: + type: "string" + description: | + This attribute represents where the dfget requests come from. Dfget will pass + this field to supernode and supernode can do some checking and filtering via + black/white list mechanism to guarantee security, or some other purposes like debugging. + minLength: 1 + filter: + type: "array" + description: | + filter is used to filter request queries in URL. + For example, when a user wants to start to download a task which has a remote URL of + a.b.com/fileA?user=xxx&auth=yyy, user can add a filter parameter ["user", "auth"] + to filter the url to a.b.com/fileA. Then this parameter can potentially avoid repeatable + downloads, if there is already a task a.b.com/fileA. + items: + type: "string" + peerID: + type: "string" + description: | + PeerID is used to uniquely identifies a peer which will be used to create a dfgetTask. + The value must be the value in the response after registering a peer. + supernodeIP: + type: "string" + description: "IP address of supernode which the peer connects to" + + + TaskCreateResponse: + type: "object" + description: "response get from task creation request." + properties: + ID: + type: "string" + description: "ID of the created task." + fileLength: + type: "integer" + description: | + The length of the file dfget requests to download in bytes. + format: int64 + pieceSize: + type: "integer" + description: | + The size of pieces which is calculated as per the following strategy + 1. If file's total size is less than 200MB, then the piece size is 4MB by default. + 2. Otherwise, it equals to the smaller value between totalSize/100MB + 2 MB and 15MB. + format: int32 + + TaskInfo: + type: "object" + description: "detailed information about task in supernode." + properties: + ID: + type: "string" + description: "ID of the task." + fileLength: + type: "integer" + description: | + The length of the file dfget requests to download in bytes + which including the header and the trailer of each piece. + format: "int64" + httpFileLength: + type: "integer" + description: | + The length of the source file in bytes. + format: "int64" + pieceSize: + type: "integer" + description: | + The size of pieces which is calculated as per the following strategy + 1. If file's total size is less than 200MB, then the piece size is 4MB by default. + 2. Otherwise, it equals to the smaller value between totalSize/100MB + 2 MB and 15MB. + format: "int32" + pieceTotal: + type: "integer" + description: "" + format: "int32" + cdnStatus: + type: "string" + description: | + The status of the created task related to CDN functionality. + enum: ["WAITING", "RUNNING", "FAILED", "SUCCESS", "SOURCE_ERROR"] + rawURL: + type: "string" + description: | + The is the resource's URL which user uses dfget to download. The location of URL can be anywhere, LAN or WAN. + For image distribution, this is image layer's URL in image registry. + The resource url is provided by command line parameter. + taskURL: + type: "string" + description: | + taskURL is generated from rawURL. rawURL may contains some queries or parameter, dfget will filter some queries via + --filter parameter of dfget. The usage of it is that different rawURL may generate the same taskID. + md5: + type: "string" + description: | + md5 checksum for the resource to distribute. dfget catches this parameter from dfget's CLI + and passes it to supernode. When supernode finishes downloading file/image from the source location, + it will validate the source file with this md5 value to check whether this is a valid file. + realMd5: + type: "string" + description: | + when supernode finishes downloading file/image from the source location, + the md5 sum of the source file will be calculated as the value of the realMd5. + And it will be used to compare with md5 value to check whether this is a valid file. + identifier: + type: "string" + description: | + special attribute of remote source file. This field is used with taskURL to generate new taskID to + identify different downloading task of remote source file. For example, if user A and user B uses + the same taskURL and taskID to download file, A and B will share the same peer network to distribute files. + If user A additionally adds an identifier with taskURL, while user B still carries only taskURL, then A's + generated taskID is different from B, and the result is that two users use different peer networks. + headers: + type: "object" + description: | + extra HTTP headers sent to the rawURL. + This field is carried with the request to supernode. + Supernode will extract these HTTP headers, and set them in HTTP downloading requests + from source server as user's wish. + additionalProperties: + type: "string" + + TaskUpdateRequest: + type: "object" + description: "request used to update task attributes." + properties: + peerID: + type: "string" + description: "ID of the peer which has finished to download the whole task." + + PieceInfo: + type: "object" + description: "Peer's detailed information in supernode." + properties: + pID: + type: "string" + description: "the peerID that dfget task should download from" + pieceRange: + type: "string" + description: | + the range of specific piece in the task, example "0-45565". + pieceSize: + type: "integer" + description: | + The size of pieces which is calculated as per the following strategy + 1. If file's total size is less than 200MB, then the piece size is 4MB by default. + 2. Otherwise, it equals to the smaller value between totalSize/100MB + 2 MB and 15MB. + format: int32 + pieceMD5: + type: "string" + description: | + the MD5 information of piece which is generated by supernode when doing CDN cache. + This value will be returned to dfget in order to validate the piece's completeness. + peerIP: + type: string + description: | + When dfget needs to download a piece from another peer. Supernode will return a PieceInfo + that contains a peerIP. This peerIP represents the IP of this dfget's target peer. + peerPort: + type: "integer" + format: "int32" + description: | + When dfget needs to download a piece from another peer. Supernode will return a PieceInfo + that contains a peerPort. This peerPort represents the port of this dfget's target peer's uploader. + path: + type: "string" + description: | + The URL path to download the specific piece from the target peer's uploader. + + PieceUpdateRequest: + type: "object" + description: "request used to update piece attributes." + properties: + clientID: + type: "string" + description: | + the downloader clientID + dstPID: + type: "string" + description: | + the uploader peerID + pieceStatus: + type: "string" + description: | + pieceStatus indicates whether the peer task successfully download the piece. + enum: ["FAILED", "SUCCESS", "INVALID", "SEMISUC"] + + PiecePullRequest: + type: "object" + description: "request used to pull pieces that have not been downloaded." + properties: + dstPID: + type: "string" + description: | + the uploader peerID + dfgetTaskStatus: + type: "string" + description: | + dfgetTaskStatus indicates whether the dfgetTask is running. + enum: ["STARTED", "RUNNING", "FINISHED"] + pieceResult: + type: "string" + description: | + pieceResult It indicates whether the dfgetTask successfully download the piece. + It's only useful when `status` is `RUNNING`. + enum: ["FAILED", "SUCCESS", "INVALID", "SEMISUC"] + pieceRange: + type: "string" + description: | + the range of specific piece in the task, example "0-45565". + + PreheatInfo: + type: "object" + description: | + return detailed information of a preheat task in supernode. An image preheat task may contain multiple downloading + task because that an image may have more than one layer. + properties: + ID: + type: "string" + description: | + ID of preheat task. + status: + type: "string" + description: | + The status of preheat task. + WAITING -----> RUNNING -----> SUCCESS + |--> FAILED + The initial status of a created preheat task is WAITING. + It's finished when a preheat task's status is FAILED or SUCCESS. + A finished preheat task's information can be queried within 24 hours. + enum: ["WAITING", "RUNNING", "FAILED", "SUCCESS"] + startTime: + type: "string" + format: "date-time" + description: "the preheat task start time" + finishTime: + type: "string" + format: "date-time" + description: "the preheat task finish time" + + PreheatCreateRequest: + type: "object" + description: | + Request option of creating a preheat task in supernode. + properties: + type: + type: "string" + description: | + this must be image or file + url: + type: "string" + description: "the image or file location" + filter: + type: "string" + description: | + URL may contains some changeful query parameters such as authentication parameters. Dragonfly will + filter these parameter via 'filter'. The usage of it is that different URL may generate the same + download taskID. + identifier: + type: "string" + description: | + This field is used for generating new downloading taskID to identify different downloading task of remote URL. + headers: + type: "object" + description: | + If there is any authentication step of the remote server, the headers should contains authenticated information. + Dragonfly will sent request taking the headers to remote server. + additionalProperties: + type: "string" + + PreheatCreateResponse: + type: "object" + description: | + Response of a preheat creation request. + properties: + ID: + type: "string" + + DfGetTask: + type: "object" + description: | + A download process initiated by dfget or other clients. + properties: + taskId: + type: "string" + pieceSize: + type: "integer" + description: | + The size of pieces which is calculated as per the following strategy + 1. If file's total size is less than 200MB, then the piece size is 4MB by default. + 2. Otherwise, it equals to the smaller value between totalSize/100MB + 2 MB and 15MB. + format: int32 + cID: + type: "string" + description: | + CID means the client ID. It maps to the specific dfget process. + When user wishes to download an image/file, user would start a dfget process to do this. + This dfget is treated a client and carries a client ID. + Thus, multiple dfget processes on the same peer have different CIDs. + path: + type: "string" + description: | + path is used in one peer A for uploading functionality. When peer B hopes + to get piece C from peer A, B must provide a URL for piece C. + Then when creating a task in supernode, peer A must provide this URL in request. + status: + type: "string" + description: | + The status of Dfget download process. + enum: ["WAITING", "RUNNING", "FAILED", "SUCCESS",] + peerID: + type: "string" + description: | + PeerID uniquely identifies a peer, and the cID uniquely identifies a + download task belonging to a peer. One peer can initiate multiple download tasks, + which means that one peer corresponds to multiple cIDs. + supernodeIP: + type: "string" + description: "IP address of supernode which the peer connects to" + dfdaemon: + type: "boolean" + description: | + tells whether it is a call from dfdaemon. dfdaemon is a long running + process which works for container engines. It translates the image + pulling request into raw requests into those dfget recganises. + callSystem: + type: "string" + description: | + This attribute represents where the dfget requests come from. Dfget will pass + this field to supernode and supernode can do some checking and filtering via + black/white list mechanism to guarantee security, or some other purposes like debugging. + minLength: 1 + + ErrorResponse: + type: "object" + description: | + It contains a code that identify which error occurred for client processing and a detailed error message to read. + properties: + code: + type: "integer" + description: | + the code of this error, it's convenient for client to process with certain error. + message: + type: "string" + description: "detailed error message" + + +responses: + 401ErrorResponse: + description: An unexpected 401 error occurred. + schema: + $ref: "#/definitions/Error" + 404ErrorResponse: + description: An unexpected 404 error occurred. + schema: + $ref: "#/definitions/Error" + 500ErrorResponse: + description: An unexpected server error occurred. + schema: + $ref: "#/definitions/Error" diff --git a/apis/types/task_register_request.go b/apis/types/task_register_request.go index 4b20c61d1..85a5def96 100644 --- a/apis/types/task_register_request.go +++ b/apis/types/task_register_request.go @@ -60,6 +60,10 @@ type TaskRegisterRequest struct { // Identifier string `json:"identifier,omitempty"` + // tells whether skip secure verify when supernode download the remote source file. + // + Insecure bool `json:"insecure,omitempty"` + // md5 checksum for the resource to distribute. dfget catches this parameter from dfget's CLI // and passes it to supernode. When supernode finishes downloading file/image from the source location, // it will validate the source file with this md5 value to check whether this is a valid file. @@ -86,6 +90,10 @@ type TaskRegisterRequest struct { // RawURL string `json:"rawURL,omitempty"` + // The root ca cert from client used to download the remote source file. + // + RootCAs []strfmt.Base64 `json:"rootCAs"` + // The address of supernode that the client can connect to SuperNodeIP string `json:"superNodeIp,omitempty"` @@ -118,6 +126,10 @@ func (m *TaskRegisterRequest) Validate(formats strfmt.Registry) error { res = append(res, err) } + if err := m.validateRootCAs(formats); err != nil { + res = append(res, err) + } + if len(res) > 0 { return errors.CompositeValidationError(res...) } @@ -180,6 +192,21 @@ func (m *TaskRegisterRequest) validatePort(formats strfmt.Registry) error { return nil } +func (m *TaskRegisterRequest) validateRootCAs(formats strfmt.Registry) error { + + if swag.IsZero(m.RootCAs) { // not required + return nil + } + + for i := 0; i < len(m.RootCAs); i++ { + + // Format "byte" (base64 string) is already validated when unmarshalled + + } + + return nil +} + // MarshalBinary interface implementation func (m *TaskRegisterRequest) MarshalBinary() ([]byte, error) { if m == nil { diff --git a/cmd/dfget/app/root.go b/cmd/dfget/app/root.go index dcd4ecac4..ddb187c2d 100644 --- a/cmd/dfget/app/root.go +++ b/cmd/dfget/app/root.go @@ -210,6 +210,8 @@ func initFlags() { flagSet.StringVar(&cfg.CallSystem, "callsystem", "", "The name of dfget caller which is for debugging. Once set, it will be passed to all components around the request to make debugging easy") + flagSet.StringSliceVar(&cfg.Cacerts, "cacerts", nil, + "The cacert file which is used to verify remote server when supernode interact with the source.") flagSet.StringVarP(&cfg.Pattern, "pattern", "p", "p2p", "download pattern, must be p2p/cdn/source, cdn and source do not support flag --totallimit") flagSet.StringVarP(&filter, "filter", "f", "", @@ -224,6 +226,8 @@ func initFlags() { "disable back source downloading for requested file when p2p fails to download it") flagSet.BoolVar(&cfg.DFDaemon, "dfdaemon", false, "identify whether the request is from dfdaemon") + flagSet.BoolVar(&cfg.Insecure, "insecure", false, + "identify whether supernode should skip secure verify when interact with the source.") flagSet.IntVar(&cfg.ClientQueueSize, "clientqueue", config.DefaultClientQueueSize, "specify the size of client queue which controls the number of pieces that can be processed simultaneously") diff --git a/dfdaemon/config/config.go b/dfdaemon/config/config.go index 4e4b68448..15877b330 100644 --- a/dfdaemon/config/config.go +++ b/dfdaemon/config/config.go @@ -147,22 +147,27 @@ func (p *Properties) DFGetConfig() DFGetConfig { dfgetFlags = append(dfgetFlags, "--verbose") } - return DFGetConfig{ + dfgetConfig := DFGetConfig{ DfgetFlags: dfgetFlags, SuperNodes: p.SuperNodes, RateLimit: p.RateLimit, DFRepo: p.DFRepo, DFPath: p.DFPath, } + if p.HijackHTTPS != nil { + dfgetConfig.HostsConfig = p.HijackHTTPS.Hosts + } + return dfgetConfig } // DFGetConfig configures how dfdaemon calls dfget type DFGetConfig struct { - DfgetFlags []string `yaml:"dfget_flags"` - SuperNodes []string `yaml:"supernodes"` - RateLimit string `yaml:"ratelimit"` - DFRepo string `yaml:"localrepo"` - DFPath string `yaml:"dfpath"` + DfgetFlags []string `yaml:"dfget_flags"` + SuperNodes []string `yaml:"supernodes"` + RateLimit string `yaml:"ratelimit"` + DFRepo string `yaml:"localrepo"` + DFPath string `yaml:"dfpath"` + HostsConfig []*HijackHost `yaml:"hosts" json:"hosts"` } // RegistryMirror configures the mirror of the official docker registry @@ -261,7 +266,7 @@ func (u *URL) MarshalYAML() (interface{}, error) { // CertPool is a wrapper around x509.CertPool, which can be unmarshalled and // constructed from a list of filenames type CertPool struct { - files []string + Files []string *x509.CertPool } @@ -276,11 +281,11 @@ func (cp *CertPool) UnmarshalJSON(b []byte) error { } func (cp *CertPool) unmarshal(unmarshal func(interface{}) error) error { - if err := unmarshal(&cp.files); err != nil { + if err := unmarshal(&cp.Files); err != nil { return err } - pool, err := certPoolFromFiles(cp.files...) + pool, err := certPoolFromFiles(cp.Files...) if err != nil { return err } @@ -291,12 +296,12 @@ func (cp *CertPool) unmarshal(unmarshal func(interface{}) error) error { // MarshalJSON implements json.Marshaller to print the cert pool func (cp *CertPool) MarshalJSON() ([]byte, error) { - return json.Marshal(cp.files) + return json.Marshal(cp.Files) } // MarshalYAML implements yaml.Marshaller to print the cert pool func (cp *CertPool) MarshalYAML() (interface{}, error) { - return cp.files, nil + return cp.Files, nil } // Regexp is simple wrapper around regexp.Regexp to make it unmarshallable from a string diff --git a/dfdaemon/downloader/dfget/dfget.go b/dfdaemon/downloader/dfget/dfget.go index a36a087e2..779e4903e 100644 --- a/dfdaemon/downloader/dfget/dfget.go +++ b/dfdaemon/downloader/dfget/dfget.go @@ -18,17 +18,18 @@ package dfget import ( "fmt" + netUrl "net/url" "os/exec" "path/filepath" "strings" "syscall" "time" - log "github.com/sirupsen/logrus" - "github.com/dragonflyoss/Dragonfly/dfdaemon/config" "github.com/dragonflyoss/Dragonfly/dfdaemon/constant" "github.com/dragonflyoss/Dragonfly/dfdaemon/exception" + + log "github.com/sirupsen/logrus" ) // DFGetter implements Downloader to download file by dragonfly @@ -92,5 +93,17 @@ func (dfGetter *DFGetter) getCommand( } } + urlInfo, _ := netUrl.Parse(url) + for _, h := range dfGetter.config.HostsConfig { + if urlInfo != nil && h.Regx.MatchString(urlInfo.Host) { + if h.Insecure { + args = append(args, "--insecure") + } + if h.Certs != nil && len(h.Certs.Files) != 0 { + add("--cacerts", strings.Join(h.Certs.Files, ",")) + } + } + } + return exec.Command(dfGetter.config.DFPath, args...) } diff --git a/dfget/config/config.go b/dfget/config/config.go index dc0f194af..4a85d3e3e 100644 --- a/dfget/config/config.go +++ b/dfget/config/config.go @@ -167,6 +167,9 @@ type Config struct { // default:`p2p`. Pattern string `json:"pattern,omitempty"` + // CA certificate to verify when supernode interact with the source. + Cacerts []string `json:"cacert,omitempty"` + // Filter filter some query params of url, use char '&' to separate different params. // eg: -f 'key&sign' will filter 'key' and 'sign' query param. // in this way, different urls correspond one same download task that can use p2p mode. @@ -185,6 +188,9 @@ type Config struct { // DFDaemon indicates whether the caller is from dfdaemon DFDaemon bool `json:"dfdaemon,omitempty"` + // Insecure indicates whether skip secure verify when supernode interact with the source. + Insecure bool `json:"insecure,omitempty"` + // Version show version. Version bool `json:"version,omitempty"` diff --git a/dfget/core/regist/register.go b/dfget/core/regist/register.go index e21248289..fd9816f19 100644 --- a/dfget/core/regist/register.go +++ b/dfget/core/regist/register.go @@ -17,6 +17,7 @@ package regist import ( + "io/ioutil" "os" "time" @@ -139,12 +140,23 @@ func (s *supernodeRegister) constructRegisterRequest(port int) *types.RegisterRe CallSystem: cfg.CallSystem, Headers: cfg.Header, Dfdaemon: cfg.DFDaemon, + Insecure: cfg.Insecure, } if cfg.Md5 != "" { req.Md5 = cfg.Md5 } else if cfg.Identifier != "" { req.Identifier = cfg.Identifier } + + for _, certPath := range cfg.Cacerts { + caBytes, err := ioutil.ReadFile(certPath) + if err != nil { + logrus.Errorf("read cert file fail:%v", err) + continue + } + req.RootCAs = append(req.RootCAs, caBytes) + } + return req } diff --git a/dfget/types/register_request.go b/dfget/types/register_request.go index dc7fbdea0..614a472b6 100644 --- a/dfget/types/register_request.go +++ b/dfget/types/register_request.go @@ -37,6 +37,8 @@ type RegisterRequest struct { CallSystem string `json:"callSystem,omitempty"` Headers []string `json:"headers,omitempty"` Dfdaemon bool `json:"dfdaemon,omitempty"` + Insecure bool `json:"insecure,omitempty"` + RootCAs [][]byte `json:"rootCAs,omitempty"` } func (r *RegisterRequest) String() string { diff --git a/pkg/httputils/http_util.go b/pkg/httputils/http_util.go index ca7e5c77a..a618f4add 100644 --- a/pkg/httputils/http_util.go +++ b/pkg/httputils/http_util.go @@ -29,8 +29,6 @@ import ( "time" "github.com/dragonflyoss/Dragonfly/pkg/errortypes" - "github.com/dragonflyoss/Dragonfly/pkg/netutils" - "github.com/dragonflyoss/Dragonfly/pkg/stringutils" "github.com/dragonflyoss/Dragonfly/pkg/util" "github.com/pkg/errors" @@ -293,67 +291,6 @@ func CheckConnect(ip string, port int, timeout int) (localIP string, e error) { return } -// IsExpired checks if a resource received or stored is the same. -func IsExpired(url string, headers map[string]string, lastModified int64, eTag string) (bool, error) { - if lastModified <= 0 && stringutils.IsEmptyStr(eTag) { - return true, nil - } - - // set headers - if headers == nil { - headers = make(map[string]string) - } - if lastModified > 0 { - lastModifiedStr, _ := netutils.ConvertTimeIntToString(lastModified) - headers["If-Modified-Since"] = lastModifiedStr - } - if !stringutils.IsEmptyStr(eTag) { - headers["If-None-Match"] = eTag - } - - // send request - resp, err := HTTPGetTimeout(url, headers, 4*time.Second) - if err != nil { - return false, err - } - resp.Body.Close() - - return resp.StatusCode != http.StatusNotModified, nil -} - -// IsSupportRange checks if the source url support partial requests. -func IsSupportRange(url string, headers map[string]string) (bool, error) { - // set headers - if headers == nil { - headers = make(map[string]string) - } - headers["Range"] = "bytes=0-0" - - // send request - resp, err := HTTPGetTimeout(url, headers, 4*time.Second) - if err != nil { - return false, err - } - resp.Body.Close() - - if resp.StatusCode == http.StatusPartialContent { - return true, nil - } - return false, nil -} - -// GetContentLength send a head request to get file length. -func GetContentLength(url string, headers map[string]string) (int64, int, error) { - // send request - resp, err := HTTPGetTimeout(url, headers, 4*time.Second) - if err != nil { - return 0, 0, err - } - resp.Body.Close() - - return resp.ContentLength, resp.StatusCode, nil -} - // ConstructRangeStr wrap the rangeStr as a HTTP Range header value. func ConstructRangeStr(rangeStr string) string { return fmt.Sprintf("bytes=%s", rangeStr) diff --git a/supernode/daemon/mgr/cdn/cache_detector.go b/supernode/daemon/mgr/cdn/cache_detector.go index b033d6837..b28929acb 100644 --- a/supernode/daemon/mgr/cdn/cache_detector.go +++ b/supernode/daemon/mgr/cdn/cache_detector.go @@ -4,8 +4,8 @@ import ( "context" "github.com/dragonflyoss/Dragonfly/apis/types" - "github.com/dragonflyoss/Dragonfly/pkg/httputils" "github.com/dragonflyoss/Dragonfly/pkg/stringutils" + "github.com/dragonflyoss/Dragonfly/supernode/httpclient" "github.com/dragonflyoss/Dragonfly/supernode/store" "github.com/sirupsen/logrus" @@ -14,12 +14,14 @@ import ( type cacheDetector struct { cacheStore *store.Store metaDataManager *fileMetaDataManager + OriginClient httpclient.OriginHTTPClient } -func newCacheDetector(cacheStore *store.Store, metaDataManager *fileMetaDataManager) *cacheDetector { +func newCacheDetector(cacheStore *store.Store, metaDataManager *fileMetaDataManager, originClient httpclient.OriginHTTPClient) *cacheDetector { return &cacheDetector{ cacheStore: cacheStore, metaDataManager: metaDataManager, + OriginClient: originClient, } } @@ -50,7 +52,7 @@ func (cd *cacheDetector) detectCache(ctx context.Context, task *types.TaskInfo) } func (cd *cacheDetector) parseBreakNum(ctx context.Context, task *types.TaskInfo, metaData *fileMetaData) int { - expired, err := httputils.IsExpired(task.RawURL, task.Headers, metaData.LastModified, metaData.ETag) + expired, err := cd.OriginClient.IsExpired(task.RawURL, task.Headers, metaData.LastModified, metaData.ETag) if err != nil { logrus.Errorf("failed to check whether the task(%s) has expired: %v", task.ID, err) } @@ -67,7 +69,7 @@ func (cd *cacheDetector) parseBreakNum(ctx context.Context, task *types.TaskInfo return 0 } - supportRange, err := httputils.IsSupportRange(task.TaskURL, task.Headers) + supportRange, err := cd.OriginClient.IsSupportRange(task.TaskURL, task.Headers) if err != nil { logrus.Errorf("failed to check whether the task(%s) supports partial requests: %v", task.ID, err) } diff --git a/supernode/daemon/mgr/cdn/downloader.go b/supernode/daemon/mgr/cdn/downloader.go index 8e30ec28f..790766ca0 100644 --- a/supernode/daemon/mgr/cdn/downloader.go +++ b/supernode/daemon/mgr/cdn/downloader.go @@ -2,7 +2,6 @@ package cdn import ( "context" - "fmt" "net/http" errorType "github.com/dragonflyoss/Dragonfly/pkg/errortypes" @@ -36,18 +35,5 @@ func (cm *Manager) download(ctx context.Context, taskID, url string, headers map } logrus.Infof("start to download for taskId(%s) with fileUrl: %s header: %v checkCode: %d", taskID, url, headers, checkCode) - return getWithURL(url, headers, checkCode) -} - -func getWithURL(url string, headers map[string]string, checkCode int) (*http.Response, error) { - // TODO: add timeout - resp, err := httputils.HTTPGet(url, headers) - if err != nil { - return nil, err - } - - if resp.StatusCode == checkCode { - return resp, nil - } - return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode) + return cm.originClient.Download(url, headers, checkCode) } diff --git a/supernode/daemon/mgr/cdn/downloader_test.go b/supernode/daemon/mgr/cdn/downloader_test.go index 12e0a5944..4ad381d8b 100644 --- a/supernode/daemon/mgr/cdn/downloader_test.go +++ b/supernode/daemon/mgr/cdn/downloader_test.go @@ -28,6 +28,7 @@ import ( "github.com/dragonflyoss/Dragonfly/pkg/httputils" "github.com/dragonflyoss/Dragonfly/pkg/stringutils" "github.com/dragonflyoss/Dragonfly/supernode/config" + "github.com/dragonflyoss/Dragonfly/supernode/httpclient" "github.com/go-check/check" ) @@ -44,7 +45,7 @@ func init() { } func (s *CDNDownloadTestSuite) TestDownload(c *check.C) { - cm, _ := NewManager(config.NewConfig(), nil, nil) + cm, _ := NewManager(config.NewConfig(), nil, nil, httpclient.NewOriginClient()) bytes := []byte("hello world") bytesLength := int64(len(bytes)) diff --git a/supernode/daemon/mgr/cdn/manager.go b/supernode/daemon/mgr/cdn/manager.go index 7e3d1f8bd..1d9301140 100644 --- a/supernode/daemon/mgr/cdn/manager.go +++ b/supernode/daemon/mgr/cdn/manager.go @@ -15,6 +15,7 @@ import ( "github.com/dragonflyoss/Dragonfly/supernode/store" "github.com/dragonflyoss/Dragonfly/supernode/util" + "github.com/dragonflyoss/Dragonfly/supernode/httpclient" "github.com/sirupsen/logrus" ) @@ -31,12 +32,13 @@ type Manager struct { metaDataManager *fileMetaDataManager cdnReporter *reporter detector *cacheDetector + originClient httpclient.OriginHTTPClient pieceMD5Manager *pieceMD5Mgr writer *superWriter } // NewManager returns a new Manager. -func NewManager(cfg *config.Config, cacheStore *store.Store, progressManager mgr.ProgressMgr) (*Manager, error) { +func NewManager(cfg *config.Config, cacheStore *store.Store, progressManager mgr.ProgressMgr, originClient httpclient.OriginHTTPClient) (*Manager, error) { rateLimiter := ratelimiter.NewRateLimiter(ratelimiter.TransRate(config.TransLimit(cfg.MaxBandwidth-cfg.SystemReservedBandwidth)), 2) metaDataManager := newFileMetaDataManager(cacheStore) pieceMD5Manager := newpieceMD5Mgr() @@ -50,7 +52,8 @@ func NewManager(cfg *config.Config, cacheStore *store.Store, progressManager mgr metaDataManager: metaDataManager, pieceMD5Manager: pieceMD5Manager, cdnReporter: cdnReporter, - detector: newCacheDetector(cacheStore, metaDataManager), + detector: newCacheDetector(cacheStore, metaDataManager, originClient), + originClient: originClient, writer: newSuperWriter(cacheStore, cdnReporter), }, nil } diff --git a/supernode/daemon/mgr/task/manager.go b/supernode/daemon/mgr/task/manager.go index faf602a08..fa6452f54 100644 --- a/supernode/daemon/mgr/task/manager.go +++ b/supernode/daemon/mgr/task/manager.go @@ -6,13 +6,13 @@ import ( "github.com/dragonflyoss/Dragonfly/apis/types" "github.com/dragonflyoss/Dragonfly/pkg/errortypes" - "github.com/dragonflyoss/Dragonfly/pkg/httputils" "github.com/dragonflyoss/Dragonfly/pkg/stringutils" "github.com/dragonflyoss/Dragonfly/pkg/syncmap" "github.com/dragonflyoss/Dragonfly/pkg/timeutils" "github.com/dragonflyoss/Dragonfly/supernode/config" "github.com/dragonflyoss/Dragonfly/supernode/daemon/mgr" dutil "github.com/dragonflyoss/Dragonfly/supernode/daemon/util" + "github.com/dragonflyoss/Dragonfly/supernode/httpclient" "github.com/dragonflyoss/Dragonfly/supernode/util" "github.com/pkg/errors" @@ -25,10 +25,6 @@ const ( var _ mgr.TaskMgr = &Manager{} -// using a variable getContentLength to reference the function util.GetContentLength, -// and it helps using stub functions in the test with gostub. -var getContentLength = httputils.GetContentLength - // Manager is an implementation of the interface of TaskMgr. type Manager struct { cfg *config.Config @@ -43,11 +39,12 @@ type Manager struct { progressMgr mgr.ProgressMgr cdnMgr mgr.CDNMgr schedulerMgr mgr.SchedulerMgr + OriginClient httpclient.OriginHTTPClient } // NewManager returns a new Manager Object. func NewManager(cfg *config.Config, peerMgr mgr.PeerMgr, dfgetTaskMgr mgr.DfgetTaskMgr, - progressMgr mgr.ProgressMgr, cdnMgr mgr.CDNMgr, schedulerMgr mgr.SchedulerMgr) (*Manager, error) { + progressMgr mgr.ProgressMgr, cdnMgr mgr.CDNMgr, schedulerMgr mgr.SchedulerMgr, originClient httpclient.OriginHTTPClient) (*Manager, error) { return &Manager{ cfg: cfg, taskStore: dutil.NewStore(), @@ -59,6 +56,7 @@ func NewManager(cfg *config.Config, peerMgr mgr.PeerMgr, dfgetTaskMgr mgr.DfgetT schedulerMgr: schedulerMgr, accessTimeMap: syncmap.NewSyncMap(), taskURLUnReachableStore: syncmap.NewSyncMap(), + OriginClient: originClient, }, nil } diff --git a/supernode/daemon/mgr/task/manager_test.go b/supernode/daemon/mgr/task/manager_test.go index 3c84d7197..e79c830f1 100644 --- a/supernode/daemon/mgr/task/manager_test.go +++ b/supernode/daemon/mgr/task/manager_test.go @@ -25,10 +25,10 @@ import ( "github.com/dragonflyoss/Dragonfly/supernode/config" "github.com/dragonflyoss/Dragonfly/supernode/daemon/mgr/mock" dutil "github.com/dragonflyoss/Dragonfly/supernode/daemon/util" + cMock "github.com/dragonflyoss/Dragonfly/supernode/httpclient/mock" "github.com/go-check/check" "github.com/golang/mock/gomock" - "github.com/prashantv/gostub" ) func Test(t *testing.T) { @@ -46,9 +46,9 @@ type TaskMgrTestSuite struct { mockPeerMgr *mock.MockPeerMgr mockProgressMgr *mock.MockProgressMgr mockSchedulerMgr *mock.MockSchedulerMgr + mockOriginClient *cMock.MockOriginHTTPClient - taskManager *Manager - contentLengthStub *gostub.Stubs + taskManager *Manager } func (s *TaskMgrTestSuite) SetUpSuite(c *check.C) { @@ -59,22 +59,19 @@ func (s *TaskMgrTestSuite) SetUpSuite(c *check.C) { s.mockDfgetTaskMgr = mock.NewMockDfgetTaskMgr(s.mockCtl) s.mockProgressMgr = mock.NewMockProgressMgr(s.mockCtl) s.mockSchedulerMgr = mock.NewMockSchedulerMgr(s.mockCtl) + s.mockOriginClient = cMock.NewMockOriginHTTPClient(s.mockCtl) s.mockCDNMgr.EXPECT().TriggerCDN(gomock.Any(), gomock.Any()).Return(nil, nil).AnyTimes() s.mockDfgetTaskMgr.EXPECT().Add(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() s.mockProgressMgr.EXPECT().InitProgress(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() - + s.mockOriginClient.EXPECT().GetContentLength(gomock.Any(), gomock.Any()).Return(int64(1000), 200, nil) cfg := config.NewConfig() - s.taskManager, _ = NewManager(cfg, s.mockPeerMgr, s.mockDfgetTaskMgr, - s.mockProgressMgr, s.mockCDNMgr, s.mockSchedulerMgr) - s.contentLengthStub = gostub.Stub(&getContentLength, func(url string, headers map[string]string) (int64, int, error) { - return 1000, 200, nil - }) + s.taskManager, _ = NewManager(cfg, s.mockPeerMgr, s.mockDfgetTaskMgr, + s.mockProgressMgr, s.mockCDNMgr, s.mockSchedulerMgr, s.mockOriginClient) } func (s *TaskMgrTestSuite) TearDownSuite(c *check.C) { - s.contentLengthStub.Reset() s.mockCtl.Finish() } @@ -164,5 +161,4 @@ func (s *TaskMgrTestSuite) TestUpdateTaskInfo(c *check.C) { c.Check(err, check.IsNil) c.Check(task.CdnStatus, check.Equals, types.TaskInfoCdnStatusSUCCESS) c.Check(task.FileLength, check.Equals, int64(2000)) - } diff --git a/supernode/daemon/mgr/task/manager_util.go b/supernode/daemon/mgr/task/manager_util.go index 7b63d7c1c..1f795f7f0 100644 --- a/supernode/daemon/mgr/task/manager_util.go +++ b/supernode/daemon/mgr/task/manager_util.go @@ -82,7 +82,7 @@ func (tm *Manager) addOrUpdateTask(ctx context.Context, req *types.TaskCreateReq } // get fileLength with req.Headers - fileLength, err := getHTTPFileLength(taskID, task.RawURL, req.Headers) + fileLength, err := tm.getHTTPFileLength(taskID, task.RawURL, req.Headers) if err != nil { if errortypes.IsURLNotReachable(err) { tm.taskURLUnReachableStore.Add(taskID, time.Now()) @@ -496,8 +496,8 @@ func isWait(CDNStatus string) bool { return CDNStatus == types.TaskInfoCdnStatusWAITING } -func getHTTPFileLength(taskID, url string, headers map[string]string) (int64, error) { - fileLength, code, err := getContentLength(url, headers) +func (tm *Manager) getHTTPFileLength(taskID, url string, headers map[string]string) (int64, error) { + fileLength, code, err := tm.OriginClient.GetContentLength(url, headers) if err != nil { return -1, errors.Wrapf(errortypes.ErrUnknowError, "failed to get http file Length: %v", err) } diff --git a/supernode/daemon/mgr/task/manager_util_test.go b/supernode/daemon/mgr/task/manager_util_test.go index 3fb7cf944..5edfc21b8 100644 --- a/supernode/daemon/mgr/task/manager_util_test.go +++ b/supernode/daemon/mgr/task/manager_util_test.go @@ -20,10 +20,10 @@ import ( "github.com/dragonflyoss/Dragonfly/apis/types" "github.com/dragonflyoss/Dragonfly/supernode/config" "github.com/dragonflyoss/Dragonfly/supernode/daemon/mgr/mock" + cMock "github.com/dragonflyoss/Dragonfly/supernode/httpclient/mock" "github.com/go-check/check" "github.com/golang/mock/gomock" - "github.com/prashantv/gostub" ) func init() { @@ -37,9 +37,9 @@ type TaskUtilTestSuite struct { mockPeerMgr *mock.MockPeerMgr mockProgressMgr *mock.MockProgressMgr mockSchedulerMgr *mock.MockSchedulerMgr + mockOriginClient *cMock.MockOriginHTTPClient - taskManager *Manager - contentLengthStub *gostub.Stubs + taskManager *Manager } func (s *TaskUtilTestSuite) SetUpSuite(c *check.C) { @@ -50,16 +50,15 @@ func (s *TaskUtilTestSuite) SetUpSuite(c *check.C) { s.mockDfgetTaskMgr = mock.NewMockDfgetTaskMgr(s.mockCtl) s.mockProgressMgr = mock.NewMockProgressMgr(s.mockCtl) s.mockSchedulerMgr = mock.NewMockSchedulerMgr(s.mockCtl) - s.taskManager, _ = NewManager(config.NewConfig(), s.mockPeerMgr, s.mockDfgetTaskMgr, - s.mockProgressMgr, s.mockCDNMgr, s.mockSchedulerMgr) + s.mockOriginClient = cMock.NewMockOriginHTTPClient(s.mockCtl) - s.contentLengthStub = gostub.Stub(&getContentLength, func(url string, headers map[string]string) (int64, int, error) { - return 1000, 200, nil - }) + s.taskManager, _ = NewManager(config.NewConfig(), s.mockPeerMgr, s.mockDfgetTaskMgr, + s.mockProgressMgr, s.mockCDNMgr, s.mockSchedulerMgr, s.mockOriginClient) + s.mockOriginClient.EXPECT().GetContentLength(gomock.Any(), gomock.Any()).Return(int64(1000), 200, nil) } func (s *TaskUtilTestSuite) TearDownSuite(c *check.C) { - s.contentLengthStub.Reset() + s.mockCtl.Finish() } func (s *TaskUtilTestSuite) TestEqualsTask(c *check.C) { diff --git a/supernode/httpclient/mock/mock_origin_http_client.go b/supernode/httpclient/mock/mock_origin_http_client.go new file mode 100644 index 000000000..8000928b1 --- /dev/null +++ b/supernode/httpclient/mock/mock_origin_http_client.go @@ -0,0 +1,109 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: supernode/httpclient/origin_http_client.go + +// Package mock is a generated GoMock package. +package mock + +import ( + http "net/http" + reflect "reflect" + + strfmt "github.com/go-openapi/strfmt" + gomock "github.com/golang/mock/gomock" +) + +// MockOriginHTTPClient is a mock of OriginHTTPClient interface +type MockOriginHTTPClient struct { + ctrl *gomock.Controller + recorder *MockOriginHTTPClientMockRecorder +} + +// MockOriginHTTPClientMockRecorder is the mock recorder for MockOriginHTTPClient +type MockOriginHTTPClientMockRecorder struct { + mock *MockOriginHTTPClient +} + +// NewMockOriginHTTPClient creates a new mock instance +func NewMockOriginHTTPClient(ctrl *gomock.Controller) *MockOriginHTTPClient { + mock := &MockOriginHTTPClient{ctrl: ctrl} + mock.recorder = &MockOriginHTTPClientMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockOriginHTTPClient) EXPECT() *MockOriginHTTPClientMockRecorder { + return m.recorder +} + +// RegisterTLSConfig mocks base method +func (m *MockOriginHTTPClient) RegisterTLSConfig(rawURL string, insecure bool, caBlock []strfmt.Base64) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "RegisterTLSConfig", rawURL, insecure, caBlock) +} + +// RegisterTLSConfig indicates an expected call of RegisterTLSConfig +func (mr *MockOriginHTTPClientMockRecorder) RegisterTLSConfig(rawURL, insecure, caBlock interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RegisterTLSConfig", reflect.TypeOf((*MockOriginHTTPClient)(nil).RegisterTLSConfig), rawURL, insecure, caBlock) +} + +// GetContentLength mocks base method +func (m *MockOriginHTTPClient) GetContentLength(url string, headers map[string]string) (int64, int, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetContentLength", url, headers) + ret0, _ := ret[0].(int64) + ret1, _ := ret[1].(int) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// GetContentLength indicates an expected call of GetContentLength +func (mr *MockOriginHTTPClientMockRecorder) GetContentLength(url, headers interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetContentLength", reflect.TypeOf((*MockOriginHTTPClient)(nil).GetContentLength), url, headers) +} + +// IsSupportRange mocks base method +func (m *MockOriginHTTPClient) IsSupportRange(url string, headers map[string]string) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsSupportRange", url, headers) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// IsSupportRange indicates an expected call of IsSupportRange +func (mr *MockOriginHTTPClientMockRecorder) IsSupportRange(url, headers interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsSupportRange", reflect.TypeOf((*MockOriginHTTPClient)(nil).IsSupportRange), url, headers) +} + +// IsExpired mocks base method +func (m *MockOriginHTTPClient) IsExpired(url string, headers map[string]string, lastModified int64, eTag string) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsExpired", url, headers, lastModified, eTag) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// IsExpired indicates an expected call of IsExpired +func (mr *MockOriginHTTPClientMockRecorder) IsExpired(url, headers, lastModified, eTag interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsExpired", reflect.TypeOf((*MockOriginHTTPClient)(nil).IsExpired), url, headers, lastModified, eTag) +} + +// Download mocks base method +func (m *MockOriginHTTPClient) Download(url string, headers map[string]string, checkCode int) (*http.Response, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Download", url, headers, checkCode) + ret0, _ := ret[0].(*http.Response) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Download indicates an expected call of Download +func (mr *MockOriginHTTPClientMockRecorder) Download(url, headers, checkCode interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Download", reflect.TypeOf((*MockOriginHTTPClient)(nil).Download), url, headers, checkCode) +} diff --git a/supernode/httpclient/origin_http_client.go b/supernode/httpclient/origin_http_client.go new file mode 100644 index 000000000..9f7c023a3 --- /dev/null +++ b/supernode/httpclient/origin_http_client.go @@ -0,0 +1,183 @@ +package httpclient + +import ( + "context" + "crypto/tls" + "crypto/x509" + "fmt" + "net" + "net/http" + netUrl "net/url" + "sync" + "time" + + "github.com/dragonflyoss/Dragonfly/pkg/errortypes" + "github.com/dragonflyoss/Dragonfly/pkg/netutils" + "github.com/dragonflyoss/Dragonfly/pkg/stringutils" + + strfmt "github.com/go-openapi/strfmt" + "github.com/pkg/errors" +) + +// OriginHTTPClient supply apis that interact with the source. +type OriginHTTPClient interface { + RegisterTLSConfig(rawURL string, insecure bool, caBlock []strfmt.Base64) + GetContentLength(url string, headers map[string]string) (int64, int, error) + IsSupportRange(url string, headers map[string]string) (bool, error) + IsExpired(url string, headers map[string]string, lastModified int64, eTag string) (bool, error) + Download(url string, headers map[string]string, checkCode int) (*http.Response, error) +} + +// OriginClient is an implementation of the interface of OriginHTTPClient. +type OriginClient struct { + clientMap *sync.Map +} + +// NewOriginClient returns a new OriginClient. +func NewOriginClient() OriginHTTPClient { + return &OriginClient{ + clientMap: &sync.Map{}, + } +} + +// RegisterTLSConfig save tls config into map as http client. +// tlsMap: +// key->host value->*http.Client +func (client *OriginClient) RegisterTLSConfig(rawURL string, insecure bool, caBlock []strfmt.Base64) { + url, err := netUrl.Parse(rawURL) + if err != nil { + return + } + + tlsConfig := &tls.Config{ + InsecureSkipVerify: insecure, + } + appendSuccess := false + roots := x509.NewCertPool() + for _, caBytes := range caBlock { + appendSuccess = appendSuccess || roots.AppendCertsFromPEM(caBytes) + } + if appendSuccess { + tlsConfig.RootCAs = roots + } + + client.clientMap.Store(url.Host, &http.Client{ + Transport: &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 3 * time.Second, + KeepAlive: 30 * time.Second, + DualStack: true, + }).DialContext, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + TLSClientConfig: tlsConfig, + }, + }) +} + +// GetContentLength send a head request to get file length. +func (client *OriginClient) GetContentLength(url string, headers map[string]string) (int64, int, error) { + // send request + resp, err := client.HTTPWithHeaders("GET", url, headers, 4*time.Second) + if err != nil { + return 0, 0, err + } + resp.Body.Close() + + return resp.ContentLength, resp.StatusCode, nil +} + +// IsSupportRange checks if the source url support partial requests. +func (client *OriginClient) IsSupportRange(url string, headers map[string]string) (bool, error) { + // set headers + if headers == nil { + headers = make(map[string]string) + } + headers["Range"] = "bytes=0-0" + + // send request + resp, err := client.HTTPWithHeaders("GET", url, headers, 4*time.Second) + if err != nil { + return false, err + } + resp.Body.Close() + + if resp.StatusCode == http.StatusPartialContent { + return true, nil + } + return false, nil +} + +// IsExpired checks if a resource received or stored is the same. +func (client *OriginClient) IsExpired(url string, headers map[string]string, lastModified int64, eTag string) (bool, error) { + if lastModified <= 0 && stringutils.IsEmptyStr(eTag) { + return true, nil + } + + // set headers + if headers == nil { + headers = make(map[string]string) + } + if lastModified > 0 { + lastModifiedStr, _ := netutils.ConvertTimeIntToString(lastModified) + headers["If-Modified-Since"] = lastModifiedStr + } + if !stringutils.IsEmptyStr(eTag) { + headers["If-None-Match"] = eTag + } + + // send request + resp, err := client.HTTPWithHeaders("GET", url, headers, 4*time.Second) + if err != nil { + return false, err + } + resp.Body.Close() + + return resp.StatusCode != http.StatusNotModified, nil +} + +// Download downloads the file from the original address +func (client *OriginClient) Download(url string, headers map[string]string, checkCode int) (*http.Response, error) { + // TODO: add timeout + resp, err := client.HTTPWithHeaders("GET", url, headers, 0) + if err != nil { + return nil, err + } + + if resp.StatusCode == checkCode { + return resp, nil + } + return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode) +} + +// HTTPWithHeaders use host-matched client to request the origin resource. +func (client *OriginClient) HTTPWithHeaders(method, url string, headers map[string]string, timeout time.Duration) (*http.Response, error) { + req, err := http.NewRequest(method, url, nil) + if err != nil { + return nil, err + } + + if timeout > 0 { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + req = req.WithContext(ctx) + defer cancel() + } + + for k, v := range headers { + req.Header.Add(k, v) + } + + httpClientObject, existed := client.clientMap.Load(req.Host) + if !existed { + httpClientObject = http.DefaultClient + } + + httpClient, ok := httpClientObject.(*http.Client) + if !ok { + return nil, errors.Wrapf(errortypes.ErrInvalidValue, "http client type check error: %T", httpClientObject) + } + return httpClient.Do(req) +} diff --git a/supernode/server/0.3_bridge.go b/supernode/server/0.3_bridge.go index 0505edef9..ff14a0730 100644 --- a/supernode/server/0.3_bridge.go +++ b/supernode/server/0.3_bridge.go @@ -89,6 +89,7 @@ func (s *Server) registry(ctx context.Context, rw http.ResponseWriter, req *http TaskURL: request.TaskURL, SupernodeIP: request.SuperNodeIP, } + s.OriginClient.RegisterTLSConfig(taskCreateRequest.RawURL, request.Insecure, request.RootCAs) resp, err := s.TaskMgr.Register(ctx, taskCreateRequest) if err != nil { logrus.Errorf("failed to register task %+v: %v", taskCreateRequest, err) diff --git a/supernode/server/server.go b/supernode/server/server.go index a769bd887..94993fde6 100644 --- a/supernode/server/server.go +++ b/supernode/server/server.go @@ -14,6 +14,7 @@ import ( "github.com/dragonflyoss/Dragonfly/supernode/daemon/mgr/progress" "github.com/dragonflyoss/Dragonfly/supernode/daemon/mgr/scheduler" "github.com/dragonflyoss/Dragonfly/supernode/daemon/mgr/task" + "github.com/dragonflyoss/Dragonfly/supernode/httpclient" "github.com/dragonflyoss/Dragonfly/supernode/store" "github.com/dragonflyoss/Dragonfly/version" @@ -27,6 +28,7 @@ type Server struct { TaskMgr mgr.TaskMgr DfgetTaskMgr mgr.DfgetTaskMgr ProgressMgr mgr.ProgressMgr + OriginClient httpclient.OriginHTTPClient } // New creates a brand new server instance. @@ -40,6 +42,8 @@ func New(cfg *config.Config) (*Server, error) { return nil, err } + originClient := httpclient.NewOriginClient() + peerMgr, err := peer.NewManager() if err != nil { return nil, err @@ -60,12 +64,12 @@ func New(cfg *config.Config) (*Server, error) { return nil, err } - cdnMgr, err := cdn.NewManager(cfg, storeLocal, progressMgr) + cdnMgr, err := cdn.NewManager(cfg, storeLocal, progressMgr, originClient) if err != nil { return nil, err } - taskMgr, err := task.NewManager(cfg, peerMgr, dfgetTaskMgr, progressMgr, cdnMgr, schedulerMgr) + taskMgr, err := task.NewManager(cfg, peerMgr, dfgetTaskMgr, progressMgr, cdnMgr, schedulerMgr, originClient) if err != nil { return nil, err } @@ -76,6 +80,7 @@ func New(cfg *config.Config) (*Server, error) { TaskMgr: taskMgr, DfgetTaskMgr: dfgetTaskMgr, ProgressMgr: progressMgr, + OriginClient: originClient, }, nil }