[feat] q-dev-plugin-collect-s3

backend/go.mod

@@ -3,6 +3,7 @@ module github.com/apache/incubator-devlake
 go 1.20
 
 require (
+	github.com/aws/aws-sdk-go v1.55.6
 	github.com/cockroachdb/errors v1.11.1
 	github.com/gin-contrib/cors v1.6.0
 	github.com/gin-gonic/gin v1.9.1

backend/go.sum

@@ -54,6 +54,7 @@ github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hC
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
+github.com/aws/aws-sdk-go v1.55.6/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
 github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM=
 github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=

backend/plugins/q_dev/Q_DEV_deploy_guide.md

@@ -0,0 +1,85 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+
+# DevLake Development Environment Deployment Guide
+
+## Environment Requirements
+- Docker v19.03.10+
+- Golang v1.19+
+- GNU Make
+    - Mac (pre-installed)
+    - Windows: [Download](http://gnuwin32.sourceforge.net/packages/make.htm)
+    - Ubuntu: `sudo apt-get install build-essential libssl-dev`
+
+## How to Set Up the Development Environment
+The following guide will explain how to run DevLake's frontend (config-ui) and backend in development mode.
+
+### Clone the Repository
+Navigate to where you want to install this project and clone the repository:
+
+```bash
+git clone https://github.com/apache/incubator-devlake.git
+cd incubator-devlake
+```
+
+### Install Plugin Dependencies
+
+RefDiff plugin:
+Install Go packages
+```bash
+cd backend
+go get
+cd ..
+```
+
+### Configure Environment File
+Copy the example configuration file to a new local file:
+
+```bash
+cp env.example .env
+```
+
+Update the following variables in the `.env` file:
+
+- `DB_URL`: Replace `mysql:3306` with `127.0.0.1:3306`
+- `DISABLED_REMOTE_PLUGINS`: Set to `True`
+
+### Start MySQL and Grafana Containers
+
+Make sure the Docker daemon is running before this step.
+
+> Grafana needs to rebuild the image, then change the image in docker-compose.datasources.yml to `image: grafana:latest`
+
+```bash
+docker-compose -f docker-compose-dev.yml up -d mysql grafana
+```
+
+### Run in Development Mode
+Run devlake and config-ui in development mode in two separate terminals:
+
+```bash
+# Install poetry, follow the guide: https://python-poetry.org/docs/#installation
+# Run devlake, only using the q dev plugin here
+DEVLAKE_PLUGINS=q_dev nohup make dev &
+# Run config-ui
+make configure-dev
+```
+
+For common errors, please refer to the troubleshooting documentation.
+
+Config UI runs on localhost:4000

backend/plugins/q_dev/README.md

@@ -0,0 +1,97 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+
+# Q Developer Plugin
+
+This plugin is used to retrieve AWS Q Developer usage data from AWS S3, and process and analyze it.
+
+## Features
+
+- Retrieve CSV files from a specified prefix in AWS S3
+- Parse user usage data from CSV files
+- Aggregate data by user and calculate various metrics
+
+## Configuration
+
+Configuration items include:
+
+1. AWS Access Key ID
+2. AWS Secret Key
+3. AWS Region
+4. S3 Bucket Name
+5. Rate Limit (per hour)
+
+You can create a connection using the following curl command:
+```bash
+curl 'http://localhost:8080/plugins/q_dev/connections' \
+--header 'Content-Type: application/json' \
+--data-raw '{
+    "name": "q_dev_connection",
+    "accessKeyId": "<YOUR_ACCESS_KEY_ID>",
+    "secretAccessKey": "<YOUR_SECRET_ACCESS_KEY>",
+    "region": "<AWS_REGION>",
+    "bucket": "<YOUR_S3_BUCKET_NAME>",
+    "rateLimitPerHour": 20000
+}'
+```
+Please replace the following placeholders with actual values:
+<YOUR_ACCESS_KEY_ID>: Your AWS access key ID
+<YOUR_SECRET_ACCESS_KEY>: Your AWS secret access key
+<YOUR_S3_BUCKET_NAME>: The S3 bucket name you want to use
+<AWS_REGION>: The region where your S3 bucket is located
+
+You can get all connections using the following curl command:
+```bash
+curl Get 'http://localhost:8080/plugins/q_dev/connections'
+```
+
+## Data Flow
+
+The plugin includes the following tasks:
+
+1. `collectQDevS3Files`: Collects file metadata information from S3, without downloading file content
+2. `extractQDevS3Data`: Uses S3 file metadata to download CSV data and parse it into the database
+3. `convertQDevUserMetrics`: Converts user data into aggregated metrics, calculating averages and totals
+
+## Data Tables
+
+- `_tool_q_dev_connections`: Stores AWS S3 connection information
+- `_tool_q_dev_s3_file_meta`: Stores S3 file metadata
+- `_tool_q_dev_user_data`: Stores user data parsed from CSV files
+- `_tool_q_dev_user_metrics`: Stores aggregated user metrics
+
+## Data Collection Configuration
+Steps to collect data:
+1. On the Config UI page, select `Advanced Mode` on the left, click `Blueprints`
+2. Create a new Blueprint
+3. ![img.png](img.png) Click the gear icon on the right
+4. Paste the following JSON configuration into `JSON Configuration`:
+
+```json
+[
+  [
+    {
+      "plugin": "q_dev",
+      "subtasks": null,
+      "options": {
+        "connectionId": 5,
+        "s3Prefix": ""
+      }
+    }
+  ]
+]
+```

backend/plugins/q_dev/api/connection.go

@@ -0,0 +1,96 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+	"net/http"
+
+	"github.com/apache/incubator-devlake/core/errors"
+	"github.com/apache/incubator-devlake/core/plugin"
+	"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+	"github.com/apache/incubator-devlake/plugins/q_dev/models"
+)
+
+// 连接项目的CRUD API
+
+// PostConnections 创建新连接
+func PostConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+	// 创建连接
+	connection := &models.QDevConnection{}
+	err := api.Decode(input.Body, connection, vld)
+	if err != nil {
+		return nil, err
+	}
+	// 验证
+	// 保存到数据库
+	err = connectionHelper.Create(connection, input)
+	if err != nil {
+		return nil, err
+	}
+	return &plugin.ApiResourceOutput{Body: connection.Sanitize(), Status: http.StatusOK}, nil
+}
+
+// PatchConnection 更新现有连接
+func PatchConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+	connection := &models.QDevConnection{}
+	if err := connectionHelper.First(&connection, input.Params); err != nil {
+		return nil, err
+	}
+	if err := (&models.QDevConnection{}).MergeFromRequest(connection, input.Body); err != nil {
+		return nil, errors.Convert(err)
+	}
+	if err := connectionHelper.SaveWithCreateOrUpdate(connection); err != nil {
+		return nil, err
+	}
+	return &plugin.ApiResourceOutput{Body: connection.Sanitize(), Status: http.StatusOK}, nil
+}
+
+// DeleteConnection 删除连接
+func DeleteConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+	conn := &models.QDevConnection{}
+	output, err := connectionHelper.Delete(conn, input)
+	if err != nil {
+		return output, err
+	}
+	output.Body = conn.Sanitize()
+	return output, nil
+}
+
+// ListConnections 列出所有连接
+func ListConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+	var connections []models.QDevConnection
+	err := connectionHelper.List(&connections)
+	if err != nil {
+		return nil, err
+	}
+	// 敏感信息脱敏
+	for i := 0; i < len(connections); i++ {
+		connections[i] = connections[i].Sanitize()
+	}
+	return &plugin.ApiResourceOutput{Body: connections}, nil
+}
+
+// GetConnection 获取单个连接详情
+func GetConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+	connection := &models.QDevConnection{}
+	err := connectionHelper.First(connection, input.Params)
+	if err != nil {
+		return nil, err
+	}
+	return &plugin.ApiResourceOutput{Body: connection.Sanitize()}, err
+}

backend/plugins/q_dev/api/init.go

@@ -0,0 +1,39 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+	"github.com/apache/incubator-devlake/core/context"
+	"github.com/apache/incubator-devlake/core/plugin"
+	"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+	"github.com/go-playground/validator/v10"
+)
+
+var vld *validator.Validate
+var connectionHelper *api.ConnectionApiHelper
+var basicRes context.BasicRes
+
+func Init(br context.BasicRes, p plugin.PluginMeta) {
+	basicRes = br
+	vld = validator.New()
+	connectionHelper = api.NewConnectionHelper(
+		basicRes,
+		vld,
+		p.Name(),
+	)
+}