diff --git a/cmd/laas/docs/docs.go b/cmd/laas/docs/docs.go index dd45f0e..3eae770 100644 --- a/cmd/laas/docs/docs.go +++ b/cmd/laas/docs/docs.go @@ -425,6 +425,71 @@ const docTemplate = `{ } } }, + "/licenses/import": { + "post": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Import licenses by uploading a json file", + "consumes": [ + "multipart/form-data" + ], + "produces": [ + "application/json" + ], + "tags": [ + "Licenses" + ], + "summary": "Import licenses by uploading a json file", + "operationId": "ImportLicenses", + "parameters": [ + { + "type": "file", + "description": "licenses json file list", + "name": "file", + "in": "formData", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "allOf": [ + { + "$ref": "#/definitions/models.ImportLicensesResponse" + }, + { + "type": "object", + "properties": { + "data": { + "type": "array", + "items": { + "$ref": "#/definitions/models.LicenseImportStatus" + } + } + } + } + ] + } + }, + "400": { + "description": "input file must be present", + "schema": { + "$ref": "#/definitions/models.LicenseError" + } + }, + "500": { + "description": "Internal server error", + "schema": { + "$ref": "#/definitions/models.LicenseError" + } + } + } + } + }, "/licenses/{shortname}": { "get": { "description": "Get a single license by its shortname", @@ -1421,6 +1486,20 @@ const docTemplate = `{ } } }, + "models.ImportLicensesResponse": { + "type": "object", + "properties": { + "data": { + "description": "can be of type models.LicenseError or models.LicenseImportStatus", + "type": "array", + "items": {} + }, + "status": { + "type": "integer", + "example": 200 + } + } + }, "models.ImportObligationsResponse": { "type": "object", "properties": { @@ -1541,6 +1620,31 @@ const docTemplate = `{ } } }, + "models.LicenseId": { + "type": "object", + "properties": { + "id": { + "type": "integer", + "example": 31 + }, + "shortname": { + "type": "string", + "example": "MIT" + } + } + }, + "models.LicenseImportStatus": { + "type": "object", + "properties": { + "data": { + "$ref": "#/definitions/models.LicenseId" + }, + "status": { + "type": "integer", + "example": 200 + } + } + }, "models.LicenseMapShortnamesElement": { "type": "object", "properties": { diff --git a/cmd/laas/docs/swagger.json b/cmd/laas/docs/swagger.json index 18e7a95..ec801a8 100644 --- a/cmd/laas/docs/swagger.json +++ b/cmd/laas/docs/swagger.json @@ -418,6 +418,71 @@ } } }, + "/licenses/import": { + "post": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Import licenses by uploading a json file", + "consumes": [ + "multipart/form-data" + ], + "produces": [ + "application/json" + ], + "tags": [ + "Licenses" + ], + "summary": "Import licenses by uploading a json file", + "operationId": "ImportLicenses", + "parameters": [ + { + "type": "file", + "description": "licenses json file list", + "name": "file", + "in": "formData", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "allOf": [ + { + "$ref": "#/definitions/models.ImportLicensesResponse" + }, + { + "type": "object", + "properties": { + "data": { + "type": "array", + "items": { + "$ref": "#/definitions/models.LicenseImportStatus" + } + } + } + } + ] + } + }, + "400": { + "description": "input file must be present", + "schema": { + "$ref": "#/definitions/models.LicenseError" + } + }, + "500": { + "description": "Internal server error", + "schema": { + "$ref": "#/definitions/models.LicenseError" + } + } + } + } + }, "/licenses/{shortname}": { "get": { "description": "Get a single license by its shortname", @@ -1414,6 +1479,20 @@ } } }, + "models.ImportLicensesResponse": { + "type": "object", + "properties": { + "data": { + "description": "can be of type models.LicenseError or models.LicenseImportStatus", + "type": "array", + "items": {} + }, + "status": { + "type": "integer", + "example": 200 + } + } + }, "models.ImportObligationsResponse": { "type": "object", "properties": { @@ -1534,6 +1613,31 @@ } } }, + "models.LicenseId": { + "type": "object", + "properties": { + "id": { + "type": "integer", + "example": 31 + }, + "shortname": { + "type": "string", + "example": "MIT" + } + } + }, + "models.LicenseImportStatus": { + "type": "object", + "properties": { + "data": { + "$ref": "#/definitions/models.LicenseId" + }, + "status": { + "type": "integer", + "example": 200 + } + } + }, "models.LicenseMapShortnamesElement": { "type": "object", "properties": { diff --git a/cmd/laas/docs/swagger.yaml b/cmd/laas/docs/swagger.yaml index d9a7f3a..e20bc75 100644 --- a/cmd/laas/docs/swagger.yaml +++ b/cmd/laas/docs/swagger.yaml @@ -62,6 +62,16 @@ definitions: example: 200 type: integer type: object + models.ImportLicensesResponse: + properties: + data: + description: can be of type models.LicenseError or models.LicenseImportStatus + items: {} + type: array + status: + example: 200 + type: integer + type: object models.ImportObligationsResponse: properties: data: @@ -147,6 +157,23 @@ definitions: example: "2023-12-01T10:00:51+05:30" type: string type: object + models.LicenseId: + properties: + id: + example: 31 + type: integer + shortname: + example: MIT + type: string + type: object + models.LicenseImportStatus: + properties: + data: + $ref: '#/definitions/models.LicenseId' + status: + example: 200 + type: integer + type: object models.LicenseMapShortnamesElement: properties: add: @@ -874,6 +901,45 @@ paths: summary: Update a license tags: - Licenses + /licenses/import: + post: + consumes: + - multipart/form-data + description: Import licenses by uploading a json file + operationId: ImportLicenses + parameters: + - description: licenses json file list + in: formData + name: file + required: true + type: file + produces: + - application/json + responses: + "200": + description: OK + schema: + allOf: + - $ref: '#/definitions/models.ImportLicensesResponse' + - properties: + data: + items: + $ref: '#/definitions/models.LicenseImportStatus' + type: array + type: object + "400": + description: input file must be present + schema: + $ref: '#/definitions/models.LicenseError' + "500": + description: Internal server error + schema: + $ref: '#/definitions/models.LicenseError' + security: + - ApiKeyAuth: [] + summary: Import licenses by uploading a json file + tags: + - Licenses /login: post: consumes: diff --git a/pkg/api/api.go b/pkg/api/api.go index 5805249..8ab3393 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -104,6 +104,7 @@ func Router() *gin.Engine { { licenses.POST("", CreateLicense) licenses.PATCH(":shortname", UpdateLicense) + licenses.POST("import", ImportLicenses) } users := authorizedv1.Group("/users") { diff --git a/pkg/api/licenses.go b/pkg/api/licenses.go index 0df0412..7894fd7 100644 --- a/pkg/api/licenses.go +++ b/pkg/api/licenses.go @@ -11,6 +11,7 @@ import ( "errors" "fmt" "net/http" + "path/filepath" "strconv" "time" @@ -884,3 +885,308 @@ func SearchInLicense(c *gin.Context) { } c.JSON(http.StatusOK, res) } + +// ImportLicenses creates new licenses records via a json file. +// +// @Summary Import licenses by uploading a json file +// @Description Import licenses by uploading a json file +// @Id ImportLicenses +// @Tags Licenses +// @Accept multipart/form-data +// @Produce json +// @Param file formData file true "licenses json file list" +// @Success 200 {object} models.ImportLicensesResponse{data=[]models.LicenseImportStatus} +// @Failure 400 {object} models.LicenseError "input file must be present" +// @Failure 500 {object} models.LicenseError "Internal server error" +// @Security ApiKeyAuth +// @Router /licenses/import [post] +func ImportLicenses(c *gin.Context) { + username := c.GetString("username") + file, header, err := c.Request.FormFile("file") + if err != nil { + er := models.LicenseError{ + Status: http.StatusBadRequest, + Message: "input file must be present", + Error: err.Error(), + Path: c.Request.URL.Path, + Timestamp: time.Now().Format(time.RFC3339), + } + c.JSON(http.StatusBadRequest, er) + return + } + defer file.Close() + + if filepath.Ext(header.Filename) != ".json" { + er := models.LicenseError{ + Status: http.StatusBadRequest, + Message: "only files with format *.json are allowed", + Error: "only files with format *.json are allowed", + Path: c.Request.URL.Path, + Timestamp: time.Now().Format(time.RFC3339), + } + c.JSON(http.StatusBadRequest, er) + return + } + + var licenses []models.LicenseImport + decoder := json.NewDecoder(file) + if err := decoder.Decode(&licenses); err != nil { + er := models.LicenseError{ + Status: http.StatusInternalServerError, + Message: "invalid json", + Error: err.Error(), + Path: c.Request.URL.Path, + Timestamp: time.Now().Format(time.RFC3339), + } + c.JSON(http.StatusInternalServerError, er) + return + } + + res := models.ImportLicensesResponse{ + Status: http.StatusOK, + } + + for _, license := range licenses { + _ = db.DB.Transaction(func(tx *gorm.DB) error { + newLicenseMap := make(map[string]interface{}) + if license.Shortname.IsDefinedAndNotNull { + newLicenseMap["rf_shortname"] = license.Shortname.Value + } else { + res.Data = append(res.Data, models.LicenseError{ + Status: http.StatusInternalServerError, + Message: "field rf_shortname cannot be null", + Error: "", + Path: c.Request.URL.Path, + Timestamp: time.Now().Format(time.RFC3339), + }) + return errors.New("field rf_shortname cannot be null") + } + if license.Fullname.IsDefinedAndNotNull { + newLicenseMap["rf_fullname"] = license.Fullname.Value + } else { + res.Data = append(res.Data, models.LicenseError{ + Status: http.StatusInternalServerError, + Message: "field rf_fullname cannot be null", + Error: license.Shortname.Value, + Path: c.Request.URL.Path, + Timestamp: time.Now().Format(time.RFC3339), + }) + return errors.New("field rf_fullname cannot be null") + } + if license.Text.IsDefinedAndNotNull { + newLicenseMap["rf_text"] = license.Text.Value + } else { + res.Data = append(res.Data, models.LicenseError{ + Status: http.StatusInternalServerError, + Message: "field rf_text cannot be null", + Error: license.Shortname.Value, + Path: c.Request.URL.Path, + Timestamp: time.Now().Format(time.RFC3339), + }) + return errors.New("field rf_text cannot be null") + } + if license.Url.IsDefinedAndNotNull { + newLicenseMap["rf_url"] = license.Url.Value + } else { + res.Data = append(res.Data, models.LicenseError{ + Status: http.StatusInternalServerError, + Message: "field rf_url cannot be null", + Error: license.Shortname.Value, + Path: c.Request.URL.Path, + Timestamp: time.Now().Format(time.RFC3339), + }) + return errors.New("field rf_url cannot be null") + } + if license.Active.IsDefinedAndNotNull { + newLicenseMap["rf_active"] = license.Active.Value + } else { + res.Data = append(res.Data, models.LicenseError{ + Status: http.StatusInternalServerError, + Message: "field rf_active cannot be null", + Error: license.Shortname.Value, + Path: c.Request.URL.Path, + Timestamp: time.Now().Format(time.RFC3339), + }) + return errors.New("field rf_active cannot be null") + } + if license.Source.IsDefinedAndNotNull { + newLicenseMap["rf_source"] = license.Source.Value + } else { + res.Data = append(res.Data, models.LicenseError{ + Status: http.StatusInternalServerError, + Message: "field rf_source cannot be null", + Error: license.Shortname.Value, + Path: c.Request.URL.Path, + Timestamp: time.Now().Format(time.RFC3339), + }) + return errors.New("field rf_source cannot be null") + } + if license.SpdxId.IsDefinedAndNotNull { + newLicenseMap["rf_spdx_id"] = license.SpdxId.Value + } else { + res.Data = append(res.Data, models.LicenseError{ + Status: http.StatusInternalServerError, + Message: "field rf_spdx_id cannot be null", + Error: license.Shortname.Value, + Path: c.Request.URL.Path, + Timestamp: time.Now().Format(time.RFC3339), + }) + return errors.New("field rf_spdx_id cannot be null") + } + if license.Risk.IsDefinedAndNotNull { + newLicenseMap["rf_risk"] = license.Risk.Value + } else { + res.Data = append(res.Data, models.LicenseError{ + Status: http.StatusInternalServerError, + Message: "field rf_risk cannot be null", + Error: license.Shortname.Value, + Path: c.Request.URL.Path, + Timestamp: time.Now().Format(time.RFC3339), + }) + return errors.New("field rf_risk cannot be null") + } + if license.Copyleft.IsDefinedAndNotNull { + newLicenseMap["rf_copyleft"] = license.Copyleft.Value + } + if license.FSFfree.IsDefinedAndNotNull { + newLicenseMap["rf_FSFfree"] = license.FSFfree.Value + } + if license.OSIapproved.IsDefinedAndNotNull { + newLicenseMap["rf_OSIapproved"] = license.OSIapproved.Value + } + if license.GPLv2compatible.IsDefinedAndNotNull { + newLicenseMap["rf_GPLv2compatible"] = license.GPLv2compatible.Value + } + if license.GPLv3compatible.IsDefinedAndNotNull { + newLicenseMap["rf_GPLv3compatible"] = license.GPLv3compatible.Value + } + if license.Notes.IsDefinedAndNotNull { + newLicenseMap["rf_notes"] = license.Notes.Value + } + if license.Fedora.IsDefinedAndNotNull { + newLicenseMap["rf_Fedora"] = license.Fedora.Value + } + if license.DetectorType.IsDefinedAndNotNull { + newLicenseMap["rf_detector_type"] = license.DetectorType.Value + } + if license.Flag.IsDefinedAndNotNull { + newLicenseMap["rf_flag"] = license.Flag.Value + } + if license.Marydone.IsDefinedAndNotNull { + newLicenseMap["marydone"] = license.Marydone.Value + } + if license.ExternalRef.IsDefinedAndNotNull { + newLicenseMap["external_ref"] = license.ExternalRef.Value + } + + errMessage, importStatus, newLicense, oldLicense := InsertOrUpdateLicenseOnImport(tx, newLicenseMap) + + if importStatus == models.IMPORT_FAILED { + res.Data = append(res.Data, models.LicenseError{ + Status: http.StatusInternalServerError, + Message: errMessage, + Error: newLicense.Shortname, + Path: c.Request.URL.Path, + Timestamp: time.Now().Format(time.RFC3339), + }) + return errors.New(errMessage) + } else if importStatus == models.IMPORT_LICENSE_CREATED { + res.Data = append(res.Data, models.LicenseImportStatus{ + Data: models.LicenseId{Id: oldLicense.Id, Shortname: oldLicense.Shortname}, + Status: http.StatusCreated, + }) + } else if importStatus == models.IMPORT_LICENSE_UPDATED { + if err := addChangelogsForLicenseUpdate(tx, username, newLicense, oldLicense); err != nil { + res.Data = append(res.Data, models.LicenseError{ + Status: http.StatusInternalServerError, + Message: "Failed to update license", + Error: newLicense.Shortname, + Path: c.Request.URL.Path, + Timestamp: time.Now().Format(time.RFC3339), + }) + return err + } + res.Data = append(res.Data, models.LicenseImportStatus{ + Data: models.LicenseId{Id: newLicense.Id, Shortname: newLicense.Shortname}, + Status: http.StatusOK, + }) + } else if importStatus == models.IMPORT_LICENSE_UPDATED_EXCEPT_TEXT { + if err := addChangelogsForLicenseUpdate(tx, username, newLicense, oldLicense); err != nil { + res.Data = append(res.Data, models.LicenseError{ + Status: http.StatusInternalServerError, + Message: "Failed to update license", + Error: newLicense.Shortname, + Path: c.Request.URL.Path, + Timestamp: time.Now().Format(time.RFC3339), + }) + return err + } + + res.Data = append(res.Data, models.LicenseError{ + Status: http.StatusConflict, + Message: "All fields except Text were updated. Text was updated manually and cannot be overwritten in an import.", + Error: newLicense.Shortname, + Path: c.Request.URL.Path, + Timestamp: time.Now().Format(time.RFC3339), + }) + // error is not returned here as it will rollback the transaction + } + + return nil + }) + } + + c.JSON(http.StatusOK, res) +} + +// Updates/Creates a license from a map containing license values +func InsertOrUpdateLicenseOnImport(tx *gorm.DB, newLicenseMap map[string]interface{}) (string, models.LicenseImportStatusCode, *models.LicenseDB, *models.LicenseDB) { + var message string + var importStatus models.LicenseImportStatusCode + var newLicense, oldLicense models.LicenseDB + + result := tx. + Where(&models.LicenseDB{Shortname: newLicenseMap["rf_shortname"].(string)}). + FirstOrCreate(&oldLicense) + if result.Error != nil { + message = fmt.Sprintf("failed to create license: %s", result.Error.Error()) + importStatus = models.IMPORT_FAILED + return message, importStatus, &newLicense, &oldLicense + } else if result.RowsAffected == 0 { + // case when license exists in database and is updated + + // Overwrite values of existing keys, add new key value pairs and remove keys with null values. + if err := tx.Model(&models.LicenseDB{}).Where(&models.LicenseDB{Shortname: newLicenseMap["rf_shortname"].(string)}).UpdateColumn("external_ref", gorm.Expr("jsonb_strip_nulls(external_ref || ?)", newLicenseMap["external_ref"])).Error; err != nil { + message = fmt.Sprintf("failed to update license: %s", err.Error()) + importStatus = models.IMPORT_FAILED + return message, importStatus, &newLicense, &oldLicense + } + + // Update all other fields except external_ref + query := tx.Model(&newLicense).Where(&models.LicenseDB{Shortname: newLicenseMap["rf_shortname"].(string)}).Omit("external_ref") + + // Do not update text in import if it was modified manually + if oldLicense.Flag == 2 { + query = query.Omit("rf_text") + } + + if err := query.Clauses(clause.Returning{}).Updates(newLicenseMap).Error; err != nil { + message = fmt.Sprintf("failed to update license: %s", err.Error()) + importStatus = models.IMPORT_FAILED + return message, importStatus, &newLicense, &oldLicense + } + + if oldLicense.Flag == 2 { + message = "all fields except rf_text were updated. rf_text was updated manually and cannot be overwritten in an import." + importStatus = models.IMPORT_LICENSE_UPDATED_EXCEPT_TEXT + // error is not returned here as it will rollback the transaction + } else { + importStatus = models.IMPORT_LICENSE_UPDATED + } + } else { + // case when license doesn't exist in database and is inserted + importStatus = models.IMPORT_LICENSE_CREATED + } + + return message, importStatus, &newLicense, &oldLicense +} diff --git a/pkg/models/optional_data_types.go b/pkg/models/optional_data_types.go index b6bb98e..46e9eab 100644 --- a/pkg/models/optional_data_types.go +++ b/pkg/models/optional_data_types.go @@ -36,3 +36,25 @@ func (v *OptionalData[T]) UnmarshalJSON(data []byte) error { } return nil } + +type NullableAndOptionalData[T any] struct { + // This is set to true if corresponding key is present in json object + IsDefinedAndNotNull bool + rawJson json.RawMessage + Value T +} + +func (v *NullableAndOptionalData[T]) UnmarshalJSON(data []byte) error { + v.rawJson = append((v.rawJson)[0:0], data...) + if len(v.rawJson) != 0 { + var x *T + if err := json.Unmarshal(data, &x); err != nil { + return err + } + if x != nil { + v.Value = *x + v.IsDefinedAndNotNull = true + } + } + return nil +} diff --git a/pkg/models/types.go b/pkg/models/types.go index c10eeac..1069e2a 100644 --- a/pkg/models/types.go +++ b/pkg/models/types.go @@ -117,6 +117,59 @@ type UpdateExternalRefsJSONPayload struct { ExternalRef map[string]interface{} `json:"external_ref"` } +// LicenseImport represents an license record in the import json file. +type LicenseImport struct { + Shortname NullableAndOptionalData[string] `json:"rf_shortname" validate:"required" example:"MIT"` + Fullname NullableAndOptionalData[string] `json:"rf_fullname" validate:"required" example:"MIT License"` + Text NullableAndOptionalData[string] `json:"rf_text" validate:"required" example:"MIT License Text here"` + Url NullableAndOptionalData[string] `json:"rf_url" validate:"required" example:"https://opensource.org/licenses/MIT"` + Copyleft NullableAndOptionalData[bool] `json:"rf_copyleft"` + FSFfree NullableAndOptionalData[bool] `json:"rf_FSFfree"` + OSIapproved NullableAndOptionalData[bool] `json:"rf_OSIapproved"` + GPLv2compatible NullableAndOptionalData[bool] `json:"rf_GPLv2compatible"` + GPLv3compatible NullableAndOptionalData[bool] `json:"rf_GPLv3compatible"` + Notes NullableAndOptionalData[string] `json:"rf_notes" example:"This license has been superseded."` + Fedora NullableAndOptionalData[string] `json:"rf_Fedora"` + TextUpdatable NullableAndOptionalData[bool] `json:"rf_text_updatable" validate:"required"` + DetectorType NullableAndOptionalData[int64] `json:"rf_detector_type" example:"1"` + Active NullableAndOptionalData[bool] `json:"rf_active" validate:"required"` + Source NullableAndOptionalData[string] `json:"rf_source" validate:"required"` + SpdxId NullableAndOptionalData[string] `json:"rf_spdx_id" validate:"required" example:"MIT"` + Risk NullableAndOptionalData[int64] `json:"rf_risk" validate:"required"` + Flag NullableAndOptionalData[int64] `json:"rf_flag"` + Marydone NullableAndOptionalData[bool] `json:"marydone"` + ExternalRef NullableAndOptionalData[datatypes.JSONType[LicenseDBSchemaExtension]] `json:"external_ref"` +} + +// LicenseImportStatusCode is internally used for checking status of a license import +type LicenseImportStatusCode int + +// Status codes covering various scenarios that can occur on a license import +const ( + IMPORT_FAILED LicenseImportStatusCode = iota + 1 + IMPORT_LICENSE_CREATED + IMPORT_LICENSE_UPDATED + IMPORT_LICENSE_UPDATED_EXCEPT_TEXT +) + +// LicenseId is the id of successfully imported license +type LicenseId struct { + Id int64 `json:"id" example:"31"` + Shortname string `json:"shortname" example:"MIT"` +} + +// LicenseImportStatus is the status of license records successfully inserted in the database during import +type LicenseImportStatus struct { + Status int `json:"status" example:"200"` + Data LicenseId `json:"data"` +} + +// ImportObligationsResponse is the response structure for import obligation response +type ImportLicensesResponse struct { + Status int `json:"status" example:"200"` + Data []interface{} `json:"data"` // can be of type models.LicenseError or models.LicenseImportStatus +} + // The PaginationMeta struct represents additional metadata associated with a // license retrieval operation. // It contains information that provides context and supplementary details