From 44c0c37047ee15cd2856056270a1fb708473c4d6 Mon Sep 17 00:00:00 2001 From: Artur Sawicki Date: Mon, 6 May 2024 14:09:30 +0200 Subject: [PATCH] chore: Add scripts to close issues from Pre Snowflake bucket (#2762) Add scripts to close issues from Pre Snowflake bucket (for future reference; they were already run) --- ...twork_policy_attachment_acceptance_test.go | 1 - pkg/scripts/issues/README.md | 18 ++ .../20240430 - issues_to_close.csv | 188 +++++++++++++++++ pkg/scripts/issues/close-with-comment/main.go | 188 +++++++++++++++++ .../20240430 - issues_edited.csv | 17 ++ .../20240430 - issues_to_close.csv | 176 ++++++++++++++++ .../20240430 - notes.MD | 18 ++ .../filter-closeable-old-issues/main.go | 157 ++++++++++++++ .../presnowflake_bucket.csv | 193 ++++++++++++++++++ 9 files changed, 955 insertions(+), 1 deletion(-) create mode 100644 pkg/scripts/issues/close-with-comment/20240430 - issues_to_close.csv create mode 100644 pkg/scripts/issues/close-with-comment/main.go create mode 100644 pkg/scripts/issues/filter-closeable-old-issues/20240430 - issues_edited.csv create mode 100644 pkg/scripts/issues/filter-closeable-old-issues/20240430 - issues_to_close.csv create mode 100644 pkg/scripts/issues/filter-closeable-old-issues/20240430 - notes.MD create mode 100644 pkg/scripts/issues/filter-closeable-old-issues/main.go create mode 100644 pkg/scripts/issues/filter-closeable-old-issues/presnowflake_bucket.csv diff --git a/pkg/resources/network_policy_attachment_acceptance_test.go b/pkg/resources/network_policy_attachment_acceptance_test.go index 326e41e878..e64514549b 100644 --- a/pkg/resources/network_policy_attachment_acceptance_test.go +++ b/pkg/resources/network_policy_attachment_acceptance_test.go @@ -54,7 +54,6 @@ func TestAcc_NetworkPolicyAttachmentUser(t *testing.T) { }, }, }) - } func TestAcc_NetworkPolicyAttachmentAccount(t *testing.T) { diff --git a/pkg/scripts/issues/README.md b/pkg/scripts/issues/README.md index a410f991f6..a3da4a028d 100644 --- a/pkg/scripts/issues/README.md +++ b/pkg/scripts/issues/README.md @@ -1,3 +1,4 @@ +# Generating the list of open issues 1. To use the script, generate access token here: https://github.com/settings/tokens?type=beta. 2. To get all open issues invoke the [first script](./gh/main.go) setting `SF_TF_SCRIPT_GH_ACCESS_TOKEN`: ```shell @@ -9,3 +10,20 @@ cd file && go run . ``` 5. File `issues.csv` should be generated in the `file` directory. This is the CSV which summarizes all the issues we have. + +# Closing old issues (regarding https://github.com/Snowflake-Labs/terraform-provider-snowflake/discussions/2755) +1. To use the script, generate access token here: https://github.com/settings/tokens?type=beta. +2. First get all open issues by invoking: +```shell + cd gh && SF_TF_SCRIPT_GH_ACCESS_TOKEN= go run . +``` +3. File `issues.json` should be generated in the `gh` directory. This is the input file for the second script. The next script is based also on `presnowflake_bucket.csv` that was created based on the GH issues filtering. +4. To filter only closeable issues invoke [this script](./filter-closeable-old-issues/main.go): +```shell + cd filter-closeable-old-issues && go run . +``` +5. Script will output files `issues_to_close.csv` and `issues_edited.csv`. There are two files documenting closing action on 30.04.2024 (`20240430 - issues_edited.csv` and `20240430 - issues_to_close.csv`). In `20240430 - notes.MD` there are notes regarding the questionable issues and the decisions taken. +6. To close the issues with the appropriate comment provide `issues_to_close.csv` in `close-with-comment` dir. Example `20240430 - issues_to_close.csv` is given. The run: +```shell + cd close-with-comment && SF_TF_SCRIPT_GH_ACCESS_TOKEN= go run . +``` \ No newline at end of file diff --git a/pkg/scripts/issues/close-with-comment/20240430 - issues_to_close.csv b/pkg/scripts/issues/close-with-comment/20240430 - issues_to_close.csv new file mode 100644 index 0000000000..313ca59b69 --- /dev/null +++ b/pkg/scripts/issues/close-with-comment/20240430 - issues_to_close.csv @@ -0,0 +1,188 @@ +1267 +1249 +1221 +1204 +1161 +1075 +1068 +1067 +1066 +1061 +1042 +1029 +1005 +1004 +998 +994 +989 +984 +981 +980 +977 +976 +973 +970 +967 +966 +965 +955 +953 +952 +951 +950 +946 +938 +897 +896 +892 +891 +882 +867 +865 +859 +847 +845 +844 +836 +835 +825 +822 +821 +818 +810 +806 +802 +800 +794 +793 +791 +788 +787 +781 +778 +776 +774 +772 +770 +760 +757 +756 +754 +753 +749 +744 +741 +738 +737 +735 +723 +722 +719 +718 +708 +690 +689 +688 +684 +683 +679 +678 +677 +676 +675 +674 +670 +665 +663 +660 +651 +648 +644 +642 +641 +640 +636 +623 +621 +617 +616 +612 +611 +608 +606 +604 +601 +596 +594 +592 +586 +583 +569 +568 +566 +564 +562 +560 +555 +551 +531 +529 +522 +521 +514 +512 +509 +502 +500 +498 +497 +494 +490 +474 +470 +466 +460 +455 +448 +447 +445 +430 +423 +421 +419 +411 +410 +409 +408 +402 +384 +317 +308 +306 +303 +288 +285 +284 +282 +275 +273 +254 +244 +211 +200 +186 +142 +137 +37 +1000 +993 +963 +712 +587 +300 +295 +225 +222 +218 +189 +152 diff --git a/pkg/scripts/issues/close-with-comment/main.go b/pkg/scripts/issues/close-with-comment/main.go new file mode 100644 index 0000000000..14c1f9ae4b --- /dev/null +++ b/pkg/scripts/issues/close-with-comment/main.go @@ -0,0 +1,188 @@ +package main + +import ( + "bytes" + "encoding/csv" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + "strconv" + "time" +) + +func main() { + accessToken := getAccessToken() + issuesToClose := loadIssuesToClose() + s, e := commentAndCloseIssues(issuesToClose, accessToken) + fmt.Printf("\nSuccessfully updated issues:\n") + for n, i := range s { + fmt.Printf("%d: #%d\n", n, i) + } + fmt.Printf("\nUnsuccessful issues:\n") + for n, i := range e { + fmt.Printf("%d: #%d\n", n, i) + } +} + +func getAccessToken() string { + token := os.Getenv("SF_TF_SCRIPT_GH_ACCESS_TOKEN") + if token == "" { + panic(errors.New("GitHub access token missing")) + } + return token +} + +func loadIssuesToClose() []Issue { + f, err := os.Open("issues_to_close.csv") + if err != nil { + panic(err) + } + defer f.Close() + csvReader := csv.NewReader(f) + records, err := csvReader.ReadAll() + if err != nil { + panic(err) + } + issues := make([]Issue, 0) + for _, record := range records { + number, err := strconv.Atoi(record[0]) + if err != nil { + panic(err) + } + issues = append(issues, Issue{ + Number: number, + }) + } + return issues +} + +func commentAndCloseIssues(issues []Issue, accessToken string) ([]int, []int) { + failedIssues := make([]int, 0) + successfulIssues := make([]int, 0) + client := &http.Client{} + for idx, issue := range issues { + fmt.Printf("Processing issue (%d): %d\n", idx, issue.Number) + + // preparing requests + commentRequest, err := prepareCommentRequest(accessToken, issue.Number) + if err != nil { + fmt.Printf("preparing comment request for issue #%d resulted in error %v\n", issue.Number, err) + failedIssues = append(failedIssues, issue.Number) + continue + } + closeRequest, err := prepareCloseRequest(accessToken, issue.Number) + if err != nil { + fmt.Printf("preparing close request for issue #%d resulted in error %v\n", issue.Number, err) + failedIssues = append(failedIssues, issue.Number) + continue + } + + // adding a comment + commentResponseBody, status, err := invokeReq(client, commentRequest) + if err != nil { + fmt.Printf("adding comment to issue #%d resulted in error %v\n", issue.Number, err) + failedIssues = append(failedIssues, issue.Number) + continue + } + if status != 201 { + fmt.Printf("adding comment issue #%d has status %d, expecting 201; body: %s\n", issue.Number, status, commentResponseBody) + failedIssues = append(failedIssues, issue.Number) + continue + } + + // closing the issue + closeResponseBody, status, err := invokeReq(client, closeRequest) + if err != nil { + fmt.Printf("closing issue #%d resulted in error %v\n", issue.Number, err) + failedIssues = append(failedIssues, issue.Number) + continue + } + if status != 200 { + fmt.Printf("closing issue #%d has status %d, expecting 200; body: %s\n", issue.Number, status, closeResponseBody) + failedIssues = append(failedIssues, issue.Number) + continue + } + fmt.Printf("issue #%d was successfully updated\n", issue.Number) + successfulIssues = append(successfulIssues, issue.Number) + + fmt.Printf("Sleeping for a moment...\n") + time.Sleep(5 * time.Second) + } + return successfulIssues, failedIssues +} + +// https://docs.github.com/en/rest/issues/comments?apiVersion=2022-11-28#create-an-issue-comment +// expecting 201 +func prepareCommentRequest(token string, issueNumber int) (*http.Request, error) { + addCommentBody := AddComment{ + Body: "We are closing this issue as part of a cleanup described in [announcement](https://github.com/Snowflake-Labs/terraform-provider-snowflake/discussions/2755). If you believe that the issue is still valid in v0.89.0, please open a new ticket.", + } + marshalledAddCommentBody, err := json.Marshal(addCommentBody) + if err != nil { + return nil, fmt.Errorf("impossible to marshall add comment body for issue %d, err: %w", issueNumber, err) + } + url := fmt.Sprintf("https://api.github.com/repos/Snowflake-Labs/terraform-provider-snowflake/issues/%d/comments", issueNumber) + req, err := http.NewRequest("POST", url, bytes.NewReader(marshalledAddCommentBody)) + if err != nil { + return nil, fmt.Errorf("error creating comment request for issue %d, err: %w", issueNumber, err) + } + req.Header.Add("Accept", "application/vnd.github+json") + req.Header.Add("X-GitHub-Api-Version", "2022-11-28") + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token)) + fmt.Printf("Prepared URL: %s\n", req.URL.String()) + fmt.Printf("Prepared Body: %s\n", marshalledAddCommentBody) + return req, nil +} + +// https://docs.github.com/en/rest/issues/issues?apiVersion=2022-11-28#update-an-issue +// expecting 200 +func prepareCloseRequest(token string, issueNumber int) (*http.Request, error) { + closeIssueBody := UpdateIssue{ + State: "closed", + StateReason: "not_planned", + } + marshalledCloseIssueBody, err := json.Marshal(closeIssueBody) + if err != nil { + return nil, fmt.Errorf("impossible to marshall update body for issue %d, err: %w", issueNumber, err) + } + url := fmt.Sprintf("https://api.github.com/repos/Snowflake-Labs/terraform-provider-snowflake/issues/%d", issueNumber) + req, err := http.NewRequest("PATCH", url, bytes.NewReader(marshalledCloseIssueBody)) + if err != nil { + return nil, fmt.Errorf("error creating close request for issue %d, err: %w", issueNumber, err) + } + req.Header.Add("Accept", "application/vnd.github+json") + req.Header.Add("X-GitHub-Api-Version", "2022-11-28") + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token)) + fmt.Printf("Prepared URL: %s\n", req.URL.String()) + fmt.Printf("Prepared Body: %s\n", marshalledCloseIssueBody) + return req, nil +} + +func invokeReq(client *http.Client, req *http.Request) ([]byte, int, error) { + resp, err := client.Do(req) + if err != nil { + return nil, 0, fmt.Errorf("error invoking request %s, err: %w", req.URL, err) + } + defer resp.Body.Close() + bodyBytes, err := io.ReadAll(resp.Body) + if err != nil { + return nil, resp.StatusCode, fmt.Errorf("error reading response body of request %s, err: %w", req.URL, err) + } + return bodyBytes, resp.StatusCode, nil +} + +type Issue struct { + Number int +} + +type UpdateIssue struct { + State string `json:"state"` + StateReason string `json:"state_reason"` +} + +type AddComment struct { + Body string `json:"body"` +} diff --git a/pkg/scripts/issues/filter-closeable-old-issues/20240430 - issues_edited.csv b/pkg/scripts/issues/filter-closeable-old-issues/20240430 - issues_edited.csv new file mode 100644 index 0000000000..d730c354ec --- /dev/null +++ b/pkg/scripts/issues/filter-closeable-old-issues/20240430 - issues_edited.csv @@ -0,0 +1,17 @@ +1000 +993 +963 +712 +630 +587 +533 +506 +420 +300 +295 +265 +225 +222 +218 +189 +152 diff --git a/pkg/scripts/issues/filter-closeable-old-issues/20240430 - issues_to_close.csv b/pkg/scripts/issues/filter-closeable-old-issues/20240430 - issues_to_close.csv new file mode 100644 index 0000000000..c03d93bfd2 --- /dev/null +++ b/pkg/scripts/issues/filter-closeable-old-issues/20240430 - issues_to_close.csv @@ -0,0 +1,176 @@ +1267 +1249 +1221 +1204 +1161 +1075 +1068 +1067 +1066 +1061 +1042 +1029 +1005 +1004 +998 +994 +989 +984 +981 +980 +977 +976 +973 +970 +967 +966 +965 +955 +953 +952 +951 +950 +946 +938 +897 +896 +892 +891 +882 +867 +865 +859 +847 +845 +844 +836 +835 +825 +822 +821 +818 +810 +806 +802 +800 +794 +793 +791 +788 +787 +781 +778 +776 +774 +772 +770 +760 +757 +756 +754 +753 +749 +744 +741 +738 +737 +735 +723 +722 +719 +718 +708 +690 +689 +688 +684 +683 +679 +678 +677 +676 +675 +674 +670 +665 +663 +660 +651 +648 +644 +642 +641 +640 +636 +623 +621 +617 +616 +612 +611 +608 +606 +604 +601 +596 +594 +592 +586 +583 +569 +568 +566 +564 +562 +560 +555 +551 +531 +529 +522 +521 +514 +512 +509 +502 +500 +498 +497 +494 +490 +474 +470 +466 +460 +455 +448 +447 +445 +430 +423 +421 +419 +411 +410 +409 +408 +402 +384 +317 +308 +306 +303 +288 +285 +284 +282 +275 +273 +254 +244 +211 +200 +186 +142 +137 +37 diff --git a/pkg/scripts/issues/filter-closeable-old-issues/20240430 - notes.MD b/pkg/scripts/issues/filter-closeable-old-issues/20240430 - notes.MD new file mode 100644 index 0000000000..c14f714764 --- /dev/null +++ b/pkg/scripts/issues/filter-closeable-old-issues/20240430 - notes.MD @@ -0,0 +1,18 @@ +issues edited in the last 180 days: +[#1000](https://github.com/Snowflake-Labs/terraform-provider-snowflake/issues/1000) - closing; no response +[#993](https://github.com/Snowflake-Labs/terraform-provider-snowflake/issues/993) - closing; procedures updated and new import works +[#963](https://github.com/Snowflake-Labs/terraform-provider-snowflake/issues/963) - closing; label added in the meantime, that's why date updated changed +[#712](https://github.com/Snowflake-Labs/terraform-provider-snowflake/issues/712) - closing; label added in the meantime, that's why date updated changed +[#630](https://github.com/Snowflake-Labs/terraform-provider-snowflake/issues/630) - leaving open, adding to shares +[#587](https://github.com/Snowflake-Labs/terraform-provider-snowflake/issues/587) - closing; label added in the meantime, that's why date updated changed +[#533](https://github.com/Snowflake-Labs/terraform-provider-snowflake/issues/533) - leaving open, adding to pipes +[#506](https://github.com/Snowflake-Labs/terraform-provider-snowflake/issues/506) - leaving open, adding to schemas and databases +[#420](https://github.com/Snowflake-Labs/terraform-provider-snowflake/issues/420) - leaving open, adding to rename epic +[#300](https://github.com/Snowflake-Labs/terraform-provider-snowflake/issues/300) - closing; label added in the meantime, that's why date updated changed +[#295](https://github.com/Snowflake-Labs/terraform-provider-snowflake/issues/295) - closing; label added in the meantime, that's why date updated changed +[#265](https://github.com/Snowflake-Labs/terraform-provider-snowflake/issues/265) - leaving open, adding to stage +[#225](https://github.com/Snowflake-Labs/terraform-provider-snowflake/issues/225) - closing; label added in the meantime, that's why date updated changed +[#222](https://github.com/Snowflake-Labs/terraform-provider-snowflake/issues/222) - closing; label added in the meantime, that's why date updated changed +[#218](https://github.com/Snowflake-Labs/terraform-provider-snowflake/issues/218) - closing; label added in the meantime, that's why date updated changed +[#189](https://github.com/Snowflake-Labs/terraform-provider-snowflake/issues/189) - closing; label added in the meantime, that's why date updated changed +[#152](https://github.com/Snowflake-Labs/terraform-provider-snowflake/issues/152) - closing; label added in the meantime, that's why date updated changed \ No newline at end of file diff --git a/pkg/scripts/issues/filter-closeable-old-issues/main.go b/pkg/scripts/issues/filter-closeable-old-issues/main.go new file mode 100644 index 0000000000..799f8af2b1 --- /dev/null +++ b/pkg/scripts/issues/filter-closeable-old-issues/main.go @@ -0,0 +1,157 @@ +package main + +import ( + "encoding/csv" + "encoding/json" + "fmt" + "os" + "slices" + "strconv" + "time" + + i "github.com/Snowflake-Labs/terraform-provider-snowflake/pkg/scripts/issues" +) + +func main() { + preSnowflakeBucket := loadPreSnowflakeBucket() + issues := loadIssues() + issuesToClose, issuesEdited := filterIssues(issues, preSnowflakeBucket) + fmt.Printf("\nPre Snowflake bucket:\n") + for n, pre := range preSnowflakeBucket { + fmt.Printf("%d: #%d\n", n, pre.Number) + } + fmt.Printf("\nISSUES TO CLOSE:\n") + for n, c := range issuesToClose { + fmt.Printf("%d: #%d\n", n, c.ID) + } + fmt.Printf("\nISSUES EDITED:\n") + for n, e := range issuesEdited { + fmt.Printf("%d: #%d\n", n, e.ID) + } + saveIssuesToClose(issuesToClose) + saveIssuesEdited(issuesEdited) +} + +func loadPreSnowflakeBucket() []PreSnowflakeIssue { + f, err := os.Open("presnowflake_bucket.csv") + if err != nil { + panic(err) + } + defer f.Close() + csvReader := csv.NewReader(f) + records, err := csvReader.ReadAll() + if err != nil { + panic(err) + } + issues := make([]PreSnowflakeIssue, 0) + for _, record := range records { + number, err := strconv.Atoi(record[0]) + if err != nil { + panic(err) + } + issues = append(issues, PreSnowflakeIssue{ + Number: number, + }) + } + return issues +} + +func loadIssues() []i.Issue { + bytes, err := os.ReadFile("../gh/issues.json") + if err != nil { + panic(err) + } + var issues []i.Issue + err = json.Unmarshal(bytes, &issues) + if err != nil { + panic(err) + } + return issues +} + +func filterIssues(issues []i.Issue, preSnowflakeBucket []PreSnowflakeIssue) ([]IssueToClose, []IssueEdited) { + today := time.Now() + oneHundredEightyDaysAgo := today.Add(-24 * 180 * time.Hour) + issuesToClose := make([]IssueToClose, 0) + issuesEdited := make([]IssueEdited, 0) + for idx, issue := range issues { + fmt.Printf("Processing issue (%d): %d\n", idx+1, issue.Number) + if !slices.Contains(preSnowflakeBucket, PreSnowflakeIssue{issue.Number}) { + fmt.Printf("issue #%d is not in the Pre Snowflake bucket, skipping\n", issue.Number) + continue + } + if issue.UpdatedAt.After(oneHundredEightyDaysAgo) { + fmt.Printf("issue #%d was edited after %s, skipping\n", issue.Number, oneHundredEightyDaysAgo) + issueEdited := IssueEdited{ + ID: issue.Number, + URL: issue.HtmlUrl, + NamedURL: fmt.Sprintf(`=HYPERLINK("%s","#%d")`, issue.HtmlUrl, issue.Number), + UpdatedAt: issue.UpdatedAt, + } + issuesEdited = append(issuesEdited, issueEdited) + continue + } + issueToClose := IssueToClose{ + ID: issue.Number, + URL: issue.HtmlUrl, + NamedURL: fmt.Sprintf(`=HYPERLINK("%s","#%d")`, issue.HtmlUrl, issue.Number), + UpdatedAt: issue.UpdatedAt, + } + issuesToClose = append(issuesToClose, issueToClose) + } + return issuesToClose, issuesEdited +} + +func saveIssuesToClose(issues []IssueToClose) { + file, err := os.Create("issues_to_close.csv") + if err != nil { + panic(err) + } + defer file.Close() + w := csv.NewWriter(file) + + data := make([][]string, 0, len(issues)) + for _, issue := range issues { + row := []string{ + strconv.Itoa(issue.ID), + } + data = append(data, row) + } + _ = w.WriteAll(data) +} + +func saveIssuesEdited(issues []IssueEdited) { + file, err := os.Create("issues_edited.csv") + if err != nil { + panic(err) + } + defer file.Close() + w := csv.NewWriter(file) + + data := make([][]string, 0, len(issues)) + for _, issue := range issues { + row := []string{ + strconv.Itoa(issue.ID), + } + data = append(data, row) + } + _ = w.WriteAll(data) +} + +type PreSnowflakeIssue struct { + Number int +} + +type IssueToClose struct { + ID int + URL string + NamedURL string + UpdatedAt time.Time +} + +type IssueEdited struct { + ID int + URL string + NamedURL string + UpdatedAt time.Time +} diff --git a/pkg/scripts/issues/filter-closeable-old-issues/presnowflake_bucket.csv b/pkg/scripts/issues/filter-closeable-old-issues/presnowflake_bucket.csv new file mode 100644 index 0000000000..7809a83cc9 --- /dev/null +++ b/pkg/scripts/issues/filter-closeable-old-issues/presnowflake_bucket.csv @@ -0,0 +1,193 @@ +1267 +1249 +1221 +1204 +1161 +1075 +1068 +1067 +1066 +1061 +1042 +1029 +1005 +1004 +1000 +998 +994 +993 +989 +984 +981 +980 +977 +976 +973 +970 +967 +966 +965 +963 +955 +953 +952 +951 +950 +946 +938 +897 +896 +892 +891 +882 +867 +865 +859 +847 +845 +844 +836 +835 +825 +822 +821 +818 +810 +806 +802 +800 +794 +793 +791 +788 +787 +781 +778 +776 +774 +772 +770 +760 +757 +756 +754 +753 +749 +744 +741 +738 +737 +735 +723 +722 +719 +718 +712 +708 +690 +689 +688 +684 +683 +679 +678 +677 +676 +675 +674 +670 +665 +663 +660 +651 +648 +644 +642 +641 +640 +636 +630 +623 +621 +617 +616 +612 +611 +608 +606 +604 +601 +596 +594 +592 +587 +586 +583 +569 +568 +566 +564 +562 +560 +555 +551 +533 +531 +529 +522 +521 +514 +512 +509 +506 +502 +500 +498 +497 +494 +490 +474 +470 +466 +460 +455 +448 +447 +445 +430 +423 +421 +420 +419 +411 +410 +409 +408 +402 +384 +317 +308 +306 +303 +300 +295 +288 +285 +284 +282 +275 +273 +265 +254 +244 +225 +222 +218 +211 +200 +189 +186 +152 +142 +137 +37 \ No newline at end of file