diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml new file mode 100644 index 0000000..8ac1738 --- /dev/null +++ b/.github/workflows/nightly.yml @@ -0,0 +1,62 @@ +name: Nightly +on: + workflow_dispatch: + schedule: + - cron: '36 7 * * *' + + +jobs: + update-schema: + runs-on: ubuntu-latest + steps: + - uses: tibdex/github-app-token@v1 + id: generate-token + with: + app_id: ${{ secrets.APP_ID }} + private_key: ${{ secrets.APP_PRIVATE_KEY }} + + - name: Checkout latest code + uses: actions/checkout@v2.4.0 + + - name: Install Go + uses: actions/setup-go@v2.1.5 + with: + go-version: 1.17 + + - name: Get Go environment + id: go-env + run: | + echo "::set-output name=cache::$(go env GOCACHE)" + echo "::set-output name=modcache::$(go env GOMODCACHE)" + - name: Set up cache + uses: actions/cache@v2.1.7 + with: + path: | + ${{ steps.go-env.outputs.cache }} + ${{ steps.go-env.outputs.modcache }} + key: update-schema-${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + update-schema-${{ runner.os }}-go- + + - name: Update schema + id: update-schema + run: | + make update_schema | tee output.txt + echo "::set-output name=output::$(cat output.txt)" + + - name: Create PR + id: cpr + uses: peter-evans/create-pull-request@v3.12.0 + with: + author: GitHub + committer: GitHub + commit-message: 'feat(schema): update schema documentation to the latest layer' + branch: "feat/update-schema" + delete-branch: true + token: ${{ steps.generate-token.outputs.token }} + title: Update schema documentation to the latest layer. + body: | + Update schema documentation to the latest layer. + ``` + ${{ steps.update-schema.outputs.output }} + ``` diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index eeacafe..11462e9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,35 +1,12 @@ -name: release +name: pkg on: + release: push: tags: - v* + workflow_dispatch: jobs: - fetch: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Set output - id: vars - run: | - echo ::set-output name=tag::${GITHUB_REF#refs/*/} - echo ::set-output name=module::$(cat go.mod | head -n 1 | awk '{print $2}') - - name: curl - run: | - curl --show-error --fail -w "HTTP %{http_code}\n" -o - 'https://pkg.go.dev/fetch/${{ steps.vars.outputs.module }}@${{ steps.vars.outputs.tag }}' \ - -X 'POST' \ - -H 'authority: pkg.go.dev' \ - -H 'content-length: 0' \ - -H 'sec-ch-ua: "Google Chrome";v="95", "Chromium";v="95", ";Not A Brand";v="99"' \ - -H 'sec-ch-ua-mobile: ?0' \ - -H 'user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36' \ - -H 'sec-ch-ua-platform: "Linux"' \ - -H 'accept: */*' \ - -H 'origin: https://pkg.go.dev' \ - -H 'sec-fetch-site: same-origin' \ - -H 'sec-fetch-mode: cors' \ - -H 'sec-fetch-dest: empty' \ - -H 'referer: https://pkg.go.dev/${{ steps.vars.outputs.module }}@${{ steps.vars.outputs.tag }}' \ - -H 'accept-language: en-US,en;q=0.9,ru;q=0.8' \ - --compressed + run: + uses: go-faster/x/.github/workflows/release.yml@main diff --git a/.golangci.yml b/.golangci.yml index a85844e..af12400 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,14 +1,12 @@ linters-settings: govet: check-shadowing: true - golint: - min-confidence: 0 gocyclo: min-complexity: 15 maligned: suggest-new: true dupl: - threshold: 100 + threshold: 120 goconst: min-len: 2 min-occurrences: 3 @@ -31,7 +29,11 @@ linters-settings: - exitAfterDefer - whyNoLint - singleCaseSwitch + - commentedOutCode + - appendAssign - octalLiteral + - preferWriteByte + - httpNoBody linters: disable-all: true @@ -44,10 +46,9 @@ linters: - gochecknoinits - goconst - gocritic - - gocyclo - gofmt - goimports - - golint + - revive - gosec - gosimple - govet @@ -55,7 +56,6 @@ linters: - lll - misspell - nakedret - - scopelint - staticcheck - structcheck - stylecheck @@ -65,7 +65,6 @@ linters: - unused - varcheck - whitespace - - gocognit # Do not enable: # - wsl (too opinionated about newlines) @@ -74,12 +73,13 @@ linters: # - prealloc (not worth it in scope of this project) # - maligned (same as prealloc) # - funlen (gocyclo is enough) + # - gochecknoglobals (we know when it is ok to use globals) issues: exclude-rules: # Exclude go:generate from lll - source: "//go:generate" - linters: [lll] + linters: [ lll ] # Disable linters that are annoying in tests. - path: _test\.go @@ -93,10 +93,21 @@ issues: - gocognit - scopelint - lll - - gochecknoglobals + # Ignore shadowing of err. - - linters: [govet] - text: 'declaration of "err"' + - linters: [ govet ] + text: 'declaration of "(err|ctx|log)"' + # Ignore linters in main packages. - - path: main\.go - linters: [gochecknoglobals, goconst, funlen, gocognit, gocyclo] + - path: cmd\/.+\/main\.go + linters: [ goconst, funlen, gocognit, gocyclo ] + + # Intended in commands: + # G307: Deferring unsafe method "Close" on type "*os.File" + # G304: Potential file inclusion via variable + - path: cmd\/.+\/main\.go + text: G(307|304) + + - linters: [ revive ] + text: "if-return: redundant if ...; err != nil check, just return error instead." + diff --git a/Makefile b/Makefile index bc2cda7..5cac909 100644 --- a/Makefile +++ b/Makefile @@ -2,3 +2,6 @@ test: @./go.test.sh coverage: @./go.coverage.sh +update_schema: + @go run github.com/gotd/getdoc/cmd/getdoc -out-dir _schema -out-file latest.json -pretty true + @go run github.com/gotd/getdoc/cmd/getdoc -out-dir _schema -pretty true diff --git a/README.md b/README.md index 975d2d7..ca02a8f 100644 --- a/README.md +++ b/README.md @@ -6,18 +6,18 @@ Used by [gotd/td](https://github.com/gotd/td) for embedding documentation to gen ## Parsed documentation -Parsed documentation for 121 layer is available as [121.json](./_schema/121.json) with [schema](./_schema/schema.json). +Parsed documentation for 133 layer is available as [133.json](./_schema/133.json) with [schema](./_schema/schema.json). ## Example Latest schema is embedded to package, so you can just use it: ```go -doc, err := getdoc.Load(121) +doc, err := getdoc.Load(133) if err != nil { panic(err) } fmt.Printf("Layer %d, constructors: %d\n", doc.Index.Layer, len(doc.Constructors)) // Output: -// Layer 121, constructors: 851 +// Layer 133, constructors: 926 ``` ## Reference diff --git a/cmd/getdoc/main.go b/cmd/getdoc/main.go index 9ea8596..1e3ce4c 100644 --- a/cmd/getdoc/main.go +++ b/cmd/getdoc/main.go @@ -16,6 +16,8 @@ import ( func main() { dir := flag.String("dir", filepath.Join(os.TempDir(), "getdoc"), "working directory") + outDir := flag.String("out-dir", "", "path to write schema") + outFile := flag.String("out-file", "", "filename of schema") readonly := flag.Bool("readonly", false, "read-only mode") pretty := flag.Bool("pretty", false, "pretty json output") flag.Parse() @@ -29,7 +31,6 @@ func main() { } ctx := context.Background() - fmt.Println("Extracting") doc, err := getdoc.Extract(ctx, client) if err != nil { panic(err) @@ -45,8 +46,19 @@ func main() { } outFileName := fmt.Sprintf("%d.json", doc.Index.Layer) + if out := *outFile; out != "" { + outFileName = out + } + outFilePath := filepath.Join(*dir, outFileName) - if err := os.WriteFile(outFilePath, out.Bytes(), 0600); err != nil { + if out := *outDir; out != "" { + if err := os.MkdirAll(out, 0o600); err != nil { + panic(err) + } + outFilePath = filepath.Join(out, outFileName) + } + + if err := os.WriteFile(outFilePath, out.Bytes(), 0o600); err != nil { panic(err) } diff --git a/getdoc.go b/getdoc.go index dd8d022..433b775 100644 --- a/getdoc.go +++ b/getdoc.go @@ -49,14 +49,14 @@ func docDescription(doc *goquery.Document) (desc, links []string) { return } -// docTableAfter extracts table after selector "after". -func docTableAfter(doc *goquery.Document, after string) *goquery.Selection { +// docTableAfterFunc extracts table after selector "after". +func docTableAfterFunc(doc *goquery.Document, after func(s *goquery.Selection) bool) *goquery.Selection { var ( meetAfter bool table *goquery.Selection ) doc.Find("#dev_page_content").Children().EachWithBreak(func(i int, s *goquery.Selection) bool { - if s.Find(after).Length() > 0 { + if after(s) { // Found title of table. Next element will be requested table. meetAfter = true return true @@ -86,21 +86,26 @@ type ParamDescription struct { func docParams(doc *goquery.Document) map[string]ParamDescription { fields := make(map[string]ParamDescription) - docTableAfter(doc, "#parameters"). - Each(func(i int, row *goquery.Selection) { - var rowContents []string - var links []string - row.Find("td").Each(func(i int, column *goquery.Selection) { - links = addHost(href.Replace(column)) - rowContents = append(rowContents, column.Text()) - }) - if len(rowContents) == 3 { - fields[rowContents[0]] = ParamDescription{ - Name: rowContents[0], - Description: rowContents[2], - Links: links, - } - } + docTableAfterFunc(doc, func(s *goquery.Selection) bool { + return s.Find("#parameters").Length() > 0 || + // Some pages have no such selector, so we try to detect "Parameters" header by text. + // + // TODO(tdakkota): try to parse attributes + strings.HasPrefix(s.Text(), "Parameters") + }).Each(func(i int, row *goquery.Selection) { + var rowContents []string + var links []string + row.Find("td").Each(func(i int, column *goquery.Selection) { + links = addHost(href.Replace(column)) + rowContents = append(rowContents, column.Text()) }) + if len(rowContents) == 3 { + fields[rowContents[0]] = ParamDescription{ + Name: rowContents[0], + Description: rowContents[2], + Links: links, + } + } + }) return fields } diff --git a/method.go b/method.go index cb8bde2..09e3ff5 100644 --- a/method.go +++ b/method.go @@ -34,26 +34,31 @@ func docBotCanUser(doc *goquery.Document) bool { func docErrors(doc *goquery.Document) []Error { var output []Error - docTableAfter(doc, "#possible-errors"). - Each(func(i int, row *goquery.Selection) { - var rowContents []string - row.Find("td").Each(func(i int, column *goquery.Selection) { - rowContents = append(rowContents, column.Text()) - }) - if len(rowContents) != 3 { - return - } - code, err := strconv.Atoi(rowContents[0]) - if err != nil { - return - } - e := Error{ - Code: code, - Type: strings.TrimSpace(rowContents[1]), - Description: strings.TrimSpace(rowContents[2]), - } - output = append(output, e) + docTableAfterFunc(doc, func(s *goquery.Selection) bool { + return s.Find("#possible-errors").Length() > 0 || + // Some pages have no such selector, so we try to detect "Possible errors" header by text. + // + // TODO(tdakkota): try to parse attributes + strings.HasPrefix(s.Text(), "Possible errors") + }).Each(func(i int, row *goquery.Selection) { + var rowContents []string + row.Find("td").Each(func(i int, column *goquery.Selection) { + rowContents = append(rowContents, column.Text()) }) + if len(rowContents) != 3 { + return + } + code, err := strconv.Atoi(rowContents[0]) + if err != nil { + return + } + e := Error{ + Code: code, + Type: strings.TrimSpace(rowContents[1]), + Description: strings.TrimSpace(rowContents[2]), + } + output = append(output, e) + }) return output }