fix: parsing parameters and errors

.github/workflows/nightly.yml

@@ -0,0 +1,62 @@
+name: Nightly
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '36 7 * * *'
+
+
+jobs:
+  update-schema:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: tibdex/github-app-token@v1
+        id: generate-token
+        with:
+          app_id: ${{ secrets.APP_ID }}
+          private_key: ${{ secrets.APP_PRIVATE_KEY }}
+
+      - name: Checkout latest code
+        uses: actions/checkout@v2.4.0
+
+      - name: Install Go
+        uses: actions/setup-go@v2.1.5
+        with:
+          go-version: 1.17
+
+      - name: Get Go environment
+        id: go-env
+        run: |
+          echo "::set-output name=cache::$(go env GOCACHE)"
+          echo "::set-output name=modcache::$(go env GOMODCACHE)"
+      - name: Set up cache
+        uses: actions/cache@v2.1.7
+        with:
+          path: |
+            ${{ steps.go-env.outputs.cache }}
+            ${{ steps.go-env.outputs.modcache }}
+          key: update-schema-${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            update-schema-${{ runner.os }}-go-
+
+      - name: Update schema
+        id: update-schema
+        run: |
+          make update_schema | tee output.txt
+          echo "::set-output name=output::$(cat output.txt)"
+
+      - name: Create PR
+        id: cpr
+        uses: peter-evans/create-pull-request@v3.12.0
+        with:
+          author: GitHub <actions@github.com>
+          committer: GitHub <actions@github.com>
+          commit-message: 'feat(schema): update schema documentation to the latest layer'
+          branch: "feat/update-schema"
+          delete-branch: true
+          token: ${{ steps.generate-token.outputs.token }}
+          title: Update schema documentation to the latest layer.
+          body: |
+            Update schema documentation to the latest layer.
+            ```
+            ${{ steps.update-schema.outputs.output }}
+            ```

.github/workflows/release.yml

@@ -1,35 +1,12 @@
-name: release
+name: pkg
 
 on:
+  release:
   push:
     tags:
       - v*
+  workflow_dispatch:
 
 jobs:
-  fetch:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Set output
-        id: vars
-        run: |
-          echo ::set-output name=tag::${GITHUB_REF#refs/*/}
-          echo ::set-output name=module::$(cat go.mod | head -n 1 | awk '{print $2}')
-      - name: curl
-        run: |
-          curl --show-error --fail -w "HTTP %{http_code}\n" -o - 'https://pkg.go.dev/fetch/${{ steps.vars.outputs.module }}@${{ steps.vars.outputs.tag }}' \
-            -X 'POST' \
-            -H 'authority: pkg.go.dev' \
-            -H 'content-length: 0' \
-            -H 'sec-ch-ua: "Google Chrome";v="95", "Chromium";v="95", ";Not A Brand";v="99"' \
-            -H 'sec-ch-ua-mobile: ?0' \
-            -H 'user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36' \
-            -H 'sec-ch-ua-platform: "Linux"' \
-            -H 'accept: */*' \
-            -H 'origin: https://pkg.go.dev' \
-            -H 'sec-fetch-site: same-origin' \
-            -H 'sec-fetch-mode: cors' \
-            -H 'sec-fetch-dest: empty' \
-            -H 'referer: https://pkg.go.dev/${{ steps.vars.outputs.module }}@${{ steps.vars.outputs.tag }}' \
-            -H 'accept-language: en-US,en;q=0.9,ru;q=0.8' \
-            --compressed
+  run:
+    uses: go-faster/x/.github/workflows/release.yml@main

.golangci.yml

@@ -1,14 +1,12 @@
 linters-settings:
   govet:
     check-shadowing: true
-  golint:
-    min-confidence: 0
   gocyclo:
     min-complexity: 15
   maligned:
     suggest-new: true
   dupl:
-    threshold: 100
+    threshold: 120
   goconst:
     min-len: 2
     min-occurrences: 3
@@ -31,7 +29,11 @@ linters-settings:
       - exitAfterDefer
       - whyNoLint
       - singleCaseSwitch
+      - commentedOutCode
+      - appendAssign
       - octalLiteral
+      - preferWriteByte
+      - httpNoBody
 
 linters:
   disable-all: true
@@ -44,18 +46,16 @@ linters:
     - gochecknoinits
     - goconst
     - gocritic
-    - gocyclo
     - gofmt
     - goimports
-    - golint
+    - revive
     - gosec
     - gosimple
     - govet
     - ineffassign
     - lll
     - misspell
     - nakedret
-    - scopelint
     - staticcheck
     - structcheck
     - stylecheck
@@ -65,7 +65,6 @@ linters:
     - unused
     - varcheck
     - whitespace
-    - gocognit
 
   # Do not enable:
   # - wsl       (too opinionated about newlines)
@@ -74,12 +73,13 @@ linters:
   # - prealloc  (not worth it in scope of this project)
   # - maligned  (same as prealloc)
   # - funlen    (gocyclo is enough)
+  # - gochecknoglobals (we know when it is ok to use globals)
 
 issues:
   exclude-rules:
     # Exclude go:generate from lll
     - source: "//go:generate"
-      linters: [lll]
+      linters: [ lll ]
 
     # Disable linters that are annoying in tests.
     - path: _test\.go
@@ -93,10 +93,21 @@ issues:
         - gocognit
         - scopelint
         - lll
-        - gochecknoglobals
+
     # Ignore shadowing of err.
-    - linters: [govet]
-      text: 'declaration of "err"'
+    - linters: [ govet ]
+      text: 'declaration of "(err|ctx|log)"'
+
     # Ignore linters in main packages.
-    - path: main\.go
-      linters: [gochecknoglobals, goconst, funlen, gocognit, gocyclo]
+    - path: cmd\/.+\/main\.go
+      linters: [ goconst, funlen, gocognit, gocyclo ]
+
+    # Intended in commands:
+    # G307: Deferring unsafe method "Close" on type "*os.File"
+    # G304: Potential file inclusion via variable
+    - path: cmd\/.+\/main\.go
+      text: G(307|304)
+
+    - linters: [ revive ]
+      text: "if-return: redundant if ...; err != nil check, just return error instead."
+

Makefile

@@ -2,3 +2,6 @@ test:
 	@./go.test.sh
 coverage:
 	@./go.coverage.sh
+update_schema:
+	@go run github.com/gotd/getdoc/cmd/getdoc -out-dir _schema -out-file latest.json -pretty true
+	@go run github.com/gotd/getdoc/cmd/getdoc -out-dir _schema -pretty true

README.md

@@ -6,18 +6,18 @@ Used by [gotd/td](https://github.com/gotd/td) for embedding documentation to gen
 
 ## Parsed documentation
 
-Parsed documentation for 121 layer is available as [121.json](./_schema/121.json) with [schema](./_schema/schema.json).
+Parsed documentation for 133 layer is available as [133.json](./_schema/133.json) with [schema](./_schema/schema.json).
 
 ## Example
 Latest schema is embedded to package, so you can just use it:
 ```go
-doc, err := getdoc.Load(121)
+doc, err := getdoc.Load(133)
 if err != nil {
     panic(err)
 }
 fmt.Printf("Layer %d, constructors: %d\n", doc.Index.Layer, len(doc.Constructors))
 // Output:
-// Layer 121, constructors: 851
+// Layer 133, constructors: 926
 ```
 
 ## Reference

cmd/getdoc/main.go

@@ -16,6 +16,8 @@ import (
 
 func main() {
 	dir := flag.String("dir", filepath.Join(os.TempDir(), "getdoc"), "working directory")
+	outDir := flag.String("out-dir", "", "path to write schema")
+	outFile := flag.String("out-file", "", "filename of schema")
 	readonly := flag.Bool("readonly", false, "read-only mode")
 	pretty := flag.Bool("pretty", false, "pretty json output")
 	flag.Parse()
@@ -29,7 +31,6 @@ func main() {
 	}
 
 	ctx := context.Background()
-	fmt.Println("Extracting")
 	doc, err := getdoc.Extract(ctx, client)
 	if err != nil {
 		panic(err)
@@ -45,8 +46,19 @@ func main() {
 	}
 
 	outFileName := fmt.Sprintf("%d.json", doc.Index.Layer)
+	if out := *outFile; out != "" {
+		outFileName = out
+	}
+
 	outFilePath := filepath.Join(*dir, outFileName)
-	if err := os.WriteFile(outFilePath, out.Bytes(), 0600); err != nil {
+	if out := *outDir; out != "" {
+		if err := os.MkdirAll(out, 0o600); err != nil {
+			panic(err)
+		}
+		outFilePath = filepath.Join(out, outFileName)
+	}
+
+	if err := os.WriteFile(outFilePath, out.Bytes(), 0o600); err != nil {
 		panic(err)
 	}
 

getdoc.go

@@ -49,14 +49,14 @@ func docDescription(doc *goquery.Document) (desc, links []string) {
 	return
 }
 
-// docTableAfter extracts table after selector "after".
-func docTableAfter(doc *goquery.Document, after string) *goquery.Selection {
+// docTableAfterFunc extracts table after selector "after".
+func docTableAfterFunc(doc *goquery.Document, after func(s *goquery.Selection) bool) *goquery.Selection {
 	var (
 		meetAfter bool
 		table     *goquery.Selection
 	)
 	doc.Find("#dev_page_content").Children().EachWithBreak(func(i int, s *goquery.Selection) bool {
-		if s.Find(after).Length() > 0 {
+		if after(s) {
 			// Found title of table. Next <table> element will be requested table.
 			meetAfter = true
 			return true
@@ -86,21 +86,26 @@ type ParamDescription struct {
 func docParams(doc *goquery.Document) map[string]ParamDescription {
 	fields := make(map[string]ParamDescription)
 
-	docTableAfter(doc, "#parameters").
-		Each(func(i int, row *goquery.Selection) {
-			var rowContents []string
-			var links []string
-			row.Find("td").Each(func(i int, column *goquery.Selection) {
-				links = addHost(href.Replace(column))
-				rowContents = append(rowContents, column.Text())
-			})
-			if len(rowContents) == 3 {
-				fields[rowContents[0]] = ParamDescription{
-					Name:        rowContents[0],
-					Description: rowContents[2],
-					Links:       links,
-				}
-			}
+	docTableAfterFunc(doc, func(s *goquery.Selection) bool {
+		return s.Find("#parameters").Length() > 0 ||
+			// Some pages have no such selector, so we try to detect "Parameters" header by text.
+			//
+			// TODO(tdakkota): try to parse attributes
+			strings.HasPrefix(s.Text(), "Parameters")
+	}).Each(func(i int, row *goquery.Selection) {
+		var rowContents []string
+		var links []string
+		row.Find("td").Each(func(i int, column *goquery.Selection) {
+			links = addHost(href.Replace(column))
+			rowContents = append(rowContents, column.Text())
 		})
+		if len(rowContents) == 3 {
+			fields[rowContents[0]] = ParamDescription{
+				Name:        rowContents[0],
+				Description: rowContents[2],
+				Links:       links,
+			}
+		}
+	})
 	return fields
 }

method.go

@@ -34,26 +34,31 @@ func docBotCanUser(doc *goquery.Document) bool {
 func docErrors(doc *goquery.Document) []Error {
 	var output []Error
 
-	docTableAfter(doc, "#possible-errors").
-		Each(func(i int, row *goquery.Selection) {
-			var rowContents []string
-			row.Find("td").Each(func(i int, column *goquery.Selection) {
-				rowContents = append(rowContents, column.Text())
-			})
-			if len(rowContents) != 3 {
-				return
-			}
-			code, err := strconv.Atoi(rowContents[0])
-			if err != nil {
-				return
-			}
-			e := Error{
-				Code:        code,
-				Type:        strings.TrimSpace(rowContents[1]),
-				Description: strings.TrimSpace(rowContents[2]),
-			}
-			output = append(output, e)
+	docTableAfterFunc(doc, func(s *goquery.Selection) bool {
+		return s.Find("#possible-errors").Length() > 0 ||
+			// Some pages have no such selector, so we try to detect "Possible errors" header by text.
+			//
+			// TODO(tdakkota): try to parse attributes
+			strings.HasPrefix(s.Text(), "Possible errors")
+	}).Each(func(i int, row *goquery.Selection) {
+		var rowContents []string
+		row.Find("td").Each(func(i int, column *goquery.Selection) {
+			rowContents = append(rowContents, column.Text())
 		})
+		if len(rowContents) != 3 {
+			return
+		}
+		code, err := strconv.Atoi(rowContents[0])
+		if err != nil {
+			return
+		}
+		e := Error{
+			Code:        code,
+			Type:        strings.TrimSpace(rowContents[1]),
+			Description: strings.TrimSpace(rowContents[2]),
+		}
+		output = append(output, e)
+	})
 	return output
 }