Fix models scraping + minor refactoring (#4)

.gitignore

@@ -26,4 +26,6 @@ _testmain.go
 *.prof
 
 # go-syno diretories
-vendor
+vendor
+coverage.out
+

go.mod

@@ -1,16 +1,20 @@
 module jdel.org/go-syno
 
-go 1.13
+go 1.19
 
 require (
-	github.com/PuerkitoBio/goquery v1.5.0
-	github.com/andybalholm/cascadia v1.1.0 // indirect
+	github.com/PuerkitoBio/goquery v1.8.0
 	github.com/calmh/versions v1.0.1
-	github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
-	github.com/sirupsen/logrus v1.4.2
-	github.com/smartystreets/goconvey v1.6.4 // indirect
-	golang.org/x/net v0.0.0-20191126235420-ef20fe5d7933 // indirect
-	golang.org/x/sys v0.0.0-20191128015809-6d18c012aee9 // indirect
-	gopkg.in/ini.v1 v1.51.0
-	gopkg.in/yaml.v2 v2.2.7
+	github.com/robertkrimen/otto v0.2.1
+	github.com/sirupsen/logrus v1.9.0
+	gopkg.in/ini.v1 v1.67.0
+	gopkg.in/yaml.v2 v2.4.0
+)
+
+require (
+	github.com/andybalholm/cascadia v1.3.1 // indirect
+	golang.org/x/net v0.5.0 // indirect
+	golang.org/x/sys v0.4.0 // indirect
+	golang.org/x/text v0.6.0 // indirect
+	gopkg.in/sourcemap.v1 v1.0.5 // indirect
 )

go.sum

@@ -1,46 +1,41 @@
-github.com/PuerkitoBio/goquery v1.5.0 h1:uGvmFXOA73IKluu/F84Xd1tt/z07GYm8X49XKHP7EJk=
-github.com/PuerkitoBio/goquery v1.5.0/go.mod h1:qD2PgZ9lccMbQlc7eEOjaeRlFQON7xY8kdmcsrnKqMg=
-github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
-github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo=
-github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
+github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
+github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
+github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
+github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
 github.com/calmh/versions v1.0.1 h1:thI7mRSAPf26A+tjUjkIkDs62Fit7lY04YOeuy2k6/A=
 github.com/calmh/versions v1.0.1/go.mod h1:QLYDeR6z1lbfdydViIbnKUJ5jCLdDX4NV3fqBn7rTrk=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
-github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
-github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
-github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
-github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
-github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
-github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
-github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
-github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
-github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
-github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
-github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
-github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
-github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20191126235420-ef20fe5d7933 h1:e6HwijUxhDe+hPNjZQQn9bA5PW3vNmnN64U2ZW759Lk=
-golang.org/x/net v0.0.0-20191126235420-ef20fe5d7933/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191128015809-6d18c012aee9 h1:ZBzSG/7F4eNKz2L3GE9o300RX0Az1Bw5HF7PDraD+qU=
-golang.org/x/sys v0.0.0-20191128015809-6d18c012aee9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+github.com/robertkrimen/otto v0.2.1 h1:FVP0PJ0AHIjC+N4pKCG9yCDz6LHNPCwi/GKID5pGGF0=
+github.com/robertkrimen/otto v0.2.1/go.mod h1:UPwtJ1Xu7JrLcZjNWN8orJaM5n5YEtqL//farB5FlRY=
+github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
+github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
+golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw=
+golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18=
+golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k=
+golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno=
-gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
-gopkg.in/yaml.v2 v2.2.7 h1:VUgggvou5XRW9mHwD/yXxIYSMtY0zoKQf/v226p2nyo=
-gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
+gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/sourcemap.v1 v1.0.5 h1:inv58fC9f9J3TK2Y2R1NPntXEn3/wjWHkonhIUODNTI=
+gopkg.in/sourcemap.v1 v1.0.5/go.mod h1:2RlvNNSMglmRrcvhfuzp4hQHwOtjxlbjX7UPY/GXb78=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

models.go

@@ -1,12 +1,13 @@
 package syno // import jdel.org/go-syno/syno
 
 import (
-	"io/ioutil"
 	"net/http"
 	"os"
+	"regexp"
 	"strings"
 
 	"github.com/PuerkitoBio/goquery"
+	"github.com/robertkrimen/otto"
 	log "github.com/sirupsen/logrus"
 	yaml "gopkg.in/yaml.v2"
 )
@@ -70,7 +71,7 @@ func (m Models) SaveModelsFile() error {
 	if err != nil {
 		return err
 	}
-	err = ioutil.WriteFile(o.ModelsFile, yamlModels, 0755)
+	err = os.WriteFile(o.ModelsFile, yamlModels, 0755)
 	if err != nil {
 		return err
 	}
@@ -87,7 +88,7 @@ func modelsFileExists() bool {
 
 func getModelsFromModelsFile() (Models, error) {
 	var models Models
-	bytes, err := ioutil.ReadFile(o.ModelsFile)
+	bytes, err := os.ReadFile(o.ModelsFile)
 	if err != nil {
 		return nil, err
 	}
@@ -98,22 +99,45 @@ func getModelsFromModelsFile() (Models, error) {
 	return models, nil
 }
 
-// CrawlModels fetches Synology models from
-// The official Synology wiki
+// getModelsFromInternet fetches Synology models from
+// The official Synology kb
 func getModelsFromInternet() (Models, error) {
-	resp, err := http.Get("https://www.synology.com/api/knowledgebase/findDocByUri?uri=DSM%2Ftutorial%2FCompatibility_Peripherals%2FWhat_kind_of_CPU_does_my_NAS_have&lang=en-global&p_type=DSM&d_type=tutorial")
+	resp, err := http.Get("https://kb.synology.com/en-me/DSM/tutorial/What_kind_of_CPU_does_my_NAS_have")
 	if err != nil && resp != nil && resp.StatusCode != 200 && resp.StatusCode != 302 {
 		return nil, err
 	}
 
-	doc, err := goquery.NewDocumentFromResponse(resp)
+	doc, err := goquery.NewDocumentFromReader(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	// All the html we want is embedded in the last <script> tag
+	var lastDeferedScript string
+	doc.Find("script[defer]").Each(func(i int, s *goquery.Selection) {
+		lastDeferedScript = s.Text()
+	})
+
+	// Extract the content capture group
+	r := regexp.MustCompile(`"content":"(.*)","tags"`)
+	match := r.FindStringSubmatch(lastDeferedScript)[1]
+
+	// Let otto handle the double quoted JS from content
+	vm := otto.New()
+	inlineHtml, err := vm.Run(`"` + match + `"`)
+	if err != nil {
+		return nil, err
+	}
+
+	// New sub document to parse inline HTML
+	subDoc, err := goquery.NewDocumentFromReader(strings.NewReader(inlineHtml.String()))
 	if err != nil {
 		return nil, err
 	}
 
 	var models Models
 
-	doc.Find("tr").Each(func(i int, s *goquery.Selection) {
+	subDoc.Find("tr").Each(func(i int, s *goquery.Selection) {
 		tds := s.ChildrenFiltered("td")
 		if tds.Size() == 7 {
 			model := &Model{

packages.go

@@ -79,7 +79,7 @@ func (p Packages) FilterByFirmware(query string) Packages {
 func (p Packages) FilterOutBeta() Packages {
 	output := Packages{}
 	for _, synoPkg := range p {
-		if synoPkg.Beta == false {
+		if !synoPkg.Beta {
 			output = append(output, synoPkg)
 		}
 	}
@@ -232,6 +232,6 @@ func archIsInFamilyAndMatches(arch, query string) bool {
 	return false
 }
 
-func (p Package) familyOrArchMatch(query string) bool {
+func (p *Package) familyOrArchMatch(query string) bool {
 	return archIsInFamilyAndMatches(p.Arch, query) || archIsInFamilyAndMatches(query, p.Arch)
 }

packages_io.go

@@ -6,7 +6,6 @@ import (
 	"encoding/hex"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"regexp"
@@ -21,10 +20,10 @@ func (p *Package) containsFiles(fileNamePattern string) (bool, error) {
 	var spkFile *os.File
 
 	spkFile, err = os.Open(filepath.Join(o.PackagesDir, p.fileName))
-	defer spkFile.Close()
 	if err != nil {
 		return false, err
 	}
+	defer spkFile.Close()
 
 	tarReader := tar.NewReader(spkFile)
 
@@ -49,10 +48,10 @@ func (p *Package) extractFiles(fileNamePattern string) ([]string, error) {
 	var extractedFiles []string
 
 	spkFile, err = os.Open(filepath.Join(o.PackagesDir, p.fileName))
-	defer spkFile.Close()
 	if err != nil {
 		return extractedFiles, err
 	}
+	defer spkFile.Close()
 
 	tarReader := tar.NewReader(spkFile)
 
@@ -113,7 +112,7 @@ func (p *Package) extractInfo() error {
 		if err := os.Rename(filepath.Join(o.PackagesDir, p.fileName), fmt.Sprintf("%s.ignored", filepath.Join(o.PackagesDir, p.fileName))); err != nil {
 			return err
 		}
-		return fmt.Errorf("Quatantined %s: No INFO or not a tar file", p.fileName)
+		return fmt.Errorf("quatantined %s: No INFO or not a tar file", p.fileName)
 	}
 	return nil
 }
@@ -133,11 +132,11 @@ func (p *Package) getOrExtractInfo() (string, error) {
 func (p *Package) parseInfo() (*ini.File, error) {
 	spkinfoFilePath, err := p.getOrExtractInfo()
 	if err != nil {
-		return nil, fmt.Errorf("Cannot extract INFO for %s: %s", p.fileName, err)
+		return nil, fmt.Errorf("cannot extract INFO for %s: %s", p.fileName, err)
 	}
 	infoINI, err := ini.InsensitiveLoad(spkinfoFilePath)
 	if err != nil {
-		return nil, fmt.Errorf("Cannot read INFO for %s: %s", p.fileName, err)
+		return nil, fmt.Errorf("cannot read INFO for %s: %s", p.fileName, err)
 	}
 	infoINI.BlockMode = false
 	return infoINI, nil
@@ -154,10 +153,10 @@ func (p *Package) extractImages() ([]string, error) {
 
 func (p *Package) getImages() ([]string, error) {
 	var err error
-	var files []os.FileInfo
+	var files []os.DirEntry
 	var images []string
 
-	files, err = ioutil.ReadDir(filepath.Join(o.CacheDir, p.fileName))
+	files, err = os.ReadDir(filepath.Join(o.CacheDir, p.fileName))
 	if err != nil {
 		return nil, err
 	}

packages_structure.go

@@ -100,7 +100,7 @@ func (p Packages) Swap(i, j int) {
 func (p Packages) index(name, arch string) (int, error) {
 	returnIndex := 0
 	var err error
-	err = errors.New("Does not exist")
+	err = errors.New("does not exist")
 
 	for index, synoPkg := range p {
 		if synoPkg.Name == name && synoPkg.Arch == arch {

syno.go

@@ -43,6 +43,9 @@ func trimDotSlash(s string) string {
 func writeToFile(r io.Reader, f string, m os.FileMode) error {
 	os.MkdirAll(filepath.Dir(f), m)
 	outputFile, err := os.OpenFile(f, os.O_CREATE|os.O_WRONLY, m)
+	if err != nil {
+		return err
+	}
 	defer outputFile.Close()
 	_, err = io.Copy(outputFile, r)
 	return err