From 33f9535cadff4bc7246a16f6c1ed471599e9d54d Mon Sep 17 00:00:00 2001 From: Alessandro De Angelis Date: Wed, 18 Sep 2019 16:43:17 +0200 Subject: [PATCH] Fixes #36: Added support to mangadex --- README.md | 4 ++- cmd/app/downloader.go | 8 +++++ go.mod | 1 + go.sum | 8 +++++ pkg/core/core.go | 7 ++-- pkg/detector/detector.go | 2 +- pkg/sites/base.go | 1 + pkg/sites/loader.go | 2 ++ pkg/sites/mangadex.go | 67 ++++++++++++++++++++++++++++++++++++++ pkg/sites/mangadex_test.go | 37 +++++++++++++++++++++ pkg/util/util.go | 25 ++++++++++++++ 11 files changed, 157 insertions(+), 5 deletions(-) create mode 100644 pkg/sites/mangadex.go create mode 100644 pkg/sites/mangadex_test.go diff --git a/README.md b/README.md index c9447a38..94666977 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ - https://mangarock.com/ - https://www.mangareader.net/ - http://www.mangatown.com/ +- https://mangadex.org/ ## Getting Started @@ -159,7 +160,8 @@ Default is __jpg__. - [soup](https://github.com/anaskhan96/soup) - [progressbar](https://github.com/schollz/progressbar) - [logrus](https://github.com/sirupsen/logrus) -- [mri](https://github.com/BakeRolls/mri/blob/master/mri.go) +- [mri](https://github.com/bake/mri/blob/master/mri.go) +- [mangadex](https://github.com/bake/mangadex) - [archiver](https://github.com/mholt/archiver) ## Contribuiting diff --git a/cmd/app/downloader.go b/cmd/app/downloader.go index f0963afc..bb34abee 100644 --- a/cmd/app/downloader.go +++ b/cmd/app/downloader.go @@ -51,6 +51,14 @@ func download(link, format, country string, all, last, bindLogsToChannel, images continue } + if strings.Contains(source, "mangadex") && (all || last) { + msg := "`all` and `last` parameters are not supported" + log.WithFields(log.Fields{"site": u}).Warning(msg) + sendToChannel(bindLogsToChannel, msg) + all = false + last = false + } + msg := "Downloading..." log.WithFields(log.Fields{"url": u}).Info(msg) sendToChannel(bindLogsToChannel, msg) diff --git a/go.mod b/go.mod index 54e97757..94dc3972 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/BakeRolls/mri v0.0.0-20190425192657-438a0b611ec0 github.com/Girbons/mangarock v0.0.0-20190725180316-ca97fc422a34 github.com/anaskhan96/soup v1.1.1 + github.com/bake/mangadex v0.0.0-20190918122836-0261a89dc30d github.com/bmaupin/go-epub v0.5.3 github.com/dsnet/compress v0.0.1 // indirect github.com/frankban/quicktest v1.4.1 // indirect diff --git a/go.sum b/go.sum index 6df8027b..589b8f7a 100644 --- a/go.sum +++ b/go.sum @@ -8,6 +8,10 @@ github.com/Kodeworks/golang-image-ico v0.0.0-20141118225523-73f0f4cfade9/go.mod github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c= github.com/anaskhan96/soup v1.1.1 h1:Duux/0htS2Va7XLJ9qIakCSey790hg9OFRm2FwlMTy0= github.com/anaskhan96/soup v1.1.1/go.mod h1:pT5vs4HXDwA5y4KQCsKvnkpQd3D+joP7IqpiGskfWW0= +github.com/bake/httpcache v0.0.0-20190425194625-775d0adac809 h1:+WthOY6+9RdivM1SPr5KRregzgSYmu++u0uphYgmoyk= +github.com/bake/httpcache v0.0.0-20190425194625-775d0adac809/go.mod h1:4NG8FC+Gt0aSRAFe+GcCsKUGCL63u0doQyQBQffWX90= +github.com/bake/mangadex v0.0.0-20190918122836-0261a89dc30d h1:WpnqIe+TLGBW+cHHm2SA8lwlCtIMYhWNl2H+Z0peTJM= +github.com/bake/mangadex v0.0.0-20190918122836-0261a89dc30d/go.mod h1:0EIa3RFRprO7Ey+7O0TveUY6epgxsabT82R3bkUjkzU= github.com/bmaupin/go-epub v0.5.3 h1:HBOJS2KqsRGGcEzM470/9l5q2l9JwAUxQabEgPMrW90= github.com/bmaupin/go-epub v0.5.3/go.mod h1:4RBr0Zo03mRGOyGAcc25eLOqIPCkMbfz+tINVmH6clQ= github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= @@ -29,6 +33,8 @@ github.com/goki/freetype v0.0.0-20181231101311-fa8a33aabaff h1:W71vTCKoxtdXgnm1E github.com/goki/freetype v0.0.0-20181231101311-fa8a33aabaff/go.mod h1:wfqRWLHRBsRgkp5dmbG56SA0DmVtwrF5N3oPdI8t+Aw= github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/jackmordaunt/icns v0.0.0-20181231085925-4f16af745526/go.mod h1:UQkeMHVoNcyXYq9otUupF7/h/2tmHlhrS2zw7ZVvUqc= @@ -51,6 +57,8 @@ github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= github.com/nwaples/rardecode v1.0.0 h1:r7vGuS5akxOnR4JQSkko62RJ1ReCMXxQRPtxsiFMBOs= github.com/nwaples/rardecode v1.0.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= +github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= +github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/phpdave11/gofpdi v1.0.7/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI= github.com/pierrec/lz4 v2.2.6+incompatible h1:6aCX4/YZ9v8q69hTyiR7dNLnTA3fgtKHVVW5BCd5Znw= github.com/pierrec/lz4 v2.2.6+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= diff --git a/pkg/core/core.go b/pkg/core/core.go index 846a4e35..d308297d 100644 --- a/pkg/core/core.go +++ b/pkg/core/core.go @@ -276,8 +276,8 @@ func (comic *Comic) DownloadImages() (string, error) { if err != nil { return dir, err } - defer rsp.Body.Close() + // retrieve the image from the response content, _, err := comic.retrieveImageFromResponse(rsp) if err != nil { @@ -285,12 +285,13 @@ func (comic *Comic) DownloadImages() (string, error) { } imgFile, err := os.Create(fmt.Sprintf("%04d-image.%s", i, format)) - if err != nil { return dir, err } + defer imgFile.Close() - if _, err = io.Copy(imgFile, content); err != nil { + err = util.SaveImage(imgFile, content, format) + if err != nil { return dir, err } } diff --git a/pkg/detector/detector.go b/pkg/detector/detector.go index fc4980f2..8f948532 100644 --- a/pkg/detector/detector.go +++ b/pkg/detector/detector.go @@ -6,7 +6,7 @@ import ( log "github.com/sirupsen/logrus" ) -// DetectComic will look for the url source to check if a source is supported or not. +// DetectComic will look for the url source to check if a source is supported. func DetectComic(url string) (string, bool) { source, err := util.URLSource(url) diff --git a/pkg/sites/base.go b/pkg/sites/base.go index 66fa46d6..40355d66 100644 --- a/pkg/sites/base.go +++ b/pkg/sites/base.go @@ -9,6 +9,7 @@ var SupportedSites = []string{ "www.mangareader.net", "www.mangatown.com", "www.mangahere.cc", + "mangadex.org", } // BaseSite specifies an implementation of a Site which allows you diff --git a/pkg/sites/loader.go b/pkg/sites/loader.go index 265472bc..443515db 100644 --- a/pkg/sites/loader.go +++ b/pkg/sites/loader.go @@ -61,6 +61,8 @@ func LoadComicFromSource(source, url, country, format, imagesFormat string, all, siteSource = &Mangareader{} case "www.mangatown.com": siteSource = &Mangatown{} + case "mangadex.org": + siteSource = NewMangadex() default: err = fmt.Errorf("It was not possible to determine the source") return collection, err diff --git a/pkg/sites/mangadex.go b/pkg/sites/mangadex.go new file mode 100644 index 00000000..f0c4233a --- /dev/null +++ b/pkg/sites/mangadex.go @@ -0,0 +1,67 @@ +package sites + +import ( + "fmt" + + "github.com/Girbons/comics-downloader/pkg/core" + "github.com/Girbons/comics-downloader/pkg/util" + "github.com/bake/mangadex" +) + +type Mangadex struct { + Client *mangadex.Client +} + +// NewMangadex returns a Mangadex instance +func NewMangadex() *Mangadex { + return &Mangadex{ + Client: mangadex.New(), + } +} + +func (m *Mangadex) getLinks(url string) ([]string, error) { + var links []string + + parts := util.TrimAndSplitURL(url) + chapterID := parts[4] + + res, err := m.Client.Chapter(chapterID) + if err != nil { + return links, err + } + + links = res.Images() + + return links, nil +} + +func (m *Mangadex) RetrieveIssueLinks(url string, all, last bool) ([]string, error) { + return []string{url}, nil +} + +func (m *Mangadex) GetInfo(url string) (string, string) { + parts := util.TrimAndSplitURL(url) + chapterID := parts[4] + + chapterInfo, err := m.Client.Chapter(chapterID) + if err != nil { + return "", "" + } + + mangaInfo, _, err := m.Client.Manga(string(chapterInfo.MangaID)) + if err != nil { + return "", "" + } + + issueNumber := fmt.Sprintf("Vol %s Chapter %s, %s", chapterInfo.Volume, chapterInfo.Chapter, chapterInfo.Title) + + return mangaInfo.Title, issueNumber +} + +// Initialize loads links and metadata from mangadex +func (m *Mangadex) Initialize(comic *core.Comic) error { + links, err := m.getLinks(comic.URLSource) + comic.Links = links + + return err +} diff --git a/pkg/sites/mangadex_test.go b/pkg/sites/mangadex_test.go new file mode 100644 index 00000000..6e03c969 --- /dev/null +++ b/pkg/sites/mangadex_test.go @@ -0,0 +1,37 @@ +package sites + +import ( + "testing" + + "github.com/Girbons/comics-downloader/pkg/core" + "github.com/stretchr/testify/assert" +) + +func TestMangadexGetInfo(t *testing.T) { + md := NewMangadex() + + name, issueNumber := md.GetInfo("https://mangadex.org/chapter/155061/1") + assert.Equal(t, "Naruto", name) + assert.Equal(t, "Vol 60 Chapter 575, A Will of Stone", issueNumber) +} + +func TestMangadexSetup(t *testing.T) { + md := NewMangadex() + comic := new(core.Comic) + + comic.URLSource = "https://mangadex.org/chapter/155061/1" + + err := md.Initialize(comic) + + assert.Nil(t, err) + assert.Equal(t, 14, len(comic.Links)) +} + +func TestMangadexRetrieveIssueLinks(t *testing.T) { + md := NewMangadex() + + issues, err := md.RetrieveIssueLinks("https://mangadex.org/chapter/155061/1", false, false) + + assert.Nil(t, err) + assert.Equal(t, 1, len(issues)) +} diff --git a/pkg/util/util.go b/pkg/util/util.go index 6466bcfd..49d6602d 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -2,9 +2,13 @@ package util import ( "bytes" + "errors" "fmt" "image" + "image/gif" "image/jpeg" + "image/png" + "io" "net/url" "os" "path/filepath" @@ -73,6 +77,8 @@ func ImageType(mimeStr string) (tp string) { tp = "jpg" case "image/gif", "gif": tp = "gif" + case "img": + tp = "img" default: tp = "unknown" } @@ -150,3 +156,22 @@ func Parse(s string) string { return strings.Trim(replacer.Replace(s), " ") } + +// SaveImage saves an image from a given format +func SaveImage(w io.Writer, content io.Reader, format string) error { + img, _, err := image.Decode(content) + + switch strings.ToLower(format) { + case "img": + _, err = io.Copy(w, content) + return err + case "gif": + return gif.Encode(w, img, nil) + case "jpg", "jpeg": + return jpeg.Encode(w, img, nil) + case "png": + return png.Encode(w, img) + default: + return errors.New("format not found") + } +}