Skip to content

Commit

Permalink
Remove isbSearch and digit-eyes fetchers due to too restrictive usage…
Browse files Browse the repository at this point in the history 
… (calls to low)
  • Loading branch information
@jfyuen
jfyuen committed May 26, 2016
1 parent 7f22dd3 commit 79aec01
Show file tree
Hide file tree
Showing 2 changed files with 0 additions and 117 deletions.
97 changes: 0 additions & 97 deletions fetch.go
View file
Expand Up @@ -50,9 +50,6 @@ var UpcItemDbFetcher, _ = NewFetchableURL("http://www.upcitemdb.com/upc/%s", "UP
// OpenFoodFactsFetcher for openfoodfacts.org (using json api)
var OpenFoodFactsFetcher, _ = NewFetchableURL("http://fr.openfoodfacts.org/api/v0/produit/%s.json", "OpenFoodFacts", openFoodFactsParser{})

// IsbnSearchFetcher for isbnsearch.org (using json api)
var IsbnSearchFetcher, _ = NewFetchableURL("http://www.isbnsearch.org/isbn/%s", "ISBNSearch", isbnSearchParser{})

// IGalerieFetcher for some unknown website: http://90.80.54.225/?img=161277&images=1859
var IGalerieFetcher, _ = NewFetchableURL("http://90.80.54.225/?search=%s", "90.80.54.225", iGalerieParser{baseURL: "http://90.80.54.225/"})

Expand All @@ -65,9 +62,6 @@ var MisterPharmaWebFetcher, _ = NewFetchableURL("http://www.misterpharmaweb.com/
// MedisparFetcher for meddispar.fr
var MedisparFetcher, _ = NewFetchableURL("http://www.meddispar.fr/content/search?search_by_name=&search_by_cip=%s", "Medispar", medisparParser{baseURL: "http://www.meddispar.fr"})

// DigitEyesFetcher for digit-eyes.com
var DigitEyesFetcher, _ = NewFetchableURL("http://www.digit-eyes.com/upcCode/%s.html", "Digit-Eyes", digitEyesParser{})

// PicardFetcher for picard.fr
var PicardFetcher, _ = NewFetchableURL("http://www.picard.fr/recherche?q=%s", "Picard", picardParser{baseURL: "http://www.picard.fr"})

Expand Down Expand Up @@ -343,56 +337,6 @@ func (f openFoodFactsParser) ParseBody(body []byte) (Product, error) {
return p, nil
}

type isbnSearchParser struct{}

func (f isbnSearchParser) ParseBody(b []byte) (Product, error) {
doc, err := html.Parse(bytes.NewReader(b))
p := Product{}
if err != nil {
return p, err
}
var fn func(*html.Node)
fn = func(n *html.Node) {
for c := n.FirstChild; c != nil; c = c.NextSibling {
// Looking for <div class="bookinfo"><h2>$PRODUCT_NAME</h2></div>
if c.Type == html.ElementNode {
switch c.Data {
case "div":
if p.Name != "" {
return
}
if len(c.Attr) == 1 {
classAttr := c.Attr[0]
if classAttr.Val == "bookinfo" {
txt := c.FirstChild.NextSibling.FirstChild
if txt.Type == html.TextNode {
p.Name = txt.Data
return
}
}
}
case "img":
if p.ImageURL != "" {
return
}
for _, attr := range c.Attr {
if attr.Key == "src" {
p.ImageURL = attr.Val
return
}
}
}
fn(c)
}
}
}
fn(doc)
if p.Name == "" {
return p, errNotFound
}
return p, nil
}

// amazonItemSearchResponse is the base xml response, only keep needed fields (maybe more will be added later) in a "flat" struct.
type amazonItemSearchResponse struct {
TotalResults uint `xml:"Items>TotalResults"`
Expand Down Expand Up @@ -727,43 +671,6 @@ func (f medisparParser) ParseBody(b []byte) (Product, error) {
return p, nil
}

type digitEyesParser struct{}

func (f digitEyesParser) ParseBody(b []byte) (Product, error) {
charsetReader := charmap.ISO8859_1.NewDecoder().Reader(bytes.NewReader(b))
doc, err := html.Parse(charsetReader) // charset is ISO-8859-1
p := Product{}
if err != nil {
return p, err
}
var fn func(*html.Node)
fn = func(n *html.Node) {
for c := n.FirstChild; c != nil; c = c.NextSibling {
if c.Type == html.ElementNode {
if c.Data == "img" {
imageURL := ""
for _, attr := range c.Attr {
if attr.Key == "src" {
imageURL = attr.Val
}
if attr.Key == "alt" && strings.Contains(attr.Val, "image of ") {
p.Name = strings.Replace(attr.Val, "image of ", "", -1)
p.ImageURL = imageURL
return
}
}
}
fn(c)
}
}
}
fn(doc)
if p.Name == "" {
return p, errNotFound
}
return p, nil
}

type picardParser struct {
baseURL string
}
Expand Down Expand Up @@ -825,25 +732,21 @@ type DefaultFetcher struct {
// Currently supported websites:
// - upcitemdb
// - openfoodfacts
// - isbnsearch
// - iGalerie (some random IP on internet)
// - amazon (if credentials are provided)
// - StarryMart
// - MisterPharmaWeb
// - Meddispar
// - Digit-Eyes
// - more fetchers are provided as arguments (local database, ...)
// TODO: should return a warning, or info, not an error.
func NewDefaultFetcher(otherFetchers ...Fetcher) (DefaultFetcher, error) {
fetchers := []Fetcher{
UpcItemDbFetcher,
OpenFoodFactsFetcher,
IsbnSearchFetcher,
IGalerieFetcher,
StarrymartFetcher,
MisterPharmaWebFetcher,
MedisparFetcher,
DigitEyesFetcher,
PicardFetcher,
}
for _, f := range otherFetchers {
Expand Down
20 changes: 0 additions & 20 deletions fetch_test.go
View file
Expand Up @@ -82,16 +82,6 @@ func TestDefaultFetchers(t *testing.T) {
t.Errorf("Some attributes are invalid for: %v", p)
}

p, err = IsbnSearchFetcher.Fetch("9782501104265", blacklistDB)
if err != nil {
t.Error(err)
} else if p.Name != "le rugby c'est pas sorcier" || p.EAN != "9782501104265" ||
p.URL != "http://www.isbnsearch.org/isbn/9782501104265" ||
p.WebsiteURL != "http://www.isbnsearch.org/isbn/9782501104265" ||
p.ImageURL != "http://ecx.images-amazon.com/images/I/51V4iimUfML._SL194_.jpg" {
t.Errorf("Some attributes are invalid for: %v", p)
}

p, err = IGalerieFetcher.Fetch("8714789941011", blacklistDB)
if err != nil {
t.Error(err)
Expand Down Expand Up @@ -142,16 +132,6 @@ func TestDefaultFetchers(t *testing.T) {
t.Errorf("Some attributes are invalid for: %v", p)
}

p, err = DigitEyesFetcher.Fetch("3350033006737", blacklistDB)
if err != nil {
t.Error(err)
} else if p.Name != "Atomiseur D'eau Minerale" || p.EAN != "3350033006737" ||
p.URL != "http://www.digit-eyes.com/upcCode/3350033006737.html" ||
p.WebsiteURL != "http://www.digit-eyes.com/upcCode/3350033006737.html" ||
p.ImageURL != "https://www.digit-eyes.com/thumbs/233/033/3350033006737.jpg" {
t.Errorf("Some attributes are invalid for: %v", p)
}

p, err = PicardFetcher.Fetch("3270160891382", blacklistDB)
if err != nil {
t.Error(err)
Expand Down

0 comments on commit 79aec01

Please sign in to comment.