Permalink
Browse files

반디앤루니스 도서 보유 서점 조회 추가

[변경 사항]
- 반디앤루니스 도서 보유 서점 조회 추가
- 너무 많은 http 요청으로 인해 Composite 하는 과정에서 일부 비동기 처리
- 도서 정보(title, description)은 네이버 API를 이용(Book을 만듬)
- 도서 보유 서점 조회는 알라딘, 반디, 교보 모두 조회
    - 이때 List<BookStore>를 반환하도록 변경
- 도서가 없는 경우 NotFoundBookException을 throw, 에러 처리는 따로 구현해야됨

[성능]
- 너무 많은 http 요청으로 인해 성능이 많이 저하됨.
- 총 5번의 http 요청됨
    1. 네이버 도서 검색
    2. 반디앤루니스 도서 검색
    3. 반디앤루니스 도서 보유 서점 검색
    4. 알라딘 도서 보유 서점 검색
    5. 교보 도서 보유 서점 검색
- 모두 비동기로 처리한 경우 응답 : 약 2000 ms
- 1, 3, 4, 5를 일부 비동기로 처리한 경우 응답 : 500ms ~ 1000ms
- 그래도 성능은 좀 더 보완해야됨. (어디를 손대야할지..)

[개선 사항]
- 급하게 작성하는 바람에 테스트 코드 불충분
- 복잡한 html 구조 덕분에 더러워진 코드 리팩토링
- 성능을 위한 비동기 처리를 고려해볼것
- ControllerAdvice를 이용해 Exception 처리
  • Loading branch information...
woniper committed Jun 20, 2018
1 parent 6839097 commit 9b85967f3052e4ea31a0680a34e580f1d17b3390
Showing with 331 additions and 160 deletions.
  1. +1 −1 bookup-api/src/main/java/io/bookup/book/api/BookFindController.java
  2. +1 −1 bookup-api/src/main/java/io/bookup/book/api/representation/NaverBookResponseDto.java
  3. +40 −24 bookup-api/src/main/java/io/bookup/book/app/BookStoreCompositeAppService.java
  4. +5 −1 bookup-api/src/main/java/io/bookup/book/domain/Book.java
  5. +11 −0 bookup-api/src/main/java/io/bookup/book/domain/NotFoundBookException.java
  6. +10 −66 bookup-api/src/main/java/io/bookup/book/infra/crawler/AladinBookCrawler.java
  7. +125 −0 bookup-api/src/main/java/io/bookup/book/infra/crawler/BandinLunisBookCrawler.java
  8. +33 −0 bookup-api/src/main/java/io/bookup/book/infra/rest/BandinLunisBook.java
  9. +42 −0 bookup-api/src/main/java/io/bookup/book/infra/rest/BandinLunisRestTemplate.java
  10. +21 −8 bookup-api/src/main/java/io/bookup/book/infra/rest/KyoboBookRestTemplate.java
  11. +1 −1 bookup-api/src/main/java/io/bookup/book/{domain → infra/rest}/KyoboBookStore.java
  12. +1 −1 bookup-api/src/main/java/io/bookup/book/{domain → infra/rest}/NaverBook.java
  13. +1 −2 bookup-api/src/main/java/io/bookup/book/infra/rest/NaverBookRestTemplate.java
  14. +20 −0 bookup-api/src/main/java/io/bookup/common/utils/FutureUtils.java
  15. +4 −0 bookup-api/src/main/resources/application.yml
  16. +8 −13 bookup-api/src/test/java/io/bookup/book/infra/crawler/AladinBookCrawlerTests.java
  17. +0 −34 bookup-api/src/test/java/io/bookup/book/infra/crawler/BookStoreCompositeAppServiceTests.java
  18. +7 −8 bookup-api/src/test/java/io/bookup/book/infra/crawler/MockGenerator.java
@@ -3,7 +3,7 @@
import io.bookup.book.api.representation.NaverBookResponseDto;
import io.bookup.book.api.representation.NaverBookResponseDto.Item;
import io.bookup.book.app.BookStoreCompositeAppService;
import io.bookup.book.domain.NaverBook;
import io.bookup.book.infra.rest.NaverBook;
import io.bookup.book.infra.rest.NaverBookRestTemplate;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.GetMapping;
@@ -1,6 +1,6 @@
package io.bookup.book.api.representation;
import io.bookup.book.domain.NaverBook;
import io.bookup.book.infra.rest.NaverBook;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
@@ -2,14 +2,20 @@
import io.bookup.book.domain.Book;
import io.bookup.book.domain.BookStore;
import io.bookup.book.domain.KyoboBookStore;
import io.bookup.book.domain.NotFoundBookException;
import io.bookup.book.infra.BookFinder;
import io.bookup.book.infra.crawler.AladinBookCrawler;
import io.bookup.book.infra.crawler.BandinLunisBookCrawler;
import io.bookup.book.infra.rest.KyoboBookRestTemplate;
import io.bookup.book.infra.rest.KyoboProperties;
import io.bookup.book.infra.rest.NaverBook;
import io.bookup.book.infra.rest.NaverBookRestTemplate;
import io.bookup.common.utils.FutureUtils;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors;
import org.apache.commons.lang.StringUtils;
import org.springframework.stereotype.Service;
/**
@@ -18,35 +24,45 @@
@Service
public class BookStoreCompositeAppService {
private final AladinBookCrawler aladinBookCrawler;
private final KyoboBookRestTemplate kyoboBookRestTemplate;
private final KyoboProperties kyoboProperties;
private final NaverBookRestTemplate naverBookRestTemplate;
private final List<BookFinder<List<BookStore>>> bookStoreFinders;
public BookStoreCompositeAppService(AladinBookCrawler aladinBookCrawler,
KyoboBookRestTemplate kyoboBookRestTemplate,
KyoboProperties kyoboProperties) {
public BookStoreCompositeAppService(NaverBookRestTemplate naverBookRestTemplate,
AladinBookCrawler aladinBookCrawler,
BandinLunisBookCrawler bandinLunisBookCrawler,
KyoboBookRestTemplate kyoboBookRestTemplate) {
this.aladinBookCrawler = aladinBookCrawler;
this.kyoboBookRestTemplate = kyoboBookRestTemplate;
this.kyoboProperties = kyoboProperties;
this.naverBookRestTemplate = naverBookRestTemplate;
this.bookStoreFinders = Arrays.asList(
aladinBookCrawler,
bandinLunisBookCrawler,
kyoboBookRestTemplate
);
}
public Book getBook(String isbn) {
Book aladinBook = aladinBookCrawler.findByIsbn(isbn);
List<BookStore> bookStores = mapToBookStore(isbn, kyoboBookRestTemplate.findByIsbn(isbn));
aladinBook.merge(bookStores);
CompletableFuture<Book> bookFuture =
CompletableFuture.supplyAsync(() -> mapBook(naverBookRestTemplate.findByIsbn(isbn)))
.thenApplyAsync(x -> x.merge(getBookStores(isbn)));
return aladinBook;
return FutureUtils.getFutureItem(bookFuture)
.orElseThrow(() -> new NotFoundBookException(isbn));
}
private List<BookStore> mapToBookStore(String isbn, Optional<KyoboBookStore> kyoboBookStore) {
if (StringUtils.isEmpty(isbn) || !kyoboBookStore.isPresent())
return null;
private List<BookStore> getBookStores(String isbn) {
List<BookStore> bookStores = new ArrayList<>();
return kyoboBookStore.get().getItems().stream()
.filter(x -> x.getAmount() > 0)
.map(x -> new BookStore(x.getStoreName(), kyoboProperties.createUrl(x.getStoreId(), isbn)))
.collect(Collectors.toList());
bookStoreFinders.stream()
.map(x -> CompletableFuture.supplyAsync(() -> x.findByIsbn(isbn)))
.collect(Collectors.toList())
.forEach(x -> bookStores.addAll(FutureUtils.getFutureItem(x).orElse(Collections.emptyList())));
return bookStores;
}
private Book mapBook(NaverBook.Item item) {
return new Book(item.getTitle(), item.getDescription());
}
}
@@ -18,9 +18,13 @@
private String description;
private Collection<BookStore> bookStores = new ArrayList<>();
public Book(String title, String description, Collection<BookStore> bookStores) {
public Book(String title, String description) {
this.title = title;
this.description = description;
}
public Book(String title, String description, Collection<BookStore> bookStores) {
this(title, description);
this.bookStores = bookStores;
}
@@ -0,0 +1,11 @@
package io.bookup.book.domain;
/**
* @author woniper
*/
public class NotFoundBookException extends RuntimeException {
public NotFoundBookException(String message) {
super(message);
}
}
@@ -1,9 +1,9 @@
package io.bookup.book.infra.crawler;
import io.bookup.book.domain.Book;
import io.bookup.book.infra.BookFinder;
import io.bookup.book.domain.BookStore;
import java.util.Collection;
import io.bookup.book.infra.BookFinder;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
import org.jsoup.Jsoup;
@@ -18,13 +18,9 @@
* @author woniper
*/
@Component
public class AladinBookCrawler implements BookFinder<Book> {
public class AladinBookCrawler implements BookFinder<List<BookStore>> {
private final String HTML_CLASS_NAME_SS_BOOK_BOX = "ss_book_box";
private final String HTML_CLASS_NAME_SS_BOOK_LIST = "ss_book_list";
private final String HTML_TAG_NAME_LI = "li";
private final String HTML_CLASS_NAME_BO3 = "bo3";
private final String HTML_CLASS_NAME_SS_F_G2 = "ss_f_g2";
private final String HTML_CLASS_NAME_USED_SHOP_OFF_TEXT3 = "usedshop_off_text3";
private final String HTML_ATTRIBUTE_NAME_HREF = "href";
@@ -39,21 +35,19 @@
}
@Override
public Book findByIsbn(String isbn) {
public List<BookStore> findByIsbn(String isbn) {
Element bodyElement = getBodyElement(createUrl(isbn));
if (hasNotBook(bodyElement)) return null;
Elements bookElements = getBookElements(bodyElement);
return new Book(getTitle(bookElements), getDescription(bookElements), findBookStores(bodyElement));
return findBookStores(bodyElement);
}
private Collection<BookStore> findBookStores(Element element) {
if (Objects.isNull(element)) return null;
private List<BookStore> findBookStores(Element element) {
if (Objects.isNull(element)) return Collections.emptyList();
if (hasNotBook(element)) return Collections.emptyList();
Elements boxElements = element.getElementsByClass(HTML_CLASS_NAME_SS_BOOK_BOX);
if (CollectionUtils.isEmpty(boxElements)) return null;
if (CollectionUtils.isEmpty(boxElements)) return Collections.emptyList();
return getBookStoreElement(boxElements).stream()
.map(x -> new BookStore("알라딘 : " + x.text(), getBookStoreHref(x)))
@@ -74,56 +68,6 @@ private String getBookStoreHref(Element element) {
return element.attr(HTML_ATTRIBUTE_NAME_HREF);
}
private Elements getBookElements(Element element) {
if (Objects.isNull(element)) return null;
Elements boxElements = element.getElementsByClass(HTML_CLASS_NAME_SS_BOOK_BOX);
if (CollectionUtils.isEmpty(boxElements)) return null;
Elements listElements = boxElements.first().getElementsByClass(HTML_CLASS_NAME_SS_BOOK_LIST);
if (CollectionUtils.isEmpty(listElements)) return null;
Elements liElements = listElements.first().getElementsByTag(HTML_TAG_NAME_LI);
if (CollectionUtils.isEmpty(liElements)) return null;
return liElements;
}
private String getTitle(Elements bookElements) {
if (Objects.isNull(bookElements)) return null;
Element bookElement = bookElements.first();
if (Objects.isNull(bookElement)) return null;
Elements titleElements = bookElement.getElementsByClass(HTML_CLASS_NAME_BO3);
if (Objects.isNull(titleElements)) return null;
Element titleElement = titleElements.first();
if (Objects.isNull(titleElement)) return null;
String title = titleElement.text();
Elements subTitleElements = bookElement.getElementsByClass(HTML_CLASS_NAME_SS_F_G2);
if (!CollectionUtils.isEmpty(subTitleElements)) {
title += subTitleElements.first().text();
}
return title;
}
private String getDescription(Elements elements) {
if (CollectionUtils.isEmpty(elements)) return null;
return elements.last().text();
}
private Elements getBookStoreElement(Elements elements) {
Element bookElement = elements.first();
return bookElement.getElementsByClass(HTML_CLASS_NAME_USED_SHOP_OFF_TEXT3);
@@ -0,0 +1,125 @@
package io.bookup.book.infra.crawler;
import io.bookup.book.domain.BookStore;
import io.bookup.book.infra.BookFinder;
import io.bookup.book.infra.rest.BandinLunisBook;
import io.bookup.book.infra.rest.BandinLunisRestTemplate;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import org.springframework.util.StringUtils;
import org.springframework.web.client.RestTemplate;
/**
* @author woniper
*/
@Component
public class BandinLunisBookCrawler implements BookFinder<List<BookStore>> {
private final String HTML_CLASS_NAME_BOOK_MT3 = "mt3";
private final String HTML_TAG_NAME_TBODY = "tbody";
private final String HTML_TAG_NAME_TR = "tr";
private final String HTML_TAG_NAME_TH = "th";
private final String HTML_TAG_NAME_TD = "td";
private final String HTML_TAG_NAME_A = "a";
private final String HTML_ATTR_NAME_HREF = "href";
private final String url;
private final String hrefUrl;
private final RestTemplate restTemplate;
private final BandinLunisRestTemplate bandinLunisRestTemplate;
public BandinLunisBookCrawler(@Value("${bookup.crawler.bandi.storeUrl}") String url,
@Value("${bookup.crawler.bandi.hrefUrl}") String hrefUrl,
RestTemplate restTemplate,
BandinLunisRestTemplate bandinLunisRestTemplate) {
this.url = url;
this.hrefUrl = hrefUrl;
this.restTemplate = restTemplate;
this.bandinLunisRestTemplate = bandinLunisRestTemplate;
}
@Override
public List<BookStore> findByIsbn(String isbn) {
String productId = bandinLunisRestTemplate.findByIsbn(isbn)
.map(BandinLunisBook::getItems)
.orElse(Collections.emptyList()).stream()
.findFirst()
.map(BandinLunisBook.Item::getProductId)
.orElse(null);
if (!StringUtils.hasText(productId)) return Collections.emptyList();
Element tbodyElement = getTBodyElement(productId);
return findBookStores(productId, tbodyElement);
}
private List<BookStore> findBookStores(String productId, Element tbodyElement) {
if (Objects.isNull(tbodyElement)) return Collections.emptyList();
List<BookStore> bookStores = new ArrayList<>();
Elements trs = tbodyElement.getElementsByTag(HTML_TAG_NAME_TR);
for (int trIndex = 0; trIndex < trs.size() - 1; trIndex++) {
Elements ths = trs.get(trIndex).getElementsByTag(HTML_TAG_NAME_TH);
Elements tds = trs.get(trIndex + 1).getElementsByTag(HTML_TAG_NAME_TD);
if (ths.size() == tds.size()) {
for (int thIndex = 0; thIndex < ths.size(); thIndex++) {
String storeName = ths.get(thIndex).text().trim();
Elements aElements = tds.get(thIndex).getElementsByTag(HTML_TAG_NAME_A);
if (!CollectionUtils.isEmpty(aElements)) {
int amount = Integer.parseInt(aElements.first().text().trim());
String hrefAttribute = aElements.attr(HTML_ATTR_NAME_HREF);
String storeId = hrefAttribute.substring(
hrefAttribute.indexOf("'"),
hrefAttribute.lastIndexOf("'"))
.replaceAll("'", "");
if (StringUtils.hasText(storeName) && amount > 0) {
bookStores.add(new BookStore(
String.format("반디앤루니스 : %s", storeName),
String.format(hrefUrl, productId, storeId)));
}
}
}
}
}
return bookStores;
}
private Element getTBodyElement(String productId) {
Element bodyElement = getBodyElement(String.format(url, productId));
if (hasNotBook(bodyElement)) return null;
Elements tbodyElements = bodyElement.getElementsByTag(HTML_TAG_NAME_TBODY);
if (CollectionUtils.isEmpty(tbodyElements)) return null;
return tbodyElements.first();
}
private boolean hasNotBook(Element element) {
if (Objects.isNull(element)) return false;
return !element.getAllElements().hasClass(HTML_CLASS_NAME_BOOK_MT3);
}
private Element getBodyElement(String url) {
String html = restTemplate.getForObject(url, String.class);
return Jsoup.parse(html).body();
}
}
Oops, something went wrong.

0 comments on commit 9b85967

Please sign in to comment.