feat(m6): support 6play platform

Backend/src/main/java/lan/dk/podcastserver/business/update/UpdatePodcastBusiness.java

@@ -140,16 +140,13 @@ private Set<Item> attachNewItemsToPodcast(Podcast podcast, Set<Item> items, Pred
         Set<Item> itemsToAdd = items
                 .filter(filter)
                 .map(item -> item.setPodcast(podcast))
-                .filter(item -> validator.validate(item).isEmpty())
-                .peek(item -> log.debug("Add Item {} to Podcast {}", item.getTitle(), podcast.getTitle()));
+                .filter(item -> validator.validate(item).isEmpty());
 
         if (itemsToAdd.isEmpty()) {
             return itemsToAdd;
         }
 
-        itemsToAdd
-            .peek(podcast::add)
-            .forEach(itemRepository::save);
+        itemsToAdd.peek(podcast::add).forEach(itemRepository::save);
 
         podcastBusiness.save(podcast.lastUpdateToNow());
 

Backend/src/main/java/lan/dk/podcastserver/manager/worker/downloader/SixPlayDownloader.java

@@ -0,0 +1,89 @@
+package lan.dk.podcastserver.manager.worker.downloader;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.jayway.jsonpath.DocumentContext;
+import com.jayway.jsonpath.TypeRef;
+import javaslang.collection.HashSet;
+import javaslang.collection.Set;
+import javaslang.control.Option;
+import lan.dk.podcastserver.entity.Item;
+import lan.dk.podcastserver.repository.ItemRepository;
+import lan.dk.podcastserver.repository.PodcastRepository;
+import lan.dk.podcastserver.service.*;
+import lan.dk.podcastserver.service.properties.PodcastServerParameters;
+import lombok.Getter;
+import lombok.Setter;
+import org.apache.commons.lang3.StringUtils;
+import org.jsoup.nodes.Element;
+import org.jsoup.select.Elements;
+import org.springframework.messaging.simp.SimpMessagingTemplate;
+
+import static java.util.Objects.nonNull;
+
+/**
+ * Created by kevin on 22/03/2017 for Podcast Server
+ */
+public class SixPlayDownloader extends M3U8Downloader {
+
+    private static final TypeRef<Set<M6PlayItem>> TYPE_ITEMS = new TypeRef<Set<M6PlayItem>>() {};
+
+    private final HtmlService htmlService;
+    private final JsonService jsonService;
+
+    String url = null;
+
+    public SixPlayDownloader(ItemRepository itemRepository, PodcastRepository podcastRepository, PodcastServerParameters podcastServerParameters, SimpMessagingTemplate template, MimeTypeService mimeTypeService, UrlService urlService, M3U8Service m3U8Service, FfmpegService ffmpegService, ProcessService processService, HtmlService htmlService, JsonService jsonService) {
+        super(itemRepository, podcastRepository, podcastServerParameters, template, mimeTypeService, urlService, m3U8Service, ffmpegService, processService);
+        this.htmlService = htmlService;
+        this.jsonService = jsonService;
+    }
+
+    @Override
+    public String getItemUrl(Item item) {
+
+        if (nonNull(this.item) && !this.item.equals(item)) {
+            return item.getUrl();
+        }
+
+        if (nonNull(url)) {
+            return url;
+        }
+
+        url = htmlService.get(item.getUrl())
+                .map(d -> this.extractUrl(d.select("script")))
+                .getOrElseThrow(() -> new RuntimeException("Url not found for " + item.getUrl()));
+
+        return url;
+    }
+
+    private String extractUrl(Elements script) {
+
+        url = extractJson(script)
+                .map(JsonService.to("mainStoreState.video.currentVideo.clips[*].assets[*]", TYPE_ITEMS))
+                .getOrElse(HashSet.empty())
+                .filter(i -> "usp_hls_h264".equals(i.getType()))
+                .map(M6PlayItem::getFull_physical_path)
+                .getOrElseThrow(() -> new RuntimeException("Stream \"usp_hls_h264\" not found for item " + this.item.getTitle()));
+
+        return url;
+    }
+
+    private Option<DocumentContext> extractJson(Elements elements) {
+        return HashSet.ofAll(elements)
+                .find(s -> s.html().contains("root.__6play"))
+                .map(Element::html)
+                .map(s -> StringUtils.substringBetween(s, "root.__6play = ", "}(this));"))
+                .map(jsonService::parse);
+    }
+
+    @Override
+    public Integer compatibility(String url) {
+        return nonNull(url) && url.startsWith("http://www.6play.fr/") ? 1 : Integer.MAX_VALUE;
+    }
+
+    @JsonIgnoreProperties(ignoreUnknown = true)
+    public static class M6PlayItem {
+        @Setter @Getter private String full_physical_path;
+        @Setter @Getter private String type;
+    }
+}

Backend/src/main/java/lan/dk/podcastserver/manager/worker/finder/SixPlayFinder.java

@@ -0,0 +1,76 @@
+package lan.dk.podcastserver.manager.worker.finder;
+
+import javaslang.collection.HashSet;
+import lan.dk.podcastserver.entity.Cover;
+import lan.dk.podcastserver.entity.Podcast;
+import lan.dk.podcastserver.service.HtmlService;
+import lan.dk.podcastserver.service.ImageService;
+import lan.dk.podcastserver.service.JsonService;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import net.minidev.json.JSONArray;
+import org.apache.commons.lang3.StringUtils;
+import org.hibernate.validator.constraints.NotEmpty;
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+import org.jsoup.select.Elements;
+import org.springframework.stereotype.Service;
+
+import static java.util.Objects.nonNull;
+
+/**
+ * Created by kevin on 20/12/2016 for Podcast Server
+ */
+@Slf4j
+@Service
+@RequiredArgsConstructor
+public class SixPlayFinder implements Finder {
+
+    private final HtmlService htmlService;
+    private final ImageService imageService;
+    private final JsonService jsonService;
+
+    @Override
+    public Podcast find(String url) {
+        return htmlService.get(url)
+                .map(this::htmlToPodcast)
+                .getOrElse(Podcast.DEFAULT_PODCAST);
+    }
+
+    private Podcast htmlToPodcast(Document document) {
+        return Podcast.builder()
+                    .title(document.select("h1.tile-section__title").text())
+                    .url(document.select("link[rel=canonical]").attr("href"))
+                    .description(getDescription(document.select("script")))
+                    .cover(getCover(document.select("div.header-image__image").attr("style")))
+                    .type("SixPlay")
+                .build();
+    }
+
+    private String getDescription(Elements script) {
+        return HashSet
+                .ofAll(script)
+                .find(s -> s.html().contains("root.__6play"))
+                .map(Element::html)
+                .map(s -> StringUtils.substringBetween(s, "root.__6play = ", "}(this));"))
+                .map(jsonService::parse)
+                .map(d -> (JSONArray) d.read("context.dispatcher.stores.ProgramStore.programs.*.description"))
+                .flatMap(r -> HashSet.ofAll(r).headOption())
+                .map(Object::toString)
+                .getOrElse(() -> null);
+    }
+
+    private Cover getCover(String style) {
+        return HashSet
+            .of(style.split(";"))
+            .find(s -> s.contains("background-image"))
+            .map(s -> StringUtils.substringBetween(s, "(", ")"))
+            .map(imageService::getCoverFromURL)
+            .getOrElse(Cover.DEFAULT_COVER);
+    }
+
+    @Override
+    public Integer compatibility(@NotEmpty String url) {
+        return nonNull(url) && url.contains("www.6play.fr") ? 1 : Integer.MAX_VALUE;
+    }
+}

Backend/src/main/java/lan/dk/podcastserver/manager/worker/updater/AbstractUpdater.java

@@ -34,6 +34,7 @@ public Tuple3<Podcast, Set<Item>, Predicate<Item>> update(Podcast podcast) {
                 .filter(signature -> !StringUtils.equals(signature, podcast.getSignature()))
                 .andThen(podcast::setSignature)
                 .map(s -> Tuple.of(podcast, getItems(podcast), notIn(podcast)))
+                .onFailure(e -> log.error("Error during update : {}", e))
                 .getOrElse(NO_MODIFICATION_TUPLE);
     }
 

Backend/src/main/java/lan/dk/podcastserver/manager/worker/updater/SixPlayUpdater.java

@@ -0,0 +1,176 @@
+package lan.dk.podcastserver.manager.worker.updater;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.jayway.jsonpath.DocumentContext;
+import com.jayway.jsonpath.TypeRef;
+import javaslang.Value;
+import javaslang.collection.HashMap;
+import javaslang.collection.HashSet;
+import javaslang.collection.Set;
+import javaslang.control.Option;
+import lan.dk.podcastserver.entity.Item;
+import lan.dk.podcastserver.entity.Podcast;
+import lan.dk.podcastserver.service.HtmlService;
+import lan.dk.podcastserver.service.ImageService;
+import lan.dk.podcastserver.service.JsonService;
+import lan.dk.podcastserver.service.SignatureService;
+import lan.dk.podcastserver.service.properties.PodcastServerParameters;
+import lombok.Getter;
+import lombok.Setter;
+import lombok.ToString;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.jsoup.nodes.Element;
+import org.jsoup.select.Elements;
+import org.springframework.stereotype.Component;
+
+import javax.validation.Validator;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.Locale;
+
+import static java.util.Objects.nonNull;
+
+/**
+ * Created by kevin on 20/12/2016 for Podcast Server
+ */
+@Slf4j
+@Component("SixPlayUpdater")
+public class SixPlayUpdater extends AbstractUpdater {
+
+    private static final DateTimeFormatter DATE_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss", Locale.ENGLISH);
+    private static final TypeRef<Set<SixPlayItem>> TYPE_ITEMS = new TypeRef<Set<SixPlayItem>>(){};
+    private static final TypeRef<HashMap<String, Object>> TYPE_KEYS = new TypeRef<HashMap<String, Object>>(){};
+    private static final TypeRef<Set<Integer>> TYPE_IDS = new TypeRef<Set<Integer>>(){};
+
+    private static final String VIDEOS_SELECTOR = "mainStoreState.video.programVideosBySubCategory.%d.%d";
+    private static final String URL_TEMPLATE_PODCAST = "http://www.6play.fr/%s-p_%d/";
+    private static final String SUB_CAT_SELECTOR = "context.dispatcher.stores.ProgramStore.subcats.%d[*].id";
+    private static final String PROGRAM_CODE_SELECTOR = "context.dispatcher.stores.ProgramStore.programs.%d.code";
+    private static final String PROGRAM_ID_SELECTOR = "context.dispatcher.stores.ProgramStore.programs";
+
+    private final HtmlService htmlService;
+    private final JsonService jsonService;
+    private final ImageService imageService;
+
+    protected SixPlayUpdater(PodcastServerParameters podcastServerParameters, SignatureService signatureService, Validator validator, HtmlService htmlService, JsonService jsonService, ImageService imageService) {
+        super(podcastServerParameters, signatureService, validator);
+        this.htmlService = htmlService;
+        this.jsonService = jsonService;
+        this.imageService = imageService;
+    }
+
+    @Override
+    public Set<Item> getItems(Podcast podcast) {
+        return htmlService.get(podcast.getUrl())
+                .map(d -> this.extractItems(d.select("script")))
+                .getOrElse(HashSet::empty);
+    }
+
+    private Set<Item> extractItems(Elements script) {
+        Option<DocumentContext> root6Play = extractJson(script);
+
+        Integer programId = root6Play
+                .map(JsonService.to(PROGRAM_ID_SELECTOR, TYPE_KEYS))
+                .map(HashMap::keySet)
+                .flatMap(Value::getOption)
+                .map(Integer::valueOf)
+                .getOrElseThrow(() -> new RuntimeException("programId not found in root.__6play"));
+
+        Set<Integer> subCatIds = root6Play
+                .map(JsonService.to(String.format(SUB_CAT_SELECTOR, programId), TYPE_IDS))
+                .getOrElseThrow(() -> new RuntimeException("subcatId not found in root.__6play"));
+
+        String programCode = root6Play
+                .map(JsonService.to(String.format(PROGRAM_CODE_SELECTOR, programId), String.class))
+                .getOrElseThrow(() -> new RuntimeException("programCode not found in root.__6play"));
+
+        String basePath = String.format(URL_TEMPLATE_PODCAST, programCode, programId);
+
+        return subCatIds
+                .flatMap(v -> root6Play.map(JsonService.to(String.format(VIDEOS_SELECTOR, programId, v), TYPE_ITEMS)))
+                .map(c -> c.map(s -> this.convertToItem(s, basePath)))
+                .getOrElseThrow(() -> new RuntimeException("Error during transformation of json to items"));
+    }
+
+    private Item convertToItem(SixPlayItem i, String basePath) {
+        return Item.builder()
+                    .title(i.getTitle())
+                    .pubDate(i.getLastDiffusion())
+                    .length(i.getDuration())
+                    .url(i.url(basePath))
+                    .description(i.description)
+                    .cover(imageService.getCoverFromURL(i.cover()))
+                .build();
+    }
+
+    private Option<DocumentContext> extractJson(Elements elements) {
+        return HashSet.ofAll(elements)
+                .find(s -> s.html().contains("root.__6play"))
+                .map(Element::html)
+                .map(s -> StringUtils.substringBetween(s, "root.__6play = ", "}(this));"))
+                .map(jsonService::parse);
+    }
+
+    @Override
+    public String signatureOf(Podcast podcast) {
+        return "foo";
+    }
+
+    @Override
+    public Type type() {
+        return new Type("SixPlay", "6Play");
+    }
+
+    @Override
+    public Integer compatibility(String url) {
+        return nonNull(url) && url.startsWith("http://www.6play.fr/") ? 1 : Integer.MAX_VALUE;
+    }
+
+    @ToString
+    @JsonIgnoreProperties(ignoreUnknown = true)
+    private static class SixPlayItem{
+
+        @Getter @Setter Set<Image> images;
+        @Getter @Setter String code;
+        @Getter @Setter String description;
+        @Getter @Setter String title;
+        @Setter String lastDiffusion; /* 2016-12-18 11:20:00 */
+        @Getter @Setter Long duration;
+        @Getter @Setter String id;
+
+        ZonedDateTime getLastDiffusion() {
+            return Option.of(lastDiffusion)
+                    .map(l -> ZonedDateTime.of(LocalDateTime.parse(l, DATE_FORMATTER), ZoneId.of("Europe/Paris")))
+                    .getOrElse(() -> null);
+        }
+
+        String url(String basePath) {
+            return basePath + code + "-c_" + StringUtils.substringAfter(id, "_");
+        }
+
+        String cover() {
+            return images.headOption()
+                    .map(Image::url)
+                    .getOrElse(() -> null);
+        }
+
+        @JsonIgnoreProperties(ignoreUnknown = true)
+        private static class Image {
+
+            private static final String domain = "https://images.6play.fr";
+            private static final String path = "/v1/images/%s/raw?width=600&height=336&fit=max&quality=60&format=jpeg&interlace=1";
+            private static final String salt = "54b55408a530954b553ff79e98";
+
+            @Getter @Setter Integer external_key;
+
+            public String url() {
+                String path = String.format(Image.path, external_key);
+                return domain + path + "&hash=" + DigestUtils.sha1Hex(path + salt);
+            }
+        }
+    }
+}