Embed images to output document

Add ability to download images for articles
and embed to the document.
Allow to render content with footer.
Image paragraph would have possible to render base64 content in the footer.
Because image is a big file, the content would be splited and unreadable.
Using footer for references should the problem of read.
Remove empy lines between assets
Add footer only when paragrpah has one.
Support inplace images only PNG
Remove function to download images in assets folder.
Install only production dependicies for docker build.

Dockerfile

@@ -2,7 +2,7 @@ FROM crystallang/crystal:0.35.1 as builder
 
 WORKDIR /app
 COPY ./shard.yml /app/
-RUN shards install
+RUN shards install --production -v
 
 COPY . /app/
 RUN shards build --production -v

shard.lock

@@ -0,0 +1,6 @@
+version: 2.0
+shards:
+  webmock:
+    git: https://github.com/manastech/webmock.cr.git
+    version: 0.13.0+git.commit.bb3eab30f6c7d1fdc0a7ff14cd136d68e860d1a7
+

shard.yml

@@ -14,3 +14,8 @@ targets:
 crystal: 0.35.1
 
 license: LGPL-3.0
+
+development_dependencies:
+  webmock:
+    github: manastech/webmock.cr
+    branch: master

spec/medium/post/paragraph_spec.cr

@@ -11,82 +11,85 @@ describe Medium::Post::Paragraph do
   describe "#to_md" do
     it "should render header" do
       subject = Medium::Post::Paragraph.from_json(%{{"name": "d2a9", "type": 3, "text": "Modify binary files with VIM", "markups": []}})
-      subject.to_md.should eq("# Modify binary files with VIM")
+      subject.to_md[0].should eq("# Modify binary files with VIM")
     end
 
     it "renders blockquotes" do
       subject = Medium::Post::Paragraph.from_json(%{{"name": "78ee", "type": 6, "text": "TLDR text", "markups": []}})
-      subject.to_md.should eq("> TLDR text")
+      subject.to_md[0].should eq("> TLDR text")
     end
 
     it "renders blockquotes second style" do
       subject = Medium::Post::Paragraph.from_json(%{{"name": "78ee", "type": 7, "text": "TLDR text", "markups": []}})
-      subject.to_md.should eq(">> TLDR text")
+      subject.to_md[0].should eq(">> TLDR text")
     end
 
     it "renders images" do
       subject = Medium::Post::Paragraph.from_json(%{{"name": "78ee", "type": 4, "text": "Photo", "layout": 3, "metadata":{"id":"0*FbFs8aNmqNLKw4BM"}, "markups": []}})
-      subject.to_md.should eq("![Photo](./assets/0*FbFs8aNmqNLKw4BM)")
+      content, assets = subject.to_md
+      content.should eq("![Photo][image_ref_MCpGYkZzOGFObXFOTEt3NEJN]")
+      assets.should match(/^\[image_ref_MCpGYkZzOGFObXFOTEt3NEJN\]:/)
     end
 
     it "render number list" do
       subject = Medium::Post::Paragraph.from_json(%{{"name": "d2a9", "type": 10, "text": "it should be cross distributive solution", "markups": []}})
-      subject.to_md.should eq("1. it should be cross distributive solution")
+      subject.to_md[0].should eq("1. it should be cross distributive solution")
     end
 
     it "render unordered list" do
       subject = Medium::Post::Paragraph.from_json(%{{"name": "d2a9", "type": 9, "text": "it should be cross distributive solution", "markups": []}})
-      subject.to_md.should eq("* it should be cross distributive solution")
+      subject.to_md[0].should eq("* it should be cross distributive solution")
     end
 
     it "render text" do
       subject = Medium::Post::Paragraph.from_json(%{{"name": "d2a9", "type": 1, "text": "it should be cross distributive solution", "markups": []}})
-      subject.to_md.should eq("it should be cross distributive solution")
+      subject.to_md[0].should eq("it should be cross distributive solution")
     end
 
     it "render title" do
       subject = Medium::Post::Paragraph.from_json(%{{"name": "d2a9", "type": 2, "text": "render title with picture", "markups": [], "alignment": 2}})
-      subject.to_md.should eq("# render title with picture")
+      subject.to_md[0].should eq("# render title with picture")
     end
 
     it "render code block" do
       subject = Medium::Post::Paragraph.from_json(%{{"name": "d2a9", "type": 8, "text": "puts hello", "markups": []}})
-      subject.to_md.should eq("```\nputs hello\n```")
+      subject.to_md[0].should eq("```\nputs hello\n```")
     end
 
     it "render small header" do
       subject = Medium::Post::Paragraph.from_json(%{{"name": "d2a9", "type": 13, "text": "Help", "markups": []}})
-      subject.to_md.should eq("### Help")
+      subject.to_md[0].should eq("### Help")
     end
 
     it "render link references" do
       subject = Medium::Post::Paragraph.from_json(%{{"name": "d2a9", "type": 14, "text": "Socket::Addrinfo", "markups": [], "mixtapeMetadata": {"mediaResourceId": "7f3accd276b8655a927e5d50f276d49d","href":"https://crystal-lang.org/api/0.31.0/Socket/Addrinfo.html"}}})
-      subject.to_md.should eq("https://crystal-lang.org/api/0.31.0/Socket/Addrinfo.html")
+      subject.to_md[0].should eq("https://crystal-lang.org/api/0.31.0/Socket/Addrinfo.html")
     end
 
     it "render image with link" do
       subject = Medium::Post::Paragraph.from_json(%{{"name": "d2a9", "type": 4, "text": "", "layout": 3, "href": "https://asciinema.org/a/5diw0wwk6vbbovnqrk5sh1soy", "markups": [], "metadata":{"id": "1*NVLl4oVmMQtumKL-DVV1rA.png"}}})
-      subject.to_md.should eq("[![](./assets/1*NVLl4oVmMQtumKL-DVV1rA.png)](https://asciinema.org/a/5diw0wwk6vbbovnqrk5sh1soy)")
+      content, assets = subject.to_md
+      content.should eq("[![][image_ref_MSpOVkxsNG9WbU1RdHVtS0wtRFZWMXJBLnBuZw==]](https://asciinema.org/a/5diw0wwk6vbbovnqrk5sh1soy)")
     end
 
     it "render iframe inline" do
       subject = Medium::Post::Paragraph.from_json(%{{"name": "d2a9", "type": 11, "text": "", "markups": [], "iframe":{"mediaResourceId": "e7722acf2886364130e03d2c7ad29de7"}}})
-      subject.to_md.should eq(%{<iframe src="./assets/e7722acf2886364130e03d2c7ad29de7.html"></iframe>})
+      subject.to_md[0].should eq(%{<iframe src="./assets/e7722acf2886364130e03d2c7ad29de7.html"></iframe>})
     end
 
     it "renders inline code block" do
       subject = Medium::Post::Paragraph.from_json(%{{"name": "d2a9", "type": 6, "text": "TL;DR xxd ./bin/app | vim — and :%!xxd -r > ./bin/new_app", "markups": [{"type": 10,"start": 6,"end": 27},{"type": 10,"start": 32,"end": 57}]}})
-      subject.to_md.should eq("> TL;DR `xxd ./bin/app | vim —` and `:%!xxd -r > ./bin/new_app`")
+      subject.to_md[0].should eq("> TL;DR `xxd ./bin/app | vim —` and `:%!xxd -r > ./bin/new_app`")
     end
 
     it "renders bold link" do
       subject = Medium::Post::Paragraph.from_json(%{{"name": "d2a9", "type": 6, "text": "My xxd in the world.", "markups": [{"type": 3, "start": 3,"end": 6, "href": "http://example.com"},{"type": 1,"start": 3,"end": 6}]}})
-      subject.to_md.should eq("> My [**xxd**](http://example.com) in the world.")
+      subject.to_md[0].should eq("> My [**xxd**](http://example.com) in the world.")
     end
 
     it "skip render background image" do
       subject = Medium::Post::Paragraph.from_json(%{{"name": "d2a9", "type": 15, "text": "picture by me", "markups": [{"type": 3, "start": 11,"end": 15, "href": "http://example.com", "title": "", "rel": "", "anchorType": 0}]}})
-      subject.to_md.should eq("")
+      subject.to_md[0].should eq("")
     end
   end
 end

spec/medium/post_spec.cr

@@ -44,88 +44,88 @@ describe Medium::Post do
   describe "#to_md" do
     it "render full page" do
       subject = Medium::Post.from_json(post_fixture)
-      subject.to_md.size.should eq(2825)
+      subject.to_md.size.should eq(3042)
     end
 
     it "renders header" do
       subject = Medium::Post.from_json(post_fixture)
       paragraph = subject.content.bodyModel.paragraphs[0]
-      paragraph.to_md.should eq("# Modify binary files with VIM")
+      paragraph.to_md[0].should eq("# Modify binary files with VIM")
     end
 
     it "renders blockquotes" do
       subject = Medium::Post.from_json(post_fixture)
       paragraph = subject.content.bodyModel.paragraphs[1]
-      paragraph.to_md.should eq("> TL;DR `xxd ./bin/app | vim —` and `:%!xxd -r > ./bin/new_app`")
+      paragraph.to_md[0].should eq("> TL;DR `xxd ./bin/app | vim —` and `:%!xxd -r > ./bin/new_app`")
     end
 
     it "renders image" do
       subject = Medium::Post.from_json(post_fixture)
       paragraph = subject.content.bodyModel.paragraphs[2]
-      paragraph.to_md.should eq("![Photo by Markus Spiske on Unsplash](./assets/0*FbFs8aNmqNLKw4BM)")
+      paragraph.to_md[0].should eq("![Photo by Markus Spiske on Unsplash][image_ref_MCpGYkZzOGFObXFOTEt3NEJN]")
     end
 
     describe "paragraph text" do
       it "renders content with capital letter" do
         subject = Medium::Post.from_json(post_fixture)
         paragraph = subject.content.bodyModel.paragraphs[3]
-        paragraph.to_md.should contain("When I was a student")
+        paragraph.to_md[0].should contain("When I was a student")
       end
 
       it "renders content with links" do
         subject = Medium::Post.from_json(post_fixture)
         paragraph = subject.content.bodyModel.paragraphs[3]
-        paragraph.to_md.should contain("like [Total Commander](https://www.ghisler.com/) or [FAR manager](https://www.farmanager.com/)")
+        paragraph.to_md[0].should contain("like [Total Commander](https://www.ghisler.com/) or [FAR manager](https://www.farmanager.com/)")
       end
 
       it "renders content with bold text with link" do
         subject = Medium::Post.from_json(post_fixture)
         paragraph = subject.content.bodyModel.paragraphs[7]
-        paragraph.to_md.should contain("I came to: [**xxd**](http://vim.wikia.com/wiki/Hex_dump)")
+        paragraph.to_md[0].should contain("I came to: [**xxd**](http://vim.wikia.com/wiki/Hex_dump)")
       end
 
       it "renders content with inline code block" do
         subject = Medium::Post.from_json(post_fixture)
         paragraph = subject.content.bodyModel.paragraphs[7]
-        paragraph.to_md.should contain(%{of `vim-common` package})
+        paragraph.to_md[0].should contain(%{of `vim-common` package})
       end
     end
 
     it "renders numered list first" do
       subject = Medium::Post.from_json(post_fixture)
       paragraph = subject.content.bodyModel.paragraphs[5]
-      paragraph.to_md.should eq("1. it should be cross distributive solution")
+      paragraph.to_md[0].should eq("1. it should be cross distributive solution")
     end
 
     it "renders numered list second" do
       subject = Medium::Post.from_json(post_fixture)
       paragraph = subject.content.bodyModel.paragraphs[6]
-      paragraph.to_md.should eq("1. should be easy to use with Vim (as main my editor for linux machines)")
+      paragraph.to_md[0].should eq("1. should be easy to use with Vim (as main my editor for linux machines)")
     end
 
     pending "render split" do
       subject = Medium::Post.from_json(post_fixture)
       paragraph = subject.content.bodyModel.paragraphs[12]
-      paragraph.to_md.should eq("---")
+      paragraph.to_md[0].should eq("---")
     end
 
     it "render iframe" do
       subject = Medium::Post.from_json(post_fixture)
       paragraph = subject.content.bodyModel.paragraphs[12]
-      paragraph.to_md.should eq("<iframe src=\"./assets/ab24f0b378f797307fddc32f10a99685.html\"></iframe>")
+      paragraph.to_md[0].should eq("<iframe src=\"./assets/ab24f0b378f797307fddc32f10a99685.html\"></iframe>")
     end
 
     it "render code block" do
       subject = Medium::Post.from_json(post_fixture)
       paragraph = subject.content.bodyModel.paragraphs[15]
-      paragraph.to_md.should contain(%{```\n[terminal 1]})
+      paragraph.to_md[0].should contain(%{```\n[terminal 1]})
     end
 
     it "understands alignment attribute" do
       # Original: https://medium.com/@jico/the-fine-art-of-javascript-error-tracking-bc031f24c659
       subject = Medium::Post.from_json(post_fixture)
       paragraph = subject.content.bodyModel.paragraphs[21]
-      paragraph.to_md.should contain(%{# The Title with image})
+      paragraph.to_md[0].should contain(%{# The Title with image})
     end
 
     describe "metadata" do

spec/spec_helper.cr

@@ -1,6 +1,15 @@
 require "spec"
+require "webmock"
+
 require "../src/medup"
 
+Spec.before_suite do
+  WebMock.stub(:get, "https://miro.medium.com/0*FbFs8aNmqNLKw4BM")
+    .to_return(body: "some binary content")
+  WebMock.stub(:get, "https://miro.medium.com/1*NVLl4oVmMQtumKL-DVV1rA.png")
+    .to_return(body: "some binary content")
+end
+
 def fixtures(name)
   raw = File.read(File.join("spec", "fixtures", name))
   JSON.parse(raw)

src/medium/post.cr

@@ -39,8 +39,13 @@ module Medium
       end
 
       result += "---\n\n"
-      result +
-        @content.bodyModel.paragraphs.map(&.to_md).join("\n\n")
+      assets = "\n"
+      @content.bodyModel.paragraphs.map do |paragraph|
+        content, footer = paragraph.to_md
+        result += content + "\n\n"
+        assets += footer + "\n" unless footer.empty?
+      end
+      result + assets
     end
 
     def to_pretty_json

src/medium/post/paragraph.cr

@@ -1,3 +1,5 @@
+require "base64"
+
 module Medium
   class Post
     class Paragraph
@@ -19,49 +21,69 @@ module Medium
         strict: true
       )
 
-      def to_md
-        case @type
-        when 1
-          markup
-        when 2
-          "# #{markup}"
-        when 3
-          "# #{markup}"
-        when 4
-          # "[#{@text}](https://miro.medium.com/#{metadata.try(&.id)})"
-          if @href
-            "[![#{@text}](./assets/#{metadata.try(&.id)})](#{@href})"
-          else
-            "![#{@text}](./assets/#{metadata.try(&.id)})"
-          end
-        when 6
-          "> #{markup}"
-        when 7
-          ">> #{markup}"
-        when 8
-          "```\n#{@text}\n```"
-        when 9
-          "* #{markup}"
-        when 10
-          "1. #{markup}"
-        when 11
-          if @iframe.nil?
-            "<!-- Missing iframe -->"
-          else
-            frame = @iframe.not_nil!
-            "<iframe src=\"./assets/#{frame.mediaResourceId}.html\"></iframe>"
-            # Support Frame mode with inline content. Github does not support it.
-            # "<frame>#{frame.get}</frame>"
-          end
-        when 13
-          "### #{markup}"
-        when 14
-          "#{@mixtapeMetadata.try &.href}"
-        when 15
-          ""
-        else
-          raise "Unknown paragraph type #{@type} with text #{@text}"
-        end
+      def to_md : Tuple(String, String)
+        content : String = ""
+        assets = ""
+        content = case @type
+                  when 1
+                    markup
+                  when 2
+                    "# #{markup}"
+                  when 3
+                    "# #{markup}"
+                  when 4
+                    m = metadata
+                    if !m.nil? && !m.id.nil?
+                      asset_id = Base64.strict_encode(m.id || "")
+                      assets = "[image_ref_#{asset_id}]: data:image/png;base64,"
+                      img = "![#{@text}][image_ref_#{asset_id}]"
+                      encoded = download_image(m.id || "")
+                      assets += encoded
+                      if @href
+                        img = "[#{img}](#{@href})"
+                      end
+                      img
+                    else
+                      ""
+                    end
+                  when 6
+                    "> #{markup}"
+                  when 7
+                    ">> #{markup}"
+                  when 8
+                    "```\n#{@text}\n```"
+                  when 9
+                    "* #{markup}"
+                  when 10
+                    "1. #{markup}"
+                  when 11
+                    if @iframe.nil?
+                      "<!-- Missing iframe -->"
+                    else
+                      frame = @iframe.not_nil!
+                      "<iframe src=\"./assets/#{frame.mediaResourceId}.html\"></iframe>"
+                      # Support Frame mode with inline content. Github does not support it.
+                      # "<frame>#{frame.get}</frame>"
+                    end
+                  when 13
+                    "### #{markup}"
+                  when 14
+                    "#{@mixtapeMetadata.try &.href}"
+                  when 15
+                    ""
+                  else
+                    raise "Unknown paragraph type #{@type} with text #{@text}"
+                  end
+        return content, assets
+      end
+
+      def download_image(name : String)
+        download_url("https://miro.medium.com/#{name}")
+      end
+
+      def download_url(src)
+        response = HTTP::Client.get(src)
+        Base64.strict_encode(response.body)
       end
 
       def markup