From 76b0eeebe8877684343ae8e9c8c1ab7bab85c345 Mon Sep 17 00:00:00 2001 From: mydearxym Date: Sun, 7 Feb 2021 15:00:23 +0800 Subject: [PATCH 01/23] chore(clean-up): remove old wip tags --- test/helper/converter/md_to_editor_test.exs | 2 -- test/helper/converter/mention_parser_test.exs | 3 --- test/helper/utils_test.exs | 1 - 3 files changed, 6 deletions(-) diff --git a/test/helper/converter/md_to_editor_test.exs b/test/helper/converter/md_to_editor_test.exs index d46ff6b93..228497eac 100644 --- a/test/helper/converter/md_to_editor_test.exs +++ b/test/helper/converter/md_to_editor_test.exs @@ -9,7 +9,6 @@ defmodule GroupherServer.Test.Helper.Converter.MdToEditor do # alias Helper.Converter.HtmlSanitizer, as: Sanitizer describe "[basic md test]" do - @tag :wip test "basic markdown ast parser should work" do markdown = """ # header one @@ -229,7 +228,6 @@ defmodule GroupherServer.Test.Helper.Converter.MdToEditor do ] end - @tag :wip test "complex ast parser should work" do markdown = """ diff --git a/test/helper/converter/mention_parser_test.exs b/test/helper/converter/mention_parser_test.exs index f03d9e4fe..979be5db9 100644 --- a/test/helper/converter/mention_parser_test.exs +++ b/test/helper/converter/mention_parser_test.exs @@ -13,21 +13,18 @@ defmodule GroupherServer.Test.Helper.Converter.MentionParser do this is a #test #message with #a few #test tags from +me @you2 and @中文我 xxx@email.com """ - @tag :wip test "parse should return an empty for blank input" do ret = MentionParser.parse("", :mentions) assert ret == [] end - @tag :wip test "mention should parsed in list" do ret = MentionParser.parse(@test_message, :mentions) assert ret == ["@you", "@you2"] end - @tag :wip test "email should not be parsed" do ret = MentionParser.parse(@test_message, :mentions) diff --git a/test/helper/utils_test.exs b/test/helper/utils_test.exs index b7e784929..b97c86890 100644 --- a/test/helper/utils_test.exs +++ b/test/helper/utils_test.exs @@ -4,7 +4,6 @@ defmodule GroupherServer.Test.Helper.UtilsTest do alias Helper.Utils describe "map keys to string" do - @tag :wip test "atom keys should covert to string keys on nested map" do atom_map = %{ data: %{ From 72f3813f16be3676c312761934331388cc0cc754 Mon Sep 17 00:00:00 2001 From: mydearxym Date: Sun, 7 Feb 2021 20:09:15 +0800 Subject: [PATCH 02/23] test(editor): paragraph and header basic test --- config/config.exs | 5 +- lib/helper/converter/editor_to_html.ex | 23 +++--- test/helper/converter/editor_to_html_test.exs | 72 ++++++++++++++++--- test/helper/converter/md_to_editor_test.exs | 7 +- 4 files changed, 87 insertions(+), 20 deletions(-) diff --git a/config/config.exs b/config/config.exs index 6dc3ff2cb..06353923a 100644 --- a/config/config.exs +++ b/config/config.exs @@ -46,7 +46,10 @@ config :groupher_server, :general, user_achieve_star_weight: 1, user_achieve_watch_weight: 1, user_achieve_favorite_weight: 2, - user_achieve_follow_weight: 3 + user_achieve_follow_weight: 3, + # the rich editor html tag wrapper + # NOTE: DONOT CHANGE ONCE SET, OTHERWISE IT WILL CAUSE INCOMPATIBILITY ISSUE + article_viewer_tag: "article-viewer-wrapper" config :groupher_server, :customization, theme: "cyan", diff --git a/lib/helper/converter/editor_to_html.ex b/lib/helper/converter/editor_to_html.ex index a4321272b..9312e86dd 100644 --- a/lib/helper/converter/editor_to_html.ex +++ b/lib/helper/converter/editor_to_html.ex @@ -4,13 +4,15 @@ defmodule Helper.Converter.EditorToHtml do see https://editorjs.io/ """ + import Helper.Utils, only: [get_config: 2] + alias Helper.Converter.HtmlSanitizer alias Helper.Converter.EditorToHtml.Assets alias Helper.Utils alias Assets.{DelimiterIcons} - @html_class_prefix "cps-viewer" + @article_viewer_tag get_config(:general, :article_viewer_tag) @spec to_html(binary | maybe_improper_list) :: false | {:ok, <<_::64, _::_*8>>} def to_html(string) when is_binary(string) do @@ -22,10 +24,11 @@ defmodule Helper.Converter.EditorToHtml do acc <> clean_html end) - {:ok, "
#{content}
"} + {:ok, "
#{content}
"} end end + @desc "used for markdown ast to editor" def to_html(editor_blocks) when is_list(editor_blocks) do content = Enum.reduce(editor_blocks, "", fn block, acc -> @@ -33,7 +36,7 @@ defmodule Helper.Converter.EditorToHtml do acc <> clean_html end) - {:ok, "
#{content}
"} + {:ok, "
#{content}
"} end # IO.inspect(data, label: "parse header") @@ -41,21 +44,21 @@ defmodule Helper.Converter.EditorToHtml do text = get_in(data, ["text"]) level = get_in(data, ["level"]) - "#{text}" + "#{text}" end # IO.inspect(data, label: "parse paragraph") defp parse_block(%{"type" => "paragraph", "data" => data}) do text = get_in(data, ["text"]) - "

#{text}

" + "

#{text}

" end # IO.inspect(data, label: "parse image") defp parse_block(%{"type" => "image", "data" => data}) do url = get_in(data, ["file", "url"]) - "
" + "
" # |> IO.inspect(label: "iamge ret") end @@ -94,7 +97,7 @@ defmodule Helper.Converter.EditorToHtml do end end) - "
#{content}
" + "
#{content}
" # |> IO.inspect(label: "jjj") end @@ -102,7 +105,7 @@ defmodule Helper.Converter.EditorToHtml do svg_icon = DelimiterIcons.svg(type) # TODO: left-wing, righ-wing staff - {:skip_sanitize, "
#{svg_icon}
"} + {:skip_sanitize, "
#{svg_icon}
"} end # IO.inspect(data, label: "parse linkTool") @@ -110,7 +113,7 @@ defmodule Helper.Converter.EditorToHtml do defp parse_block(%{"type" => "linkTool", "data" => data}) do link = get_in(data, ["link"]) - "" + "" # |> IO.inspect(label: "linkTool ret") end @@ -118,7 +121,7 @@ defmodule Helper.Converter.EditorToHtml do defp parse_block(%{"type" => "quote", "data" => data}) do text = get_in(data, ["text"]) - "
#{text}
" + "
#{text}
" # |> IO.inspect(label: "quote ret") end diff --git a/test/helper/converter/editor_to_html_test.exs b/test/helper/converter/editor_to_html_test.exs index 730357bdc..c352618a9 100644 --- a/test/helper/converter/editor_to_html_test.exs +++ b/test/helper/converter/editor_to_html_test.exs @@ -1,24 +1,27 @@ defmodule GroupherServer.Test.Helper.Converter.EditorToHtml do @moduledoc false + # import Helper.Utils, only: [get_config: 2] use GroupherServerWeb.ConnCase, async: true alias Helper.Converter.EditorToHtml, as: Parser + # @article_viewer_tag get_config(:general, :article_viewer_tag) + @real_editor_data ~S({ "time" : 1567250876713, "blocks" : [ { - "type" : "header", + "type" : "paragraph", "data" : { - "text" : "Editor.js", - "level" : 2 + "text" : "Hey. Meet the new Editor. On this page you can see it in action — try to edit this text." } }, { - "type" : "paragraph", + "type" : "header", "data" : { - "text" : "Hey. Meet the new Editor. On this page you can see it in action — try to edit this text." + "text" : "Editor.js", + "level" : 2 } }, { @@ -183,11 +186,10 @@ defmodule GroupherServer.Test.Helper.Converter.EditorToHtml do }) describe "[basic convert]" do - test "basic string_json should work" do + test "basic string_json parse should work" do string = ~S({"time":1566184478687,"blocks":[{}],"version":"2.15.0"}) {:ok, converted} = Parser.string_to_json(string) - assert converted["time"] == 1_566_184_478_687 assert converted["version"] == "2.15.0" end @@ -206,7 +208,61 @@ defmodule GroupherServer.Test.Helper.Converter.EditorToHtml do end end - describe "[block convert]" do + describe "[block unit parse]" do + @editor_data ~S({ + "time" : 1567250876713, + "blocks" : [ + { + "type" : "paragraph", + "data" : { + "text" : "paragraph content" + } + } + ], + "version" : "2.15.0" + }) + test "paragraph parse should work" do + {:ok, converted} = Parser.to_html(@editor_data) + + assert converted == "

paragraph content

" + end + + @editor_json %{ + "time" => 1_567_250_876_713, + "blocks" => [ + %{ + "type" => "header", + "data" => %{ + "text" => "header content", + "level" => 1 + } + }, + %{ + "type" => "header", + "data" => %{ + "text" => "header content", + "level" => 2 + } + }, + %{ + "type" => "header", + "data" => %{ + "text" => "header content", + "level" => 3 + } + } + ], + "version" => "2.15.0" + } + @tag :wip + test "header parse should work" do + {:ok, editor_string} = Jason.encode(@editor_json) + {:ok, converted} = Parser.to_html(editor_string) + + assert converted == + "

header content

header content

header content

" + end + test "code block should avoid potential xss script attack" do {:ok, converted} = Parser.to_html(@real_editor_data) diff --git a/test/helper/converter/md_to_editor_test.exs b/test/helper/converter/md_to_editor_test.exs index 228497eac..0fa0e57ee 100644 --- a/test/helper/converter/md_to_editor_test.exs +++ b/test/helper/converter/md_to_editor_test.exs @@ -2,10 +2,14 @@ defmodule GroupherServer.Test.Helper.Converter.MdToEditor do @moduledoc """ parse markdown string to editorjs's json format """ + import Helper.Utils, only: [get_config: 2] + use GroupherServerWeb.ConnCase, async: true alias Helper.Converter.MdToEditor, as: Converter alias Helper.Converter.EditorToHtml + + @article_viewer_tag get_config(:general, :article_viewer_tag) # alias Helper.Converter.HtmlSanitizer, as: Sanitizer describe "[basic md test]" do @@ -228,6 +232,7 @@ defmodule GroupherServer.Test.Helper.Converter.MdToEditor do ] end + @tag :wip test "complex ast parser should work" do markdown = """ @@ -247,7 +252,7 @@ defmodule GroupherServer.Test.Helper.Converter.MdToEditor do {:ok, html} = EditorToHtml.to_html(editor_blocks) assert html == - "

hello

this is a basic markdown text

delete me

italic me

My in-line-code-content is best

" + "

hello

this is a basic markdown text

delete me

italic me

My in-line-code-content is best

" end end end From f527057efedf2cd77a2f6b45d59558b2044025ef Mon Sep 17 00:00:00 2001 From: mydearxym Date: Mon, 8 Feb 2021 22:30:30 +0800 Subject: [PATCH 03/23] test(editor): re-org codebase by parse header part --- config/config.exs | 5 +- lib/helper/converter/editor_guards.ex | 20 ++++ lib/helper/converter/editor_to_html.ex | 98 +++++++++++++++---- lib/helper/metric/article.ex | 25 +++++ test/helper/converter/editor_to_html_test.exs | 82 ++++++++++++---- test/helper/converter/md_to_editor_test.exs | 6 +- 6 files changed, 191 insertions(+), 45 deletions(-) create mode 100644 lib/helper/converter/editor_guards.ex create mode 100644 lib/helper/metric/article.ex diff --git a/config/config.exs b/config/config.exs index 06353923a..6dc3ff2cb 100644 --- a/config/config.exs +++ b/config/config.exs @@ -46,10 +46,7 @@ config :groupher_server, :general, user_achieve_star_weight: 1, user_achieve_watch_weight: 1, user_achieve_favorite_weight: 2, - user_achieve_follow_weight: 3, - # the rich editor html tag wrapper - # NOTE: DONOT CHANGE ONCE SET, OTHERWISE IT WILL CAUSE INCOMPATIBILITY ISSUE - article_viewer_tag: "article-viewer-wrapper" + user_achieve_follow_weight: 3 config :groupher_server, :customization, theme: "cyan", diff --git a/lib/helper/converter/editor_guards.ex b/lib/helper/converter/editor_guards.ex new file mode 100644 index 000000000..53d162bb5 --- /dev/null +++ b/lib/helper/converter/editor_guards.ex @@ -0,0 +1,20 @@ +defmodule Helper.Converter.EditorGuards do + @moduledoc """ + guards for incoming editor json + + map_get is not support in current version, so we have to pass each arg for guard + see: https://elixirforum.com/t/discussion-incorporating-erlang-otp-21-map-guards-in-elixir/14816 + """ + @support_header_levels [1, 2, 3] + + defguard is_valid_header(text, level) + when is_binary(text) and level in @support_header_levels + + @doc "check if eyebowTitle OR footerTitle are valid" + defguard is_valid_header(text, level, subtitle) + when is_binary(text) and level in @support_header_levels and is_binary(subtitle) + + defguard is_valid_header(text, level, eyebrow_title, footer_title) + when is_binary(text) and level in @support_header_levels and is_binary(eyebrow_title) and + is_binary(footer_title) +end diff --git a/lib/helper/converter/editor_to_html.ex b/lib/helper/converter/editor_to_html.ex index 9312e86dd..bbc6a54a6 100644 --- a/lib/helper/converter/editor_to_html.ex +++ b/lib/helper/converter/editor_to_html.ex @@ -5,14 +5,15 @@ defmodule Helper.Converter.EditorToHtml do see https://editorjs.io/ """ import Helper.Utils, only: [get_config: 2] + import Helper.Converter.EditorGuards + # alias Helper.Converter.EditorGuards, as: Guards - alias Helper.Converter.HtmlSanitizer - alias Helper.Converter.EditorToHtml.Assets - alias Helper.Utils + alias Helper.Converter.{EditorToHtml, HtmlSanitizer} + alias Helper.{Metric, Utils} - alias Assets.{DelimiterIcons} + alias EditorToHtml.Assets.{DelimiterIcons} - @article_viewer_tag get_config(:general, :article_viewer_tag) + @clazz Metric.Article.class_names(:html) @spec to_html(binary | maybe_improper_list) :: false | {:ok, <<_::64, _::_*8>>} def to_html(string) when is_binary(string) do @@ -24,7 +25,7 @@ defmodule Helper.Converter.EditorToHtml do acc <> clean_html end) - {:ok, "
#{content}
"} + {:ok, "
#{content}
"} end end @@ -36,29 +37,86 @@ defmodule Helper.Converter.EditorToHtml do acc <> clean_html end) - {:ok, "
#{content}
"} + {:ok, "
#{content}
"} end - # IO.inspect(data, label: "parse header") - defp parse_block(%{"type" => "header", "data" => data}) do - text = get_in(data, ["text"]) - level = get_in(data, ["level"]) + defp parse_block(%{ + "type" => "header", + "data" => + %{ + "text" => text, + "level" => level, + "eyebrowTitle" => eyebrowTitle, + "footerTitle" => footerTitle + } = data + }) + when is_valid_header(text, level, eyebrowTitle, footerTitle) do + """ +
+
#{eyebrowTitle}
+ #{text} +
#{footerTitle}
+
+ """ + end + + defp parse_block(%{ + "type" => "header", + "data" => + %{ + "text" => text, + "level" => level, + "eyebrowTitle" => eyebrowTitle + } = data + }) + when is_valid_header(text, level, eyebrowTitle) do + """ +
+
#{eyebrowTitle}
+ #{text} +
+ """ + end + + defp parse_block(%{ + "type" => "header", + "data" => + %{ + "text" => text, + "level" => level, + "footerTitle" => footerTitle + } = data + }) + when is_valid_header(text, level, footerTitle) do + """ +
+ #{text} +
#{footerTitle}
+
+ """ + end + defp parse_block(%{ + "type" => "header", + "data" => %{ + "text" => text, + "level" => level + } + }) + when is_valid_header(text, level) do "#{text}" end - # IO.inspect(data, label: "parse paragraph") defp parse_block(%{"type" => "paragraph", "data" => data}) do text = get_in(data, ["text"]) "

#{text}

" end - # IO.inspect(data, label: "parse image") defp parse_block(%{"type" => "image", "data" => data}) do url = get_in(data, ["file", "url"]) - "
" + "
" # |> IO.inspect(label: "iamge ret") end @@ -80,7 +138,6 @@ defmodule Helper.Converter.EditorToHtml do "
    #{content}
" end - # IO.inspect(items, label: "checklist items") # TODO: add item class defp parse_block(%{"type" => "checklist", "data" => %{"items" => items}}) do content = @@ -97,7 +154,7 @@ defmodule Helper.Converter.EditorToHtml do end end) - "
#{content}
" + "
#{content}
" # |> IO.inspect(label: "jjj") end @@ -105,7 +162,7 @@ defmodule Helper.Converter.EditorToHtml do svg_icon = DelimiterIcons.svg(type) # TODO: left-wing, righ-wing staff - {:skip_sanitize, "
#{svg_icon}
"} + {:skip_sanitize, "
#{svg_icon}
"} end # IO.inspect(data, label: "parse linkTool") @@ -113,7 +170,7 @@ defmodule Helper.Converter.EditorToHtml do defp parse_block(%{"type" => "linkTool", "data" => data}) do link = get_in(data, ["link"]) - "" + "" # |> IO.inspect(label: "linkTool ret") end @@ -121,7 +178,7 @@ defmodule Helper.Converter.EditorToHtml do defp parse_block(%{"type" => "quote", "data" => data}) do text = get_in(data, ["text"]) - "
#{text}
" + "
#{text}
" # |> IO.inspect(label: "quote ret") end @@ -135,8 +192,7 @@ defmodule Helper.Converter.EditorToHtml do end defp parse_block(_block) do - # IO.puts("[unknow block]") - "[unknow block]" + "
[unknow block]
" end def string_to_json(string), do: Jason.decode(string) diff --git a/lib/helper/metric/article.ex b/lib/helper/metric/article.ex new file mode 100644 index 000000000..0f73cab3d --- /dev/null +++ b/lib/helper/metric/article.ex @@ -0,0 +1,25 @@ +defmodule Helper.Metric.Article do + @moduledoc """ + html article class names parsed from editor.js's json data + + currently use https://editorjs.io/ as rich-text editor + # NOTE: DONOT CHANGE ONCE SET, OTHERWISE IT WILL CAUSE INCOMPATIBILITY ISSUE + """ + + @doc """ + get all the class names of the parsed editor.js's html parts + """ + def class_names(:html) do + %{ + # root wrapper + viewer: "article-viewer-wrapper", + unknow_block: "unknow-block", + # header + header: %{ + wrapper: "header-wrapper", + eyebrow_title: "eyebrow-title", + footer_title: "footer-title" + } + } + end +end diff --git a/test/helper/converter/editor_to_html_test.exs b/test/helper/converter/editor_to_html_test.exs index c352618a9..9aa5b4025 100644 --- a/test/helper/converter/editor_to_html_test.exs +++ b/test/helper/converter/editor_to_html_test.exs @@ -1,12 +1,13 @@ defmodule GroupherServer.Test.Helper.Converter.EditorToHtml do @moduledoc false - # import Helper.Utils, only: [get_config: 2] + import Helper.Utils, only: [get_config: 2] use GroupherServerWeb.ConnCase, async: true + alias Helper.Metric alias Helper.Converter.EditorToHtml, as: Parser - # @article_viewer_tag get_config(:general, :article_viewer_tag) + @clazz Metric.Article.class_names(:html) @real_editor_data ~S({ "time" : 1567250876713, @@ -209,22 +210,23 @@ defmodule GroupherServer.Test.Helper.Converter.EditorToHtml do end describe "[block unit parse]" do - @editor_data ~S({ - "time" : 1567250876713, - "blocks" : [ - { - "type" : "paragraph", - "data" : { - "text" : "paragraph content" - } - } - ], - "version" : "2.15.0" - }) + @editor_json %{ + "time" => 1_567_250_876_713, + "blocks" => [ + %{ + "type" => "paragraph", + "data" => %{ + "text" => "paragraph content" + } + } + ], + "version" => "2.15.0" + } test "paragraph parse should work" do - {:ok, converted} = Parser.to_html(@editor_data) + {:ok, editor_string} = Jason.encode(@editor_json) + {:ok, converted} = Parser.to_html(editor_string) - assert converted == "

paragraph content

" + assert converted == "

paragraph content

" end @editor_json %{ @@ -260,7 +262,53 @@ defmodule GroupherServer.Test.Helper.Converter.EditorToHtml do {:ok, converted} = Parser.to_html(editor_string) assert converted == - "

header content

header content

header content

" + "

header content

header content

header content

" + end + + @editor_json %{ + "time" => 1_567_250_876_713, + "blocks" => [ + %{ + "type" => "header", + "data" => %{ + "text" => "header content", + "level" => 1, + "eyebrowTitle" => "eyebrow title content", + "footerTitle" => "footer title content" + } + } + ], + "version" => "2.15.0" + } + @tag :wip + test "full header parse should work" do + {:ok, editor_string} = Jason.encode(@editor_json) + {:ok, converted} = Parser.to_html(editor_string) + + assert converted == + "
\n
eyebrow title content
\n

header content

\n
footer title content
\n
\n
" + end + + @editor_json %{ + "time" => 1_567_250_876_713, + "blocks" => [ + %{ + "type" => "header", + "data" => %{ + "text" => [], + "level" => 1 + } + } + ], + "version" => "2.15.0" + } + @tag :wip + test "wrong header format data should have unknow hint" do + {:ok, editor_string} = Jason.encode(@editor_json) + {:ok, converted} = Parser.to_html(editor_string) + + assert converted == + "
[unknow block]
" end test "code block should avoid potential xss script attack" do diff --git a/test/helper/converter/md_to_editor_test.exs b/test/helper/converter/md_to_editor_test.exs index 0fa0e57ee..11d825d17 100644 --- a/test/helper/converter/md_to_editor_test.exs +++ b/test/helper/converter/md_to_editor_test.exs @@ -6,10 +6,11 @@ defmodule GroupherServer.Test.Helper.Converter.MdToEditor do use GroupherServerWeb.ConnCase, async: true + alias Helper.Metric alias Helper.Converter.MdToEditor, as: Converter alias Helper.Converter.EditorToHtml - @article_viewer_tag get_config(:general, :article_viewer_tag) + @clazz Metric.Article.class_names(:html) # alias Helper.Converter.HtmlSanitizer, as: Sanitizer describe "[basic md test]" do @@ -232,7 +233,6 @@ defmodule GroupherServer.Test.Helper.Converter.MdToEditor do ] end - @tag :wip test "complex ast parser should work" do markdown = """ @@ -252,7 +252,7 @@ defmodule GroupherServer.Test.Helper.Converter.MdToEditor do {:ok, html} = EditorToHtml.to_html(editor_blocks) assert html == - "

hello

this is a basic markdown text

delete me

italic me

My in-line-code-content is best

" + "

hello

this is a basic markdown text

delete me

italic me

My in-line-code-content is best

" end end end From 995e6023c50afcb75151aac8c42bd916f3cd3bf9 Mon Sep 17 00:00:00 2001 From: mydearxym Date: Tue, 9 Feb 2021 23:15:03 +0800 Subject: [PATCH 04/23] test(editor): parse workflow adjust --- lib/helper/converter/editor_to_html.ex | 76 ++++++++++++++++--- lib/helper/metric/article.ex | 1 + test/helper/converter/editor_to_html_test.exs | 75 ++++++++++++++---- 3 files changed, 126 insertions(+), 26 deletions(-) diff --git a/lib/helper/converter/editor_to_html.ex b/lib/helper/converter/editor_to_html.ex index bbc6a54a6..8c140dbc8 100644 --- a/lib/helper/converter/editor_to_html.ex +++ b/lib/helper/converter/editor_to_html.ex @@ -1,9 +1,47 @@ +defmodule Helper.Converter.ErrorHint do + @moduledoc """ + + see https://stackoverflow.com/a/33052969/4050784 + """ + + defmacro watch(type, field) do + quote do + @doc "give error hint when #{unquote(field)} is invalid type" + defp parse_block(%{ + "type" => "#{unquote(type)}", + "data" => %{ + "#{unquote(field)}" => _ + } + }) do + invalid_hint("#{unquote(type)}", "#{unquote(field)}") + end + end + end + + defmacro watch(type, field1, field2) do + quote do + @doc "give error hint when #{unquote(field1)} or #{unquote(field2)} is invalid type" + defp parse_block(%{ + "type" => "#{unquote(type)}", + "data" => %{ + "#{unquote(field1)}" => _, + "#{unquote(field2)}" => _ + } + }) do + invalid_hint("#{unquote(type)}", "#{unquote(field1)} or #{unquote(field2)}") + end + end + end +end + defmodule Helper.Converter.EditorToHtml do @moduledoc """ parse editor.js's json data to raw html and sanitize it see https://editorjs.io/ """ + require Helper.Converter.ErrorHint, as: ErrorHint + import Helper.Utils, only: [get_config: 2] import Helper.Converter.EditorGuards # alias Helper.Converter.EditorGuards, as: Guards @@ -46,16 +84,16 @@ defmodule Helper.Converter.EditorToHtml do %{ "text" => text, "level" => level, - "eyebrowTitle" => eyebrowTitle, - "footerTitle" => footerTitle + "eyebrowTitle" => eyebrow_title, + "footerTitle" => footer_title } = data }) - when is_valid_header(text, level, eyebrowTitle, footerTitle) do + when is_valid_header(text, level, eyebrow_title, footer_title) do """
-
#{eyebrowTitle}
+
#{eyebrow_title}
#{text} -
#{footerTitle}
+
#{footer_title}
""" end @@ -66,36 +104,40 @@ defmodule Helper.Converter.EditorToHtml do %{ "text" => text, "level" => level, - "eyebrowTitle" => eyebrowTitle + "eyebrowTitle" => eyebrow_title } = data }) - when is_valid_header(text, level, eyebrowTitle) do + when is_valid_header(text, level, eyebrow_title) do """
-
#{eyebrowTitle}
- #{text} +
#{eyebrow_title}
+ #{text}
""" end + ErrorHint.watch("header", "eyebrowTitle") + defp parse_block(%{ "type" => "header", "data" => %{ "text" => text, "level" => level, - "footerTitle" => footerTitle + "footerTitle" => footer_title } = data }) - when is_valid_header(text, level, footerTitle) do + when is_valid_header(text, level, footer_title) do """
#{text} -
#{footerTitle}
+
#{footer_title}
""" end + ErrorHint.watch("header", "footerTitle") + defp parse_block(%{ "type" => "header", "data" => %{ @@ -107,6 +149,8 @@ defmodule Helper.Converter.EditorToHtml do "#{text}" end + ErrorHint.watch("header", "text", "level") + defp parse_block(%{"type" => "paragraph", "data" => data}) do text = get_in(data, ["text"]) @@ -195,6 +239,14 @@ defmodule Helper.Converter.EditorToHtml do "
[unknow block]
" end + defp invalid_hint(part, message) do + "
[invalid-block] #{part}:#{message}
" + end + + # defp invalid_hint(part, message) do + # "
[invalid-block] #{part}:#{message}
" + # end + def string_to_json(string), do: Jason.decode(string) defp valid_editor_data?(map) when is_map(map) do diff --git a/lib/helper/metric/article.ex b/lib/helper/metric/article.ex index 0f73cab3d..b9b1235e8 100644 --- a/lib/helper/metric/article.ex +++ b/lib/helper/metric/article.ex @@ -14,6 +14,7 @@ defmodule Helper.Metric.Article do # root wrapper viewer: "article-viewer-wrapper", unknow_block: "unknow-block", + invalid_block: "invalid-block", # header header: %{ wrapper: "header-wrapper", diff --git a/test/helper/converter/editor_to_html_test.exs b/test/helper/converter/editor_to_html_test.exs index 9aa5b4025..9a90097b4 100644 --- a/test/helper/converter/editor_to_html_test.exs +++ b/test/helper/converter/editor_to_html_test.exs @@ -1,7 +1,6 @@ defmodule GroupherServer.Test.Helper.Converter.EditorToHtml do @moduledoc false - import Helper.Utils, only: [get_config: 2] use GroupherServerWeb.ConnCase, async: true alias Helper.Metric @@ -286,29 +285,77 @@ defmodule GroupherServer.Test.Helper.Converter.EditorToHtml do {:ok, converted} = Parser.to_html(editor_string) assert converted == - "
\n
eyebrow title content
\n

header content

\n
footer title content
\n
\n
" + "
\n
eyebrow title content
\n

header content

\n
footer title content
\n
\n
" end @editor_json %{ "time" => 1_567_250_876_713, - "blocks" => [ - %{ - "type" => "header", - "data" => %{ - "text" => [], - "level" => 1 - } - } - ], "version" => "2.15.0" } @tag :wip - test "wrong header format data should have unknow hint" do - {:ok, editor_string} = Jason.encode(@editor_json) + test "wrong header format data should have invalid hint" do + json = + Map.merge(@editor_json, %{ + "blocks" => [ + %{ + "type" => "header", + "data" => %{ + "text" => "header content", + "level" => 1, + "eyebrowTitle" => [] + } + } + ] + }) + + {:ok, editor_string} = Jason.encode(json) + {:ok, converted} = Parser.to_html(editor_string) + + assert converted == + "
[invalid-block] header:eyebrowTitle
" + + json = + Map.merge(@editor_json, %{ + "blocks" => [ + %{ + "type" => "header", + "data" => %{ + "text" => "header content", + "level" => 1, + "footerTitle" => [] + } + } + ] + }) + + {:ok, editor_string} = Jason.encode(json) + {:ok, converted} = Parser.to_html(editor_string) + + assert converted == + "
[invalid-block] header:footerTitle
" + + json = + Map.merge(@editor_json, %{ + "blocks" => [ + %{ + "type" => "header", + "data" => %{ + "text" => "header content", + "level" => [] + } + } + ] + }) + + {:ok, editor_string} = Jason.encode(json) {:ok, converted} = Parser.to_html(editor_string) assert converted == - "
[unknow block]
" + "
[invalid-block] header:text or level
" end test "code block should avoid potential xss script attack" do From 80b8831dcec45b6c50bfa4fd1481b68af3a0971d Mon Sep 17 00:00:00 2001 From: mydearxym Date: Wed, 10 Feb 2021 12:14:11 +0800 Subject: [PATCH 05/23] test(editor-parse): re-org dir structure based on blocks --- .../converter/editor_to_html/error_hint.ex | 35 ++++ lib/helper/converter/editor_to_html/header.ex | 94 +++++++++++ .../index.ex} | 120 +------------- .../editor_to_html_test/header_test.exs | 151 ++++++++++++++++++ .../index_test.exs} | 151 +----------------- .../editor_to_html_test/paragraph_test.exs | 32 ++++ test/helper/converter/md_to_editor_test.exs | 2 - 7 files changed, 318 insertions(+), 267 deletions(-) create mode 100644 lib/helper/converter/editor_to_html/error_hint.ex create mode 100644 lib/helper/converter/editor_to_html/header.ex rename lib/helper/converter/{editor_to_html.ex => editor_to_html/index.ex} (58%) create mode 100644 test/helper/converter/editor_to_html_test/header_test.exs rename test/helper/converter/{editor_to_html_test.exs => editor_to_html_test/index_test.exs} (64%) create mode 100644 test/helper/converter/editor_to_html_test/paragraph_test.exs diff --git a/lib/helper/converter/editor_to_html/error_hint.ex b/lib/helper/converter/editor_to_html/error_hint.ex new file mode 100644 index 000000000..d2abd2d67 --- /dev/null +++ b/lib/helper/converter/editor_to_html/error_hint.ex @@ -0,0 +1,35 @@ +defmodule Helper.Converter.EditorToHTML.ErrorHint do + @moduledoc """ + + see https://stackoverflow.com/a/33052969/4050784 + """ + + defmacro watch(type, field) do + quote do + @doc "give error hint when #{unquote(field)} is invalid type" + defp parse_block(%{ + "type" => "#{unquote(type)}", + "data" => %{ + "#{unquote(field)}" => _ + } + }) do + invalid_hint("#{unquote(type)}", "#{unquote(field)}") + end + end + end + + defmacro watch(type, field1, field2) do + quote do + @doc "give error hint when #{unquote(field1)} or #{unquote(field2)} is invalid type" + defp parse_block(%{ + "type" => "#{unquote(type)}", + "data" => %{ + "#{unquote(field1)}" => _, + "#{unquote(field2)}" => _ + } + }) do + invalid_hint("#{unquote(type)}", "#{unquote(field1)} or #{unquote(field2)}") + end + end + end +end diff --git a/lib/helper/converter/editor_to_html/header.ex b/lib/helper/converter/editor_to_html/header.ex new file mode 100644 index 000000000..047ef73bc --- /dev/null +++ b/lib/helper/converter/editor_to_html/header.ex @@ -0,0 +1,94 @@ +defmodule Helper.Converter.EditorToHtml.Header do + @moduledoc """ + parse editor.js's json data to raw html and sanitize it + + see https://editorjs.io/ + """ + require Helper.Converter.EditorToHTML.ErrorHint, as: ErrorHint + + import Helper.Converter.EditorGuards + + alias Helper.Converter.{EditorToHtml, HtmlSanitizer} + alias Helper.{Metric, Utils} + + alias EditorToHtml.Assets.{DelimiterIcons} + + @clazz Metric.Article.class_names(:html) + + defmacro parse_block do + quote do + defp parse_block(%{ + "type" => "header", + "data" => + %{ + "text" => text, + "level" => level, + "eyebrowTitle" => eyebrow_title, + "footerTitle" => footer_title + } = data + }) + when is_valid_header(text, level, eyebrow_title, footer_title) do + """ +
+
#{eyebrow_title}
+ #{text} +
#{footer_title}
+
+ """ + end + + defp parse_block(%{ + "type" => "header", + "data" => + %{ + "text" => text, + "level" => level, + "eyebrowTitle" => eyebrow_title + } = data + }) + when is_valid_header(text, level, eyebrow_title) do + """ +
+
#{eyebrow_title}
+ #{text} +
+ """ + end + + ErrorHint.watch("header", "eyebrowTitle") + + defp parse_block(%{ + "type" => "header", + "data" => + %{ + "text" => text, + "level" => level, + "footerTitle" => footer_title + } = data + }) + when is_valid_header(text, level, footer_title) do + """ +
+ #{text} +
#{footer_title}
+
+ """ + end + + ErrorHint.watch("header", "footerTitle") + + defp parse_block(%{ + "type" => "header", + "data" => %{ + "text" => text, + "level" => level + } + }) + when is_valid_header(text, level) do + "#{text}" + end + + ErrorHint.watch("header", "text", "level") + end + end +end diff --git a/lib/helper/converter/editor_to_html.ex b/lib/helper/converter/editor_to_html/index.ex similarity index 58% rename from lib/helper/converter/editor_to_html.ex rename to lib/helper/converter/editor_to_html/index.ex index 8c140dbc8..f18d29f54 100644 --- a/lib/helper/converter/editor_to_html.ex +++ b/lib/helper/converter/editor_to_html/index.ex @@ -1,54 +1,19 @@ -defmodule Helper.Converter.ErrorHint do - @moduledoc """ - - see https://stackoverflow.com/a/33052969/4050784 - """ - - defmacro watch(type, field) do - quote do - @doc "give error hint when #{unquote(field)} is invalid type" - defp parse_block(%{ - "type" => "#{unquote(type)}", - "data" => %{ - "#{unquote(field)}" => _ - } - }) do - invalid_hint("#{unquote(type)}", "#{unquote(field)}") - end - end - end - - defmacro watch(type, field1, field2) do - quote do - @doc "give error hint when #{unquote(field1)} or #{unquote(field2)} is invalid type" - defp parse_block(%{ - "type" => "#{unquote(type)}", - "data" => %{ - "#{unquote(field1)}" => _, - "#{unquote(field2)}" => _ - } - }) do - invalid_hint("#{unquote(type)}", "#{unquote(field1)} or #{unquote(field2)}") - end - end - end -end - defmodule Helper.Converter.EditorToHtml do @moduledoc """ parse editor.js's json data to raw html and sanitize it see https://editorjs.io/ """ - require Helper.Converter.ErrorHint, as: ErrorHint + require Helper.Converter.EditorToHTML.ErrorHint, as: ErrorHint + require Helper.Converter.EditorToHtml.Header, as: Header - import Helper.Utils, only: [get_config: 2] import Helper.Converter.EditorGuards - # alias Helper.Converter.EditorGuards, as: Guards alias Helper.Converter.{EditorToHtml, HtmlSanitizer} alias Helper.{Metric, Utils} + # alias EditorToHtml.{Header} + alias EditorToHtml.Assets.{DelimiterIcons} @clazz Metric.Article.class_names(:html) @@ -78,78 +43,7 @@ defmodule Helper.Converter.EditorToHtml do {:ok, "
#{content}
"} end - defp parse_block(%{ - "type" => "header", - "data" => - %{ - "text" => text, - "level" => level, - "eyebrowTitle" => eyebrow_title, - "footerTitle" => footer_title - } = data - }) - when is_valid_header(text, level, eyebrow_title, footer_title) do - """ -
-
#{eyebrow_title}
- #{text} -
#{footer_title}
-
- """ - end - - defp parse_block(%{ - "type" => "header", - "data" => - %{ - "text" => text, - "level" => level, - "eyebrowTitle" => eyebrow_title - } = data - }) - when is_valid_header(text, level, eyebrow_title) do - """ -
-
#{eyebrow_title}
- #{text} -
- """ - end - - ErrorHint.watch("header", "eyebrowTitle") - - defp parse_block(%{ - "type" => "header", - "data" => - %{ - "text" => text, - "level" => level, - "footerTitle" => footer_title - } = data - }) - when is_valid_header(text, level, footer_title) do - """ -
- #{text} -
#{footer_title}
-
- """ - end - - ErrorHint.watch("header", "footerTitle") - - defp parse_block(%{ - "type" => "header", - "data" => %{ - "text" => text, - "level" => level - } - }) - when is_valid_header(text, level) do - "#{text}" - end - - ErrorHint.watch("header", "text", "level") + Header.parse_block() defp parse_block(%{"type" => "paragraph", "data" => data}) do text = get_in(data, ["text"]) @@ -243,10 +137,6 @@ defmodule Helper.Converter.EditorToHtml do "
[invalid-block] #{part}:#{message}
" end - # defp invalid_hint(part, message) do - # "
[invalid-block] #{part}:#{message}
" - # end - def string_to_json(string), do: Jason.decode(string) defp valid_editor_data?(map) when is_map(map) do diff --git a/test/helper/converter/editor_to_html_test/header_test.exs b/test/helper/converter/editor_to_html_test/header_test.exs new file mode 100644 index 000000000..c36553f02 --- /dev/null +++ b/test/helper/converter/editor_to_html_test/header_test.exs @@ -0,0 +1,151 @@ +defmodule GroupherServer.Test.Helper.Converter.EditorToHtml.Header do + @moduledoc false + + use GroupherServerWeb.ConnCase, async: true + + alias Helper.Metric + alias Helper.Converter.EditorToHtml, as: Parser + + @clazz Metric.Article.class_names(:html) + + describe "[header block unit]" do + @editor_json %{ + "time" => 1_567_250_876_713, + "blocks" => [ + %{ + "type" => "header", + "data" => %{ + "text" => "header content", + "level" => 1 + } + }, + %{ + "type" => "header", + "data" => %{ + "text" => "header content", + "level" => 2 + } + }, + %{ + "type" => "header", + "data" => %{ + "text" => "header content", + "level" => 3 + } + } + ], + "version" => "2.15.0" + } + @tag :wip + test "header parse should work" do + {:ok, editor_string} = Jason.encode(@editor_json) + {:ok, converted} = Parser.to_html(editor_string) + + assert converted == + "

header content

header content

header content

" + end + + @editor_json %{ + "time" => 1_567_250_876_713, + "blocks" => [ + %{ + "type" => "header", + "data" => %{ + "text" => "header content", + "level" => 1, + "eyebrowTitle" => "eyebrow title content", + "footerTitle" => "footer title content" + } + } + ], + "version" => "2.15.0" + } + @tag :wip + test "full header parse should work" do + {:ok, editor_string} = Jason.encode(@editor_json) + {:ok, converted} = Parser.to_html(editor_string) + + assert converted == + "
\n
eyebrow title content
\n

header content

\n
footer title content
\n
\n
" + end + + @editor_json %{ + "time" => 1_567_250_876_713, + "version" => "2.15.0" + } + @tag :wip + test "wrong header format data should have invalid hint" do + json = + Map.merge(@editor_json, %{ + "blocks" => [ + %{ + "type" => "header", + "data" => %{ + "text" => "header content", + "level" => 1, + "eyebrowTitle" => [] + } + } + ] + }) + + {:ok, editor_string} = Jason.encode(json) + {:ok, converted} = Parser.to_html(editor_string) + + assert converted == + "
[invalid-block] header:eyebrowTitle
" + + json = + Map.merge(@editor_json, %{ + "blocks" => [ + %{ + "type" => "header", + "data" => %{ + "text" => "header content", + "level" => 1, + "footerTitle" => [] + } + } + ] + }) + + {:ok, editor_string} = Jason.encode(json) + {:ok, converted} = Parser.to_html(editor_string) + + assert converted == + "
[invalid-block] header:footerTitle
" + + json = + Map.merge(@editor_json, %{ + "blocks" => [ + %{ + "type" => "header", + "data" => %{ + "text" => "header content", + "level" => [] + } + } + ] + }) + + {:ok, editor_string} = Jason.encode(json) + {:ok, converted} = Parser.to_html(editor_string) + + assert converted == + "
[invalid-block] header:text or level
" + end + + test "code block should avoid potential xss script attack" do + {:ok, converted} = Parser.to_html(@real_editor_data) + + safe_script = + "
<script>evil scripts</script>
" + + assert converted |> String.contains?(safe_script) + end + end +end diff --git a/test/helper/converter/editor_to_html_test.exs b/test/helper/converter/editor_to_html_test/index_test.exs similarity index 64% rename from test/helper/converter/editor_to_html_test.exs rename to test/helper/converter/editor_to_html_test/index_test.exs index 9a90097b4..e9a4a0cc9 100644 --- a/test/helper/converter/editor_to_html_test.exs +++ b/test/helper/converter/editor_to_html_test/index_test.exs @@ -208,156 +208,7 @@ defmodule GroupherServer.Test.Helper.Converter.EditorToHtml do end end - describe "[block unit parse]" do - @editor_json %{ - "time" => 1_567_250_876_713, - "blocks" => [ - %{ - "type" => "paragraph", - "data" => %{ - "text" => "paragraph content" - } - } - ], - "version" => "2.15.0" - } - test "paragraph parse should work" do - {:ok, editor_string} = Jason.encode(@editor_json) - {:ok, converted} = Parser.to_html(editor_string) - - assert converted == "

paragraph content

" - end - - @editor_json %{ - "time" => 1_567_250_876_713, - "blocks" => [ - %{ - "type" => "header", - "data" => %{ - "text" => "header content", - "level" => 1 - } - }, - %{ - "type" => "header", - "data" => %{ - "text" => "header content", - "level" => 2 - } - }, - %{ - "type" => "header", - "data" => %{ - "text" => "header content", - "level" => 3 - } - } - ], - "version" => "2.15.0" - } - @tag :wip - test "header parse should work" do - {:ok, editor_string} = Jason.encode(@editor_json) - {:ok, converted} = Parser.to_html(editor_string) - - assert converted == - "

header content

header content

header content

" - end - - @editor_json %{ - "time" => 1_567_250_876_713, - "blocks" => [ - %{ - "type" => "header", - "data" => %{ - "text" => "header content", - "level" => 1, - "eyebrowTitle" => "eyebrow title content", - "footerTitle" => "footer title content" - } - } - ], - "version" => "2.15.0" - } - @tag :wip - test "full header parse should work" do - {:ok, editor_string} = Jason.encode(@editor_json) - {:ok, converted} = Parser.to_html(editor_string) - - assert converted == - "
\n
eyebrow title content
\n

header content

\n
footer title content
\n
\n
" - end - - @editor_json %{ - "time" => 1_567_250_876_713, - "version" => "2.15.0" - } - @tag :wip - test "wrong header format data should have invalid hint" do - json = - Map.merge(@editor_json, %{ - "blocks" => [ - %{ - "type" => "header", - "data" => %{ - "text" => "header content", - "level" => 1, - "eyebrowTitle" => [] - } - } - ] - }) - - {:ok, editor_string} = Jason.encode(json) - {:ok, converted} = Parser.to_html(editor_string) - - assert converted == - "
[invalid-block] header:eyebrowTitle
" - - json = - Map.merge(@editor_json, %{ - "blocks" => [ - %{ - "type" => "header", - "data" => %{ - "text" => "header content", - "level" => 1, - "footerTitle" => [] - } - } - ] - }) - - {:ok, editor_string} = Jason.encode(json) - {:ok, converted} = Parser.to_html(editor_string) - - assert converted == - "
[invalid-block] header:footerTitle
" - - json = - Map.merge(@editor_json, %{ - "blocks" => [ - %{ - "type" => "header", - "data" => %{ - "text" => "header content", - "level" => [] - } - } - ] - }) - - {:ok, editor_string} = Jason.encode(json) - {:ok, converted} = Parser.to_html(editor_string) - - assert converted == - "
[invalid-block] header:text or level
" - end - + describe "[secure issues]" do test "code block should avoid potential xss script attack" do {:ok, converted} = Parser.to_html(@real_editor_data) diff --git a/test/helper/converter/editor_to_html_test/paragraph_test.exs b/test/helper/converter/editor_to_html_test/paragraph_test.exs new file mode 100644 index 000000000..c84e4b44b --- /dev/null +++ b/test/helper/converter/editor_to_html_test/paragraph_test.exs @@ -0,0 +1,32 @@ +defmodule GroupherServer.Test.Helper.Converter.EditorToHtml.Paragraph do + @moduledoc false + + use GroupherServerWeb.ConnCase, async: true + + alias Helper.Metric + alias Helper.Converter.EditorToHtml, as: Parser + + @clazz Metric.Article.class_names(:html) + + describe "[paragraph block]" do + @editor_json %{ + "time" => 1_567_250_876_713, + "blocks" => [ + %{ + "type" => "paragraph", + "data" => %{ + "text" => "paragraph content" + } + } + ], + "version" => "2.15.0" + } + @tag :wip + test "paragraph parse should work" do + {:ok, editor_string} = Jason.encode(@editor_json) + {:ok, converted} = Parser.to_html(editor_string) + + assert converted == "

paragraph content

" + end + end +end diff --git a/test/helper/converter/md_to_editor_test.exs b/test/helper/converter/md_to_editor_test.exs index 11d825d17..66233fc97 100644 --- a/test/helper/converter/md_to_editor_test.exs +++ b/test/helper/converter/md_to_editor_test.exs @@ -2,8 +2,6 @@ defmodule GroupherServer.Test.Helper.Converter.MdToEditor do @moduledoc """ parse markdown string to editorjs's json format """ - import Helper.Utils, only: [get_config: 2] - use GroupherServerWeb.ConnCase, async: true alias Helper.Metric From 7392f795c37c1b56b0fc0a638a09055ac39ab824 Mon Sep 17 00:00:00 2001 From: mydearxym Date: Wed, 10 Feb 2021 12:20:19 +0800 Subject: [PATCH 06/23] test(editor-parse): clean up imports --- lib/helper/converter/editor_to_html/header.ex | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/lib/helper/converter/editor_to_html/header.ex b/lib/helper/converter/editor_to_html/header.ex index 047ef73bc..766bf77a4 100644 --- a/lib/helper/converter/editor_to_html/header.ex +++ b/lib/helper/converter/editor_to_html/header.ex @@ -5,13 +5,8 @@ defmodule Helper.Converter.EditorToHtml.Header do see https://editorjs.io/ """ require Helper.Converter.EditorToHTML.ErrorHint, as: ErrorHint - import Helper.Converter.EditorGuards - - alias Helper.Converter.{EditorToHtml, HtmlSanitizer} - alias Helper.{Metric, Utils} - - alias EditorToHtml.Assets.{DelimiterIcons} + alias Helper.Metric @clazz Metric.Article.class_names(:html) From 221c05067de346b69554d1b37cb65c8efac0483b Mon Sep 17 00:00:00 2001 From: mydearxym Date: Wed, 10 Feb 2021 12:52:05 +0800 Subject: [PATCH 07/23] test(editor): add basic paragraph block test --- lib/helper/converter/editor_guards.ex | 2 ++ lib/helper/converter/editor_to_html/index.ex | 10 ++------ .../converter/editor_to_html/paragraph.ex | 24 +++++++++++++++++++ .../editor_to_html_test/index_test.exs | 5 ++-- .../editor_to_html_test/paragraph_test.exs | 21 ++++++++++++++++ 5 files changed, 51 insertions(+), 11 deletions(-) create mode 100644 lib/helper/converter/editor_to_html/paragraph.ex diff --git a/lib/helper/converter/editor_guards.ex b/lib/helper/converter/editor_guards.ex index 53d162bb5..3a3f465e5 100644 --- a/lib/helper/converter/editor_guards.ex +++ b/lib/helper/converter/editor_guards.ex @@ -17,4 +17,6 @@ defmodule Helper.Converter.EditorGuards do defguard is_valid_header(text, level, eyebrow_title, footer_title) when is_binary(text) and level in @support_header_levels and is_binary(eyebrow_title) and is_binary(footer_title) + + defguard is_valid_paragraph(text) when is_binary(text) end diff --git a/lib/helper/converter/editor_to_html/index.ex b/lib/helper/converter/editor_to_html/index.ex index f18d29f54..ef96cce94 100644 --- a/lib/helper/converter/editor_to_html/index.ex +++ b/lib/helper/converter/editor_to_html/index.ex @@ -6,14 +6,13 @@ defmodule Helper.Converter.EditorToHtml do """ require Helper.Converter.EditorToHTML.ErrorHint, as: ErrorHint require Helper.Converter.EditorToHtml.Header, as: Header + require Helper.Converter.EditorToHtml.Paragraph, as: Paragraph import Helper.Converter.EditorGuards alias Helper.Converter.{EditorToHtml, HtmlSanitizer} alias Helper.{Metric, Utils} - # alias EditorToHtml.{Header} - alias EditorToHtml.Assets.{DelimiterIcons} @clazz Metric.Article.class_names(:html) @@ -44,12 +43,7 @@ defmodule Helper.Converter.EditorToHtml do end Header.parse_block() - - defp parse_block(%{"type" => "paragraph", "data" => data}) do - text = get_in(data, ["text"]) - - "

#{text}

" - end + Paragraph.parse_block() defp parse_block(%{"type" => "image", "data" => data}) do url = get_in(data, ["file", "url"]) diff --git a/lib/helper/converter/editor_to_html/paragraph.ex b/lib/helper/converter/editor_to_html/paragraph.ex new file mode 100644 index 000000000..bdbe7e6b6 --- /dev/null +++ b/lib/helper/converter/editor_to_html/paragraph.ex @@ -0,0 +1,24 @@ +defmodule Helper.Converter.EditorToHtml.Paragraph do + @moduledoc """ + parse editor.js's json data to raw html and sanitize it + + see https://editorjs.io/ + """ + + require Helper.Converter.EditorToHTML.ErrorHint, as: ErrorHint + import Helper.Converter.EditorGuards + + # alias Helper.Metric + # @clazz Metric.Article.class_names(:html) + + defmacro parse_block do + quote do + defp parse_block(%{"type" => "paragraph", "data" => %{"text" => text}}) + when is_valid_paragraph(text) do + "

#{text}

" + end + + ErrorHint.watch("paragraph", "text") + end + end +end diff --git a/test/helper/converter/editor_to_html_test/index_test.exs b/test/helper/converter/editor_to_html_test/index_test.exs index e9a4a0cc9..4eb3c2cb1 100644 --- a/test/helper/converter/editor_to_html_test/index_test.exs +++ b/test/helper/converter/editor_to_html_test/index_test.exs @@ -2,11 +2,10 @@ defmodule GroupherServer.Test.Helper.Converter.EditorToHtml do @moduledoc false use GroupherServerWeb.ConnCase, async: true - - alias Helper.Metric alias Helper.Converter.EditorToHtml, as: Parser - @clazz Metric.Article.class_names(:html) + # alias Helper.Metric + # @clazz Metric.Article.class_names(:html) @real_editor_data ~S({ "time" : 1567250876713, diff --git a/test/helper/converter/editor_to_html_test/paragraph_test.exs b/test/helper/converter/editor_to_html_test/paragraph_test.exs index c84e4b44b..e8eb99766 100644 --- a/test/helper/converter/editor_to_html_test/paragraph_test.exs +++ b/test/helper/converter/editor_to_html_test/paragraph_test.exs @@ -28,5 +28,26 @@ defmodule GroupherServer.Test.Helper.Converter.EditorToHtml.Paragraph do assert converted == "

paragraph content

" end + + @editor_json %{ + "time" => 1_567_250_876_713, + "blocks" => [ + %{ + "type" => "paragraph", + "data" => %{ + "text" => [] + } + } + ], + "version" => "2.15.0" + } + @tag :wip + test "invalid paragraph should have invalid hint" do + {:ok, editor_string} = Jason.encode(@editor_json) + {:ok, converted} = Parser.to_html(editor_string) + + assert converted == + "
[invalid-block] paragraph:text
" + end end end From e074944241fde618aab8ff37b0fe1c0541c12546 Mon Sep 17 00:00:00 2001 From: mydearxym Date: Wed, 10 Feb 2021 21:46:49 +0800 Subject: [PATCH 08/23] test(editor): using macro && clean up --- .../converter/assets/delimiter_icons.ex | 2 +- .../converter/editor_to_html/error_hint.ex | 4 +-- .../guards.ex} | 2 +- lib/helper/converter/editor_to_html/header.ex | 18 ++++++++----- lib/helper/converter/editor_to_html/index.ex | 26 +++++++++++-------- .../converter/editor_to_html/paragraph.ex | 17 ++++++------ .../editor_to_html_test/header_test.exs | 13 ++-------- .../editor_to_html_test/index_test.exs | 4 +-- .../editor_to_html_test/paragraph_test.exs | 4 +-- test/helper/converter/md_to_editor_test.exs | 4 +-- 10 files changed, 46 insertions(+), 48 deletions(-) rename lib/helper/converter/{editor_guards.ex => editor_to_html/guards.ex} (94%) diff --git a/lib/helper/converter/assets/delimiter_icons.ex b/lib/helper/converter/assets/delimiter_icons.ex index 8b6b44ad3..485934230 100644 --- a/lib/helper/converter/assets/delimiter_icons.ex +++ b/lib/helper/converter/assets/delimiter_icons.ex @@ -1,4 +1,4 @@ -defmodule Helper.Converter.EditorToHtml.Assets.DelimiterIcons do +defmodule Helper.Converter.EditorToHTML.Assets.DelimiterIcons do @moduledoc """ svg icons for delimiter block NOTE: those svg should be sync with frontend svg diff --git a/lib/helper/converter/editor_to_html/error_hint.ex b/lib/helper/converter/editor_to_html/error_hint.ex index d2abd2d67..2e8d4275d 100644 --- a/lib/helper/converter/editor_to_html/error_hint.ex +++ b/lib/helper/converter/editor_to_html/error_hint.ex @@ -6,7 +6,7 @@ defmodule Helper.Converter.EditorToHTML.ErrorHint do defmacro watch(type, field) do quote do - @doc "give error hint when #{unquote(field)} is invalid type" + # "give error hint when #{unquote(field)} is invalid type" defp parse_block(%{ "type" => "#{unquote(type)}", "data" => %{ @@ -20,7 +20,7 @@ defmodule Helper.Converter.EditorToHTML.ErrorHint do defmacro watch(type, field1, field2) do quote do - @doc "give error hint when #{unquote(field1)} or #{unquote(field2)} is invalid type" + # "give error hint when #{unquote(field1)} or #{unquote(field2)} is invalid type" defp parse_block(%{ "type" => "#{unquote(type)}", "data" => %{ diff --git a/lib/helper/converter/editor_guards.ex b/lib/helper/converter/editor_to_html/guards.ex similarity index 94% rename from lib/helper/converter/editor_guards.ex rename to lib/helper/converter/editor_to_html/guards.ex index 3a3f465e5..ff3e2a57f 100644 --- a/lib/helper/converter/editor_guards.ex +++ b/lib/helper/converter/editor_to_html/guards.ex @@ -1,4 +1,4 @@ -defmodule Helper.Converter.EditorGuards do +defmodule Helper.Converter.EditorToHTML.Guards do @moduledoc """ guards for incoming editor json diff --git a/lib/helper/converter/editor_to_html/header.ex b/lib/helper/converter/editor_to_html/header.ex index 766bf77a4..2ac91b796 100644 --- a/lib/helper/converter/editor_to_html/header.ex +++ b/lib/helper/converter/editor_to_html/header.ex @@ -1,17 +1,21 @@ -defmodule Helper.Converter.EditorToHtml.Header do +defmodule Helper.Converter.EditorToHTML.Header do @moduledoc """ - parse editor.js's json data to raw html and sanitize it + parse editor.js's header block see https://editorjs.io/ """ - require Helper.Converter.EditorToHTML.ErrorHint, as: ErrorHint - import Helper.Converter.EditorGuards - alias Helper.Metric - @clazz Metric.Article.class_names(:html) + # @behaviour Helper.Converter.EditorToHTML.Parser - defmacro parse_block do + defmacro __using__(_opts) do quote do + require Helper.Converter.EditorToHTML.ErrorHint, as: ErrorHint + import Helper.Converter.EditorToHTML.Guards + + alias Helper.Metric + + @clazz Metric.Article.class_names(:html) + defp parse_block(%{ "type" => "header", "data" => diff --git a/lib/helper/converter/editor_to_html/index.ex b/lib/helper/converter/editor_to_html/index.ex index ef96cce94..691b3a19d 100644 --- a/lib/helper/converter/editor_to_html/index.ex +++ b/lib/helper/converter/editor_to_html/index.ex @@ -1,19 +1,26 @@ -defmodule Helper.Converter.EditorToHtml do +# defmodule Helper.Converter.EditorToHTML.Parser do +# @moduledoc false + +# # TODO: map should be editor_block +# @callback parse_block(editor_json :: Map.t()) :: String.t() +# end + +defmodule Helper.Converter.EditorToHTML do @moduledoc """ parse editor.js's json data to raw html and sanitize it see https://editorjs.io/ """ - require Helper.Converter.EditorToHTML.ErrorHint, as: ErrorHint - require Helper.Converter.EditorToHtml.Header, as: Header - require Helper.Converter.EditorToHtml.Paragraph, as: Paragraph - import Helper.Converter.EditorGuards + use Helper.Converter.EditorToHTML.Header + use Helper.Converter.EditorToHTML.Paragraph - alias Helper.Converter.{EditorToHtml, HtmlSanitizer} + import Helper.Converter.EditorToHTML.Guards + + alias Helper.Converter.{EditorToHTML, HtmlSanitizer} alias Helper.{Metric, Utils} - alias EditorToHtml.Assets.{DelimiterIcons} + alias EditorToHTML.Assets.{DelimiterIcons} @clazz Metric.Article.class_names(:html) @@ -31,7 +38,7 @@ defmodule Helper.Converter.EditorToHtml do end end - @desc "used for markdown ast to editor" + @doc "used for markdown ast to editor" def to_html(editor_blocks) when is_list(editor_blocks) do content = Enum.reduce(editor_blocks, "", fn block, acc -> @@ -42,9 +49,6 @@ defmodule Helper.Converter.EditorToHtml do {:ok, "
#{content}
"} end - Header.parse_block() - Paragraph.parse_block() - defp parse_block(%{"type" => "image", "data" => data}) do url = get_in(data, ["file", "url"]) diff --git a/lib/helper/converter/editor_to_html/paragraph.ex b/lib/helper/converter/editor_to_html/paragraph.ex index bdbe7e6b6..91be57a3d 100644 --- a/lib/helper/converter/editor_to_html/paragraph.ex +++ b/lib/helper/converter/editor_to_html/paragraph.ex @@ -1,18 +1,17 @@ -defmodule Helper.Converter.EditorToHtml.Paragraph do +defmodule Helper.Converter.EditorToHTML.Paragraph do @moduledoc """ - parse editor.js's json data to raw html and sanitize it + parse editor.js's paragraph block see https://editorjs.io/ """ + defmacro __using__(_opts) do + quote do + require Helper.Converter.EditorToHTML.ErrorHint, as: ErrorHint + import Helper.Converter.EditorToHTML.Guards - require Helper.Converter.EditorToHTML.ErrorHint, as: ErrorHint - import Helper.Converter.EditorGuards - - # alias Helper.Metric - # @clazz Metric.Article.class_names(:html) + # alias Helper.Metric + # @clazz Metric.Article.class_names(:html) - defmacro parse_block do - quote do defp parse_block(%{"type" => "paragraph", "data" => %{"text" => text}}) when is_valid_paragraph(text) do "

#{text}

" diff --git a/test/helper/converter/editor_to_html_test/header_test.exs b/test/helper/converter/editor_to_html_test/header_test.exs index c36553f02..62dc774d1 100644 --- a/test/helper/converter/editor_to_html_test/header_test.exs +++ b/test/helper/converter/editor_to_html_test/header_test.exs @@ -1,10 +1,10 @@ -defmodule GroupherServer.Test.Helper.Converter.EditorToHtml.Header do +defmodule GroupherServer.Test.Helper.Converter.EditorToHTML.Header do @moduledoc false use GroupherServerWeb.ConnCase, async: true alias Helper.Metric - alias Helper.Converter.EditorToHtml, as: Parser + alias Helper.Converter.EditorToHTML, as: Parser @clazz Metric.Article.class_names(:html) @@ -138,14 +138,5 @@ defmodule GroupherServer.Test.Helper.Converter.EditorToHtml.Header do assert converted == "
[invalid-block] header:text or level
" end - - test "code block should avoid potential xss script attack" do - {:ok, converted} = Parser.to_html(@real_editor_data) - - safe_script = - "
<script>evil scripts</script>
" - - assert converted |> String.contains?(safe_script) - end end end diff --git a/test/helper/converter/editor_to_html_test/index_test.exs b/test/helper/converter/editor_to_html_test/index_test.exs index 4eb3c2cb1..40d8f89ab 100644 --- a/test/helper/converter/editor_to_html_test/index_test.exs +++ b/test/helper/converter/editor_to_html_test/index_test.exs @@ -1,8 +1,8 @@ -defmodule GroupherServer.Test.Helper.Converter.EditorToHtml do +defmodule GroupherServer.Test.Helper.Converter.EditorToHTML do @moduledoc false use GroupherServerWeb.ConnCase, async: true - alias Helper.Converter.EditorToHtml, as: Parser + alias Helper.Converter.EditorToHTML, as: Parser # alias Helper.Metric # @clazz Metric.Article.class_names(:html) diff --git a/test/helper/converter/editor_to_html_test/paragraph_test.exs b/test/helper/converter/editor_to_html_test/paragraph_test.exs index e8eb99766..0f52e8d94 100644 --- a/test/helper/converter/editor_to_html_test/paragraph_test.exs +++ b/test/helper/converter/editor_to_html_test/paragraph_test.exs @@ -1,10 +1,10 @@ -defmodule GroupherServer.Test.Helper.Converter.EditorToHtml.Paragraph do +defmodule GroupherServer.Test.Helper.Converter.EditorToHTML.Paragraph do @moduledoc false use GroupherServerWeb.ConnCase, async: true alias Helper.Metric - alias Helper.Converter.EditorToHtml, as: Parser + alias Helper.Converter.EditorToHTML, as: Parser @clazz Metric.Article.class_names(:html) diff --git a/test/helper/converter/md_to_editor_test.exs b/test/helper/converter/md_to_editor_test.exs index 66233fc97..cfd3c2b9f 100644 --- a/test/helper/converter/md_to_editor_test.exs +++ b/test/helper/converter/md_to_editor_test.exs @@ -6,7 +6,7 @@ defmodule GroupherServer.Test.Helper.Converter.MdToEditor do alias Helper.Metric alias Helper.Converter.MdToEditor, as: Converter - alias Helper.Converter.EditorToHtml + alias Helper.Converter.EditorToHTML @clazz Metric.Article.class_names(:html) # alias Helper.Converter.HtmlSanitizer, as: Sanitizer @@ -247,7 +247,7 @@ defmodule GroupherServer.Test.Helper.Converter.MdToEditor do editor_blocks = Converter.parse(markdown) - {:ok, html} = EditorToHtml.to_html(editor_blocks) + {:ok, html} = EditorToHTML.to_html(editor_blocks) assert html == "

hello

this is a basic markdown text

delete me

italic me

My in-line-code-content is best

" From fd8831c166d7abedbb6cc5ea26b6c3cda6fe2d6a Mon Sep 17 00:00:00 2001 From: mydearxym Date: Mon, 15 Feb 2021 22:09:26 +0800 Subject: [PATCH 09/23] refactor(editor): re-org the editor json validte workflow --- .iex.exs | 2 + Makefile | 3 +- .../converter/editor_to_html/error_hint.ex | 35 ---- lib/helper/converter/editor_to_html/guards.ex | 22 -- lib/helper/converter/editor_to_html/header.ex | 21 +- lib/helper/converter/editor_to_html/index.ex | 15 +- lib/helper/converter/editor_to_html/list.ex | 196 ++++++++++++++++++ .../converter/editor_to_html/paragraph.ex | 8 +- .../converter/editor_to_html/validator.ex | 131 ++++++++++++ lib/helper/metric/article.ex | 4 + .../{PublicIpPlug.ex => public_ip_plug.ex} | 0 lib/helper/utils.ex | 16 ++ lib/helper/validate_by_schema.ex | 105 ++++++++++ mix.lock | 2 + .../editor_to_html_test/header_test.exs | 45 +++- .../editor_to_html_test/index_test.exs | 35 +++- .../editor_to_html_test/list_test.exs | 104 ++++++++++ test/helper/utils_test.exs | 57 +++++ 18 files changed, 693 insertions(+), 108 deletions(-) create mode 100644 .iex.exs delete mode 100644 lib/helper/converter/editor_to_html/error_hint.ex delete mode 100644 lib/helper/converter/editor_to_html/guards.ex create mode 100644 lib/helper/converter/editor_to_html/list.ex create mode 100644 lib/helper/converter/editor_to_html/validator.ex rename lib/helper/{PublicIpPlug.ex => public_ip_plug.ex} (100%) create mode 100644 lib/helper/validate_by_schema.ex create mode 100644 test/helper/converter/editor_to_html_test/list_test.exs diff --git a/.iex.exs b/.iex.exs new file mode 100644 index 000000000..88e4c63a5 --- /dev/null +++ b/.iex.exs @@ -0,0 +1,2 @@ +# see: https://github.com/blackode/elixir-tips#loading-project-module-aliases-iexexs +alias Helper.Converter.EditorToHTML diff --git a/Makefile b/Makefile index 2d629524c..425a392f9 100644 --- a/Makefile +++ b/Makefile @@ -171,9 +171,8 @@ test.watch.wip: # test.watch not work now, see: https://github.com/lpil/mix-test.watch/issues/116 # mix test.watch --only wip --stale test.watch.wip2: - mix test --listen-on-stdin --stale --trace --only wip2 + mix test --listen-on-stdin --stale --only wip2 # mix test.watch --only wip2 - mix test.watch --only wip2 test.watch.bug: mix test.watch --only bug test.report: diff --git a/lib/helper/converter/editor_to_html/error_hint.ex b/lib/helper/converter/editor_to_html/error_hint.ex deleted file mode 100644 index 2e8d4275d..000000000 --- a/lib/helper/converter/editor_to_html/error_hint.ex +++ /dev/null @@ -1,35 +0,0 @@ -defmodule Helper.Converter.EditorToHTML.ErrorHint do - @moduledoc """ - - see https://stackoverflow.com/a/33052969/4050784 - """ - - defmacro watch(type, field) do - quote do - # "give error hint when #{unquote(field)} is invalid type" - defp parse_block(%{ - "type" => "#{unquote(type)}", - "data" => %{ - "#{unquote(field)}" => _ - } - }) do - invalid_hint("#{unquote(type)}", "#{unquote(field)}") - end - end - end - - defmacro watch(type, field1, field2) do - quote do - # "give error hint when #{unquote(field1)} or #{unquote(field2)} is invalid type" - defp parse_block(%{ - "type" => "#{unquote(type)}", - "data" => %{ - "#{unquote(field1)}" => _, - "#{unquote(field2)}" => _ - } - }) do - invalid_hint("#{unquote(type)}", "#{unquote(field1)} or #{unquote(field2)}") - end - end - end -end diff --git a/lib/helper/converter/editor_to_html/guards.ex b/lib/helper/converter/editor_to_html/guards.ex deleted file mode 100644 index ff3e2a57f..000000000 --- a/lib/helper/converter/editor_to_html/guards.ex +++ /dev/null @@ -1,22 +0,0 @@ -defmodule Helper.Converter.EditorToHTML.Guards do - @moduledoc """ - guards for incoming editor json - - map_get is not support in current version, so we have to pass each arg for guard - see: https://elixirforum.com/t/discussion-incorporating-erlang-otp-21-map-guards-in-elixir/14816 - """ - @support_header_levels [1, 2, 3] - - defguard is_valid_header(text, level) - when is_binary(text) and level in @support_header_levels - - @doc "check if eyebowTitle OR footerTitle are valid" - defguard is_valid_header(text, level, subtitle) - when is_binary(text) and level in @support_header_levels and is_binary(subtitle) - - defguard is_valid_header(text, level, eyebrow_title, footer_title) - when is_binary(text) and level in @support_header_levels and is_binary(eyebrow_title) and - is_binary(footer_title) - - defguard is_valid_paragraph(text) when is_binary(text) -end diff --git a/lib/helper/converter/editor_to_html/header.ex b/lib/helper/converter/editor_to_html/header.ex index 2ac91b796..b021f1f7e 100644 --- a/lib/helper/converter/editor_to_html/header.ex +++ b/lib/helper/converter/editor_to_html/header.ex @@ -9,9 +9,6 @@ defmodule Helper.Converter.EditorToHTML.Header do defmacro __using__(_opts) do quote do - require Helper.Converter.EditorToHTML.ErrorHint, as: ErrorHint - import Helper.Converter.EditorToHTML.Guards - alias Helper.Metric @clazz Metric.Article.class_names(:html) @@ -25,8 +22,7 @@ defmodule Helper.Converter.EditorToHTML.Header do "eyebrowTitle" => eyebrow_title, "footerTitle" => footer_title } = data - }) - when is_valid_header(text, level, eyebrow_title, footer_title) do + }) do """
#{eyebrow_title}
@@ -44,8 +40,7 @@ defmodule Helper.Converter.EditorToHTML.Header do "level" => level, "eyebrowTitle" => eyebrow_title } = data - }) - when is_valid_header(text, level, eyebrow_title) do + }) do """
#{eyebrow_title}
@@ -54,8 +49,6 @@ defmodule Helper.Converter.EditorToHTML.Header do """ end - ErrorHint.watch("header", "eyebrowTitle") - defp parse_block(%{ "type" => "header", "data" => @@ -64,8 +57,7 @@ defmodule Helper.Converter.EditorToHTML.Header do "level" => level, "footerTitle" => footer_title } = data - }) - when is_valid_header(text, level, footer_title) do + }) do """
#{text} @@ -74,20 +66,15 @@ defmodule Helper.Converter.EditorToHTML.Header do """ end - ErrorHint.watch("header", "footerTitle") - defp parse_block(%{ "type" => "header", "data" => %{ "text" => text, "level" => level } - }) - when is_valid_header(text, level) do + }) do "#{text}" end - - ErrorHint.watch("header", "text", "level") end end end diff --git a/lib/helper/converter/editor_to_html/index.ex b/lib/helper/converter/editor_to_html/index.ex index 691b3a19d..0e7192fec 100644 --- a/lib/helper/converter/editor_to_html/index.ex +++ b/lib/helper/converter/editor_to_html/index.ex @@ -14,9 +14,9 @@ defmodule Helper.Converter.EditorToHTML do use Helper.Converter.EditorToHTML.Header use Helper.Converter.EditorToHTML.Paragraph + use Helper.Converter.EditorToHTML.List - import Helper.Converter.EditorToHTML.Guards - + alias Helper.Converter.EditorToHTML.Validator alias Helper.Converter.{EditorToHTML, HtmlSanitizer} alias Helper.{Metric, Utils} @@ -27,7 +27,7 @@ defmodule Helper.Converter.EditorToHTML do @spec to_html(binary | maybe_improper_list) :: false | {:ok, <<_::64, _::_*8>>} def to_html(string) when is_binary(string) do with {:ok, parsed} = string_to_json(string), - true <- valid_editor_data?(parsed) do + {:ok, _} <- Validator.is_valid(parsed) do content = Enum.reduce(parsed["blocks"], "", fn block, acc -> clean_html = block |> parse_block |> HtmlSanitizer.sanitize() @@ -136,13 +136,4 @@ defmodule Helper.Converter.EditorToHTML do end def string_to_json(string), do: Jason.decode(string) - - defp valid_editor_data?(map) when is_map(map) do - Map.has_key?(map, "time") and - Map.has_key?(map, "version") and - Map.has_key?(map, "blocks") and - is_list(map["blocks"]) and - is_binary(map["version"]) and - is_integer(map["time"]) - end end diff --git a/lib/helper/converter/editor_to_html/list.ex b/lib/helper/converter/editor_to_html/list.ex new file mode 100644 index 000000000..f83c82d22 --- /dev/null +++ b/lib/helper/converter/editor_to_html/list.ex @@ -0,0 +1,196 @@ +defmodule Helper.Converter.EditorToHTML.List do + @moduledoc """ + parse editor.js's list-like block (include checklist order/unorder list) + + see https://editorjs.io/ + """ + + # @behaviour Helper.Converter.EditorToHTML.Parser + + defmacro __using__(_opts) do + quote do + alias Helper.Metric + + @clazz Metric.Article.class_names(:html) + + defp parse_block(%{ + "type" => "list", + "data" => + %{ + "mode" => "checklist", + "items" => [ + %{ + "checked" => checked, + "hideLabel" => hide_label, + "indent" => indent, + "label" => label, + "labelType" => label_type, + "text" => text + } + ] + } = data + }) do + """ +
+ hello list +
+ """ + + #
+ #
#{eyebrow_title}
+ # #{text} + #
#{footer_title}
+ #
+ end + + defp parse_block(%{ + "type" => "list", + "data" => + %{ + "text" => text, + "level" => level, + "eyebrowTitle" => eyebrow_title + } = data + }) do + """ +
+
#{eyebrow_title}
+ #{text} +
+ """ + end + + defp parse_block(%{ + "type" => "list", + "data" => + %{ + "text" => text, + "level" => level, + "footerTitle" => footer_title + } = data + }) do + """ +
+ #{text} +
#{footer_title}
+
+ """ + end + + defp parse_block(%{ + "type" => "list", + "data" => %{ + "text" => text, + "level" => level + } + }) do + "#{text}" + end + end + end +end + +# { +# type: "list", +# data: { +# type: "checklist", +# items: [ +# { +# checked: true, +# hideLabel: false, +# indent: 0, +# label: '标签', +# labelType: 'default', +# text: "content 1.", +# }, +# { +# checked: false, +# hideLabel: false, +# indent: 1, +# label: '完成', +# labelType: 'green', +# text: "content 1.1", +# }, +# { +# checked: false, +# hideLabel: false, +# indent: 1, +# label: '未完成', +# labelType: 'red', +# text: "content 1.2", +# }, +# { +# checked: false, +# hideLabel: false, +# indent: 0, +# label: '完成', +# labelType: 'green', +# text: "content 2.", +# }, +# { +# checked: false, +# hideLabel: false, +# indent: 1, +# label: '标签', +# labelType: 'default', +# text: "content 2.1", +# }, +# { +# checked: false, +# hideLabel: false, +# indent: 2, +# label: '标签', +# labelType: 'default', +# text: "content 2.1.1", +# }, +# { +# checked: false, +# hideLabel: false, +# indent: 2, +# label: '未完成', +# labelType: 'red', +# text: "content 2.1.2", +# }, +# { +# checked: false, +# hideLabel: false, +# indent: 3, +# label: '未完成', +# labelType: 'red', +# text: "content 2.1.2.1", +# }, +# { +# checked: false, +# hideLabel: false, +# indent: 2, +# label: '完成', +# labelType: 'green', +# text: "content 2.1.3", +# }, +# { +# checked: false, +# hideLabel: false, +# indent: 3, +# label: '标签', +# labelType: 'default', +# text: "content 2.1.3.1", +# }, +# { +# checked: false, +# hideLabel: false, +# indent: 3, +# label: '完成', +# labelType: 'green', +# text: "content 2.1.3.2", +# }, +# { +# checked: false, +# hideLabel: false, +# indent: 0, +# label: '未完成', +# labelType: 'red', +# text: "content 3.", +# }, +# ] +# }, +# } diff --git a/lib/helper/converter/editor_to_html/paragraph.ex b/lib/helper/converter/editor_to_html/paragraph.ex index 91be57a3d..f363d0c57 100644 --- a/lib/helper/converter/editor_to_html/paragraph.ex +++ b/lib/helper/converter/editor_to_html/paragraph.ex @@ -6,18 +6,12 @@ defmodule Helper.Converter.EditorToHTML.Paragraph do """ defmacro __using__(_opts) do quote do - require Helper.Converter.EditorToHTML.ErrorHint, as: ErrorHint - import Helper.Converter.EditorToHTML.Guards - # alias Helper.Metric # @clazz Metric.Article.class_names(:html) - defp parse_block(%{"type" => "paragraph", "data" => %{"text" => text}}) - when is_valid_paragraph(text) do + defp parse_block(%{"type" => "paragraph", "data" => %{"text" => text}}) do "

#{text}

" end - - ErrorHint.watch("paragraph", "text") end end end diff --git a/lib/helper/converter/editor_to_html/validator.ex b/lib/helper/converter/editor_to_html/validator.ex new file mode 100644 index 000000000..938d0973e --- /dev/null +++ b/lib/helper/converter/editor_to_html/validator.ex @@ -0,0 +1,131 @@ +defmodule Helper.Converter.EditorToHTML.Validator do + @moduledoc false + + alias Helper.{Utils, ValidateBySchema} + + @valid_header_level [1, 2, 3] + + @valid_list_mode ["checklist", "order_list", "unorder_list"] + @valid_list_label_type ["success", "done", "todo"] + @valid_list_indent [0, 1, 2, 3, 4] + + # atoms dynamically and atoms are not + # garbage-collected. Therefore, string should not be an untrusted value, such as + # input received from a socket or during a web request. Consider using + # to_existing_atom/1 instead + # keys_to_atoms is using to_existing_atom under the hook, so we have to pre-define the + # trusted atoms + tursted_atoms = [ + # common + :text, + # header + :level, + :eyebrowTitle, + :footerTitle, + # list + :hideLabel, + :labelType, + :indent, + :checked, + :label + ] + + Enum.each(tursted_atoms, fn atom -> _ = atom end) + + def is_valid(map) when is_map(map) do + with atom_map <- Utils.keys_to_atoms(map), + true <- is_valid_editorjs_fmt(atom_map) do + blocks = atom_map.blocks + + try do + validate_blocks(blocks) + rescue + e in MatchError -> format_parse_error(e) + e in RuntimeError -> format_parse_error(e) + e -> format_parse_error() + end + else + false -> + {:error, "invalid editor json format"} + + _ -> + {:error, "invalid editor json"} + end + end + + defp validate_blocks([]), do: {:ok, :pass} + + defp validate_blocks(blocks) do + Enum.each(blocks, fn block -> + {:ok, _} = validate_block(block) + end) + + {:ok, :pass} + end + + defp validate_block(%{type: "list", data: %{mode: mode, items: items} = data}) + when mode in @valid_list_mode and is_list(items) do + # mode_schema = %{mode: [enum: @valid_list_mode]} + # {:ok, _} = ValidateBySchema.cast(mode_schema, data) + item_schema = %{ + checked: [:boolean], + hideLabel: [:boolean], + label: [:string], + labelType: [enum: @valid_list_label_type], + indent: [enum: @valid_list_indent], + text: [:string] + } + + Enum.each(items, fn item -> + {:ok, _} = ValidateBySchema.cast(item_schema, item) + end) + + {:ok, :pass} + end + + defp validate_block(%{type: "header", data: %{text: text, level: level} = data}) do + schema = %{ + text: [:string], + level: [enum: @valid_header_level], + eyebrowTitle: [:string, required: false], + footerTitle: [:string, required: false] + } + + {:ok, _} = ValidateBySchema.cast(schema, data) + end + + # defp validate_block(%{type: type, data: _}), do: raise("invalid data for #{type} block") + defp validate_block(%{type: type}), do: raise("undown #{type} block") + defp validate_block(_), do: raise("undown block") + + # check if the given map has the right key-value fmt of the editorjs structure + defp is_valid_editorjs_fmt(map) when is_map(map) do + Map.has_key?(map, :time) and + Map.has_key?(map, :version) and + Map.has_key?(map, :blocks) and + is_list(map.blocks) and + is_binary(map.version) and + is_integer(map.time) + end + + # defp format_parse_error({:error, errors}) do + defp format_parse_error(%MatchError{term: {:error, errors}}) do + message = + Enum.reduce(errors, "", fn x, acc -> + case String.trim(acc) do + "" -> x.message + _ -> acc <> " ; " <> x.message + end + end) + + {:error, message} + end + + # defp format_parse_error(%{message: message}), do: {:error, message} + defp format_parse_error(%{message: message}) do + {:error, message} + end + + # defp format_parse_error(e), do: IO.inspect(e, label: "e") + defp format_parse_error(), do: {:error, "undown validate error"} +end diff --git a/lib/helper/metric/article.ex b/lib/helper/metric/article.ex index b9b1235e8..c1f265d5b 100644 --- a/lib/helper/metric/article.ex +++ b/lib/helper/metric/article.ex @@ -20,6 +20,10 @@ defmodule Helper.Metric.Article do wrapper: "header-wrapper", eyebrow_title: "eyebrow-title", footer_title: "footer-title" + }, + # list + list: %{ + wrapper: "list-wrapper" } } end diff --git a/lib/helper/PublicIpPlug.ex b/lib/helper/public_ip_plug.ex similarity index 100% rename from lib/helper/PublicIpPlug.ex rename to lib/helper/public_ip_plug.ex diff --git a/lib/helper/utils.ex b/lib/helper/utils.ex index f52e5c339..fd376d586 100644 --- a/lib/helper/utils.ex +++ b/lib/helper/utils.ex @@ -94,6 +94,22 @@ defmodule Helper.Utils do map |> Enum.reduce(%{}, fn {key, val}, acc -> Map.put(acc, to_string(key), val) end) end + @doc """ + see https://stackoverflow.com/a/61559842/4050784 + adjust it for map keys from atom to string + """ + def keys_to_atoms(json) when is_map(json) do + Map.new(json, &reduce_keys_to_atoms/1) + end + + def reduce_keys_to_atoms({key, val}) when is_map(val), + do: {String.to_existing_atom(key), keys_to_atoms(val)} + + def reduce_keys_to_atoms({key, val}) when is_list(val), + do: {String.to_existing_atom(key), Enum.map(val, &keys_to_atoms(&1))} + + def reduce_keys_to_atoms({key, val}), do: {String.to_existing_atom(key), val} + @doc """ see https://stackoverflow.com/a/61559842/4050784 adjust it for map keys from atom to string diff --git a/lib/helper/validate_by_schema.ex b/lib/helper/validate_by_schema.ex new file mode 100644 index 000000000..e3cedbf52 --- /dev/null +++ b/lib/helper/validate_by_schema.ex @@ -0,0 +1,105 @@ +defmodule Helper.ValidateBySchema do + @moduledoc """ + validate json data by given schema, mostly used in editorjs validator + + currently support boolean / string / number / enum + """ + + @doc """ + cast data by given schema + + e.g: + schema = %{ + checked: [:boolean], + hideLabel: [:boolean], + label: [:string], + labelType: [:string], + indent: [enum: [0, 1, 2, 3, 4]], + text: [:string] + } + + data = %{checked: true, label: "done"} + ValidateBySchema.cast(schema, data) + """ + def cast(schema, data) do + schema_fields = Map.keys(schema) + + errors_info = + Enum.reduce(schema_fields, [], fn field, acc -> + value = get_in(data, [field]) + field_schema = get_in(schema, [field]) + + case match(field, value, field_schema) do + {:error, error} -> + acc ++ [error] + + _ -> + acc + end + end) + + case errors_info do + [] -> {:ok, :pass} + _ -> {:error, errors_info} + end + end + + # boolean field + defp match(field, nil, [:boolean, required: false]), do: done(field, nil) + + defp match(field, value, [:boolean, required: false]) when is_boolean(value) do + done(field, value) + end + + defp match(field, value, [:boolean]) when is_boolean(value), do: done(field, value) + defp match(field, value, [:boolean, required: false]), do: error(field, value, :boolean) + defp match(field, value, [:boolean]), do: error(field, value, :boolean) + + # string field + defp match(field, nil, [:string, required: false]), do: done(field, nil) + + defp match(field, value, [:string, required: false]) when is_binary(value) do + done(field, value) + end + + defp match(field, value, [:string]) when is_binary(value), do: done(field, value) + defp match(field, value, [:string, required: false]), do: error(field, value, :string) + defp match(field, value, [:string]), do: error(field, value, :string) + + # number field + defp match(field, nil, [:number, required: false]), do: done(field, nil) + + defp match(field, value, [:number, required: false]) when is_number(value) do + done(field, value) + end + + defp match(field, value, [:number]) when is_number(value), do: done(field, value) + defp match(field, value, [:number, required: false]), do: error(field, value, :number) + defp match(field, value, [:number]), do: error(field, value, :number) + + defp match(field, nil, enum: _, required: false), do: done(field, nil) + + defp match(field, value, enum: enum, required: false) do + match(field, value, enum: enum) + end + + defp match(field, value, enum: enum) do + case value in enum do + true -> + {:ok, value} + + false -> + {:error, + %{ + field: field, + message: "#{field} should be: #{enum |> Enum.join(" | ") |> to_string}" + }} + end + end + + defp done(field, value), do: {:ok, %{field: field, value: value}} + + defp error(field, value, schema) do + {:error, %{field: field, value: value, message: "#{field} should be: #{schema}"}} + end +end diff --git a/mix.lock b/mix.lock index 6bd0343af..91c426ea0 100644 --- a/mix.lock +++ b/mix.lock @@ -28,6 +28,7 @@ "ecto": {:hex, :ecto, "3.5.6", "29c77e999e471921c7ce7347732bab7bfa3e24c587640a36f17e0744d1474b8e", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "3ae1f3eaecc3e72eeb65ed43239b292bb1eaf335c7e6cea3a7fc27aadb6e93e7"}, "ecto_sql": {:hex, :ecto_sql, "3.5.4", "a9e292c40bd79fff88885f95f1ecd7b2516e09aa99c7dd0201aa84c54d2358e4", [:mix], [{:db_connection, "~> 2.2", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.5.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.3.0 or ~> 0.4.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.15.0 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "1fff1a28a898d7bbef263f1f3ea425b04ba9f33816d843238c84eff883347343"}, "elixir_make": {:hex, :elixir_make, "0.4.0", "992f38fabe705bb45821a728f20914c554b276838433349d4f2341f7a687cddf", [], [], "hexpm"}, + "elixir_uuid": {:hex, :elixir_uuid, "1.2.1", "dce506597acb7e6b0daeaff52ff6a9043f5919a4c3315abb4143f0b00378c097", [], [], "hexpm", "f7eba2ea6c3555cea09706492716b0d87397b88946e6380898c2889d68585752"}, "erlex": {:hex, :erlex, "0.2.6", "c7987d15e899c7a2f34f5420d2a2ea0d659682c06ac607572df55a43753aa12e", [:mix], [], "hexpm", "2ed2e25711feb44d52b17d2780eabf998452f6efda104877a3881c2f8c0c0c75"}, "eternal": {:hex, :eternal, "1.2.1", "d5b6b2499ba876c57be2581b5b999ee9bdf861c647401066d3eeed111d096bc4", [:mix], [], "hexpm", "b14f1dc204321429479c569cfbe8fb287541184ed040956c8862cb7a677b8406"}, "ex_unit_notifier": {:hex, :ex_unit_notifier, "0.1.4", "36a2dcab829f506e01bf17816590680dd1474407926d43e64c1263e627c364b8", [:mix], [], "hexpm", "fddf5054dd5fd2f809e837b749570baa5c9798e11d0163921baec49b7d5762f2"}, @@ -45,6 +46,7 @@ "idna": {:hex, :idna, "6.0.0", "689c46cbcdf3524c44d5f3dde8001f364cd7608a99556d8fbd8239a5798d4c10", [:rebar3], [{:unicode_util_compat, "0.4.1", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "4bdd305eb64e18b0273864920695cb18d7a2021f31a11b9c5fbcd9a253f936e2"}, "inch_ex": {:hex, :inch_ex, "2.0.0", "24268a9284a1751f2ceda569cd978e1fa394c977c45c331bb52a405de544f4de", [:mix], [{:bunt, "~> 0.2", [hex: :bunt, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "96d0ec5ecac8cf63142d02f16b7ab7152cf0f0f1a185a80161b758383c9399a8"}, "jason": {:hex, :jason, "1.1.2", "b03dedea67a99223a2eaf9f1264ce37154564de899fd3d8b9a21b1a6fd64afe7", [:mix], [{:decimal, "~> 1.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "fdf843bca858203ae1de16da2ee206f53416bbda5dc8c9e78f43243de4bc3afe"}, + "joi": {:hex, :joi, "0.1.4", "cdef436c62ddcaa596f7612d1f903c7be32050248dfc9692b6e7255a5dddd926", [:mix], [{:elixir_uuid, "~> 1.2", [hex: :elixir_uuid, repo: "hexpm", optional: false]}], "hexpm", "5ef5f35e3b716a60dfbf312ccef27b4a4543b01d9c4a59ddadca6fd688c09db6"}, "jose": {:hex, :jose, "1.11.1", "59da64010c69aad6cde2f5b9248b896b84472e99bd18f246085b7b9fe435dcdb", [:mix, :rebar3], [], "hexpm", "078f6c9fb3cd2f4cfafc972c814261a7d1e8d2b3685c0a76eb87e158efff1ac5"}, "jumper": {:hex, :jumper, "1.0.1", "3c00542ef1a83532b72269fab9f0f0c82bf23a35e27d278bfd9ed0865cecabff", [:mix], [], "hexpm", "318c59078ac220e966d27af3646026db9b5a5e6703cb2aa3e26bcfaba65b7433"}, "libring": {:hex, :libring, "1.4.0", "41246ba2f3fbc76b3971f6bce83119dfec1eee17e977a48d8a9cfaaf58c2a8d6", [:mix], [], "hexpm", "1feaf05ee886815ad047cad7ede17d6910710986148ae09cf73eee2989717b81"}, diff --git a/test/helper/converter/editor_to_html_test/header_test.exs b/test/helper/converter/editor_to_html_test/header_test.exs index 62dc774d1..0250d0a77 100644 --- a/test/helper/converter/editor_to_html_test/header_test.exs +++ b/test/helper/converter/editor_to_html_test/header_test.exs @@ -77,8 +77,8 @@ defmodule GroupherServer.Test.Helper.Converter.EditorToHTML.Header do "time" => 1_567_250_876_713, "version" => "2.15.0" } - @tag :wip - test "wrong header format data should have invalid hint" do + @tag :wip2 + test "optional field should valid properly" do json = Map.merge(@editor_json, %{ "blocks" => [ @@ -87,7 +87,7 @@ defmodule GroupherServer.Test.Helper.Converter.EditorToHTML.Header do "data" => %{ "text" => "header content", "level" => 1, - "eyebrowTitle" => [] + "eyebrowTitle" => "eyebrow title content" } } ] @@ -97,7 +97,9 @@ defmodule GroupherServer.Test.Helper.Converter.EditorToHTML.Header do {:ok, converted} = Parser.to_html(editor_string) assert converted == - "
[invalid-block] header:eyebrowTitle
" + "
\n
eyebrow title content
\n

header content

\n
\n
" json = Map.merge(@editor_json, %{ @@ -107,7 +109,7 @@ defmodule GroupherServer.Test.Helper.Converter.EditorToHTML.Header do "data" => %{ "text" => "header content", "level" => 1, - "footerTitle" => [] + "footerTitle" => "footer title content" } } ] @@ -117,8 +119,13 @@ defmodule GroupherServer.Test.Helper.Converter.EditorToHTML.Header do {:ok, converted} = Parser.to_html(editor_string) assert converted == - "
[invalid-block] header:footerTitle
" + "
\n

header content

\n
footer title content
\n
\n
" + end + @tag :wip2 + test "wrong header format data should have invalid hint" do json = Map.merge(@editor_json, %{ "blocks" => [ @@ -126,17 +133,35 @@ defmodule GroupherServer.Test.Helper.Converter.EditorToHTML.Header do "type" => "header", "data" => %{ "text" => "header content", - "level" => [] + "level" => 1, + "eyebrowTitle" => [], + "footerTitle" => true } } ] }) {:ok, editor_string} = Jason.encode(json) - {:ok, converted} = Parser.to_html(editor_string) + {:error, error} = Parser.to_html(editor_string) - assert converted == - "
[invalid-block] header:text or level
" + assert error == "eyebrowTitle should be: string ; footerTitle should be: string" + + json = + Map.merge(@editor_json, %{ + "blocks" => [ + %{ + "type" => "header", + "data" => %{ + "text" => "header content", + "level" => [] + } + } + ] + }) + + {:ok, editor_string} = Jason.encode(json) + {:error, error} = Parser.to_html(editor_string) + assert error == "level should be: 1 | 2 | 3" end end end diff --git a/test/helper/converter/editor_to_html_test/index_test.exs b/test/helper/converter/editor_to_html_test/index_test.exs index 40d8f89ab..847b38fc7 100644 --- a/test/helper/converter/editor_to_html_test/index_test.exs +++ b/test/helper/converter/editor_to_html_test/index_test.exs @@ -192,9 +192,38 @@ defmodule GroupherServer.Test.Helper.Converter.EditorToHTML do assert converted["version"] == "2.15.0" end - test "invalid string data should get error" do - string = ~S({"time":1566184478687,"blocks":[{}],"version":}) - assert {:error, converted} = Parser.string_to_json(string) + @editor_json %{ + "time" => 1_567_250_876_713, + "blocks" => [], + "version" => "2.15.0" + } + @tag :wip + test "valid editorjs json fmt should work" do + {:ok, editor_string} = Jason.encode(@editor_json) + + assert {:ok, _} = Parser.to_html(editor_string) + end + + @tag :wip + test "invalid editorjs json fmt should raise error" do + editor_json = %{ + "invalid_time" => 1_567_250_876_713, + "blocks" => [], + "version" => "2.15.0" + } + + {:ok, editor_string} = Jason.encode(editor_json) + assert {:error, "invalid editor json format"} = Parser.to_html(editor_string) + + editor_json = %{ + "time" => 1_567_250_876_713, + # invalid blocks type, should be list + "blocks" => "blocks", + "version" => "2.15.0" + } + + {:ok, editor_string} = Jason.encode(editor_json) + assert {:error, "invalid editor json format"} = Parser.to_html(editor_string) end test "real-world editor.js data should work" do diff --git a/test/helper/converter/editor_to_html_test/list_test.exs b/test/helper/converter/editor_to_html_test/list_test.exs new file mode 100644 index 000000000..f2ce2b7ea --- /dev/null +++ b/test/helper/converter/editor_to_html_test/list_test.exs @@ -0,0 +1,104 @@ +defmodule GroupherServer.Test.Helper.Converter.EditorToHTML.List do + @moduledoc false + + use GroupherServerWeb.ConnCase, async: true + + alias Helper.Metric + alias Helper.Converter.EditorToHTML, as: Parser + + @clazz Metric.Article.class_names(:html) + + describe "[list block unit]" do + @editor_json %{ + "time" => 1_567_250_876_713, + "blocks" => [ + %{ + "type" => "list", + "data" => %{ + "mode" => "checklist", + "items" => [ + %{ + "checked" => true, + "hideLabel" => "invalid", + "indent" => 5, + "label" => "label", + "labelType" => "success", + "text" => "list item" + } + ] + } + } + ], + "version" => "2.15.0" + } + @tag :wip + test "invalid list data parse should raise error message" do + {:ok, editor_string} = Jason.encode(@editor_json) + {:error, err_msg} = Parser.to_html(editor_string) + + assert err_msg == + "indent should be: 0 | 1 | 2 | 3 | 4 ; hideLabel should be: boolean" + end + + @editor_json %{ + "time" => 1_567_250_876_713, + "blocks" => [ + %{ + "type" => "list", + "data" => %{ + "mode" => "checklist", + "items" => [ + %{ + "checked" => true, + "hideLabel" => false, + "indent" => 0, + "label" => "label", + "labelType" => "success", + "text" => "list item" + } + ] + } + } + ], + "version" => "2.15.0" + } + @tag :wip + test "valid list parse should work" do + {:ok, editor_string} = Jason.encode(@editor_json) + # {:ok, converted} = Parser.to_html(editor_string) + Parser.to_html(editor_string) + + assert {:ok, converted} = Parser.to_html(editor_string) + # IO.inspect(converted, label: "->> converted") + end + + @editor_json %{ + "time" => 1_567_250_876_713, + "blocks" => [ + %{ + "type" => "list", + "data" => %{ + "mode" => "checklist", + "items" => [ + %{ + "checked" => true, + "hideLabel" => true, + "indent" => 10, + "label" => "label", + "labelType" => "success", + "text" => "list item" + } + ] + } + } + ], + "version" => "2.15.0" + } + @tag :wip + test "invalid indent field should get error" do + {:ok, editor_string} = Jason.encode(@editor_json) + # {:ok, converted} = Parser.to_html(editor_string) + assert {:error, "indent field should be: 0 | 1 | 2 | 3 | 4"} = Parser.to_html(editor_string) + end + end +end diff --git a/test/helper/utils_test.exs b/test/helper/utils_test.exs index b97c86890..ec6a42c75 100644 --- a/test/helper/utils_test.exs +++ b/test/helper/utils_test.exs @@ -23,6 +23,63 @@ defmodule GroupherServer.Test.Helper.UtilsTest do end end + describe "map keys to atom" do + test "string keys should covert to atom keys on nested map" do + atom_map = %{ + blocks: [ + %{ + data: %{ + items: [ + %{ + checked: true, + hideLabel: true, + indent: 0, + label: "label", + labelType: "success", + text: "list item" + } + ], + mode: "checklist" + }, + type: "list" + } + ] + } + + # atoms dynamically and atoms are not + # garbage-collected. Therefore, string should not be an untrusted value, such as + # input received from a socket or during a web request. Consider using + # to_existing_atom/1 instead + # keys_to_atoms is using to_existing_atom under the hook + + _ = :hideLabel + _ = :labelType + + string_map = %{ + "blocks" => [ + %{ + "data" => %{ + "items" => [ + %{ + "checked" => true, + "hideLabel" => true, + "indent" => 0, + "label" => "label", + "labelType" => "success", + "text" => "list item" + } + ], + "mode" => "checklist" + }, + "type" => "list" + } + ] + } + + assert Utils.keys_to_atoms(string_map) == atom_map + end + end + describe "[deep merge]" do test 'one level of maps without conflict' do result = Utils.deep_merge(%{a: 1}, %{b: 2}) From b255880a6d93cd0b9e19832a1363536e7176702b Mon Sep 17 00:00:00 2001 From: mydearxym Date: Mon, 15 Feb 2021 23:29:58 +0800 Subject: [PATCH 10/23] refactor(editor): return map fmt when error raised --- .../converter/editor_to_html/validator.ex | 86 ++++++++++++------- lib/helper/validate_by_schema.ex | 6 +- .../editor_to_html_test/header_test.exs | 20 ++++- .../editor_to_html_test/list_test.exs | 13 ++- .../editor_to_html_test/paragraph_test.exs | 9 +- 5 files changed, 91 insertions(+), 43 deletions(-) diff --git a/lib/helper/converter/editor_to_html/validator.ex b/lib/helper/converter/editor_to_html/validator.ex index 938d0973e..79a25fb35 100644 --- a/lib/helper/converter/editor_to_html/validator.ex +++ b/lib/helper/converter/editor_to_html/validator.ex @@ -36,13 +36,19 @@ defmodule Helper.Converter.EditorToHTML.Validator do with atom_map <- Utils.keys_to_atoms(map), true <- is_valid_editorjs_fmt(atom_map) do blocks = atom_map.blocks + # validate_blocks(blocks) try do validate_blocks(blocks) rescue - e in MatchError -> format_parse_error(e) - e in RuntimeError -> format_parse_error(e) - e -> format_parse_error() + e in MatchError -> + format_parse_error(e) + + e in RuntimeError -> + format_parse_error(e) + + e -> + format_parse_error() end else false -> @@ -57,16 +63,47 @@ defmodule Helper.Converter.EditorToHTML.Validator do defp validate_blocks(blocks) do Enum.each(blocks, fn block -> + # if error happened, will be rescued {:ok, _} = validate_block(block) end) {:ok, :pass} end + defp validate_block(%{type: "paragraph", data: %{text: text} = data}) do + schema = %{text: [:string]} + + case ValidateBySchema.cast(schema, data) do + {:error, errors} -> + format_parse_error("paragraph", errors) + + _ -> + {:ok, :pass} + end + end + + defp validate_block(%{type: "header", data: %{text: text, level: level} = data}) do + schema = %{ + text: [:string], + level: [enum: @valid_header_level], + eyebrowTitle: [:string, required: false], + footerTitle: [:string, required: false] + } + + case ValidateBySchema.cast(schema, data) do + {:error, errors} -> + format_parse_error("header", errors) + + _ -> + {:ok, :pass} + end + end + defp validate_block(%{type: "list", data: %{mode: mode, items: items} = data}) when mode in @valid_list_mode and is_list(items) do # mode_schema = %{mode: [enum: @valid_list_mode]} # {:ok, _} = ValidateBySchema.cast(mode_schema, data) + item_schema = %{ checked: [:boolean], hideLabel: [:boolean], @@ -77,24 +114,17 @@ defmodule Helper.Converter.EditorToHTML.Validator do } Enum.each(items, fn item -> - {:ok, _} = ValidateBySchema.cast(item_schema, item) - end) - - {:ok, :pass} - end + case ValidateBySchema.cast(item_schema, item) do + {:error, errors} -> + {:error, message} = format_parse_error("list(#{mode})", errors) + raise %MatchError{term: {:error, message}} - defp validate_block(%{type: "header", data: %{text: text, level: level} = data}) do - schema = %{ - text: [:string], - level: [enum: @valid_header_level], - eyebrowTitle: [:string, required: false], - footerTitle: [:string, required: false] - } - - {:ok, _} = ValidateBySchema.cast(schema, data) + _ -> + {:ok, :pass} + end + end) end - # defp validate_block(%{type: type, data: _}), do: raise("invalid data for #{type} block") defp validate_block(%{type: type}), do: raise("undown #{type} block") defp validate_block(_), do: raise("undown block") @@ -108,24 +138,20 @@ defmodule Helper.Converter.EditorToHTML.Validator do is_integer(map.time) end - # defp format_parse_error({:error, errors}) do - defp format_parse_error(%MatchError{term: {:error, errors}}) do - message = - Enum.reduce(errors, "", fn x, acc -> - case String.trim(acc) do - "" -> x.message - _ -> acc <> " ; " <> x.message - end - end) + defp format_parse_error(type, error_list) when is_list(error_list) do + {:error, + Enum.map(error_list, fn error -> + Map.merge(error, %{block: type}) + end)} + end - {:error, message} + defp format_parse_error(%MatchError{term: {:error, error}}) do + {:error, error} end - # defp format_parse_error(%{message: message}), do: {:error, message} defp format_parse_error(%{message: message}) do {:error, message} end - # defp format_parse_error(e), do: IO.inspect(e, label: "e") defp format_parse_error(), do: {:error, "undown validate error"} end diff --git a/lib/helper/validate_by_schema.ex b/lib/helper/validate_by_schema.ex index e3cedbf52..21449f3ad 100644 --- a/lib/helper/validate_by_schema.ex +++ b/lib/helper/validate_by_schema.ex @@ -91,8 +91,8 @@ defmodule Helper.ValidateBySchema do false -> {:error, %{ - field: field, - message: "#{field} should be: #{enum |> Enum.join(" | ") |> to_string}" + field: "#{field |> to_string}", + message: "should be: #{enum |> Enum.join(" | ") |> to_string}" }} end end @@ -100,6 +100,6 @@ defmodule Helper.ValidateBySchema do defp done(field, value), do: {:ok, %{field: field, value: value}} defp error(field, value, schema) do - {:error, %{field: field, value: value, message: "#{field} should be: #{schema}"}} + {:error, %{field: field |> to_string, value: value, message: "should be: #{schema}"}} end end diff --git a/test/helper/converter/editor_to_html_test/header_test.exs b/test/helper/converter/editor_to_html_test/header_test.exs index 0250d0a77..90cc3ca19 100644 --- a/test/helper/converter/editor_to_html_test/header_test.exs +++ b/test/helper/converter/editor_to_html_test/header_test.exs @@ -77,7 +77,7 @@ defmodule GroupherServer.Test.Helper.Converter.EditorToHTML.Header do "time" => 1_567_250_876_713, "version" => "2.15.0" } - @tag :wip2 + @tag :wip test "optional field should valid properly" do json = Map.merge(@editor_json, %{ @@ -144,7 +144,21 @@ defmodule GroupherServer.Test.Helper.Converter.EditorToHTML.Header do {:ok, editor_string} = Jason.encode(json) {:error, error} = Parser.to_html(editor_string) - assert error == "eyebrowTitle should be: string ; footerTitle should be: string" + assert error == + [ + %{ + block: "header", + field: "eyebrowTitle", + message: "should be: string", + value: [] + }, + %{ + block: "header", + field: "footerTitle", + message: "should be: string", + value: true + } + ] json = Map.merge(@editor_json, %{ @@ -161,7 +175,7 @@ defmodule GroupherServer.Test.Helper.Converter.EditorToHTML.Header do {:ok, editor_string} = Jason.encode(json) {:error, error} = Parser.to_html(editor_string) - assert error == "level should be: 1 | 2 | 3" + assert error == [%{block: "header", field: "level", message: "should be: 1 | 2 | 3"}] end end end diff --git a/test/helper/converter/editor_to_html_test/list_test.exs b/test/helper/converter/editor_to_html_test/list_test.exs index f2ce2b7ea..9d75a09f9 100644 --- a/test/helper/converter/editor_to_html_test/list_test.exs +++ b/test/helper/converter/editor_to_html_test/list_test.exs @@ -94,11 +94,18 @@ defmodule GroupherServer.Test.Helper.Converter.EditorToHTML.List do ], "version" => "2.15.0" } - @tag :wip + @tag :wip2 test "invalid indent field should get error" do {:ok, editor_string} = Jason.encode(@editor_json) - # {:ok, converted} = Parser.to_html(editor_string) - assert {:error, "indent field should be: 0 | 1 | 2 | 3 | 4"} = Parser.to_html(editor_string) + {:error, error} = Parser.to_html(editor_string) + + assert error === [ + %{ + block: "list(checklist)", + field: "indent", + message: "should be: 0 | 1 | 2 | 3 | 4" + } + ] end end end diff --git a/test/helper/converter/editor_to_html_test/paragraph_test.exs b/test/helper/converter/editor_to_html_test/paragraph_test.exs index 0f52e8d94..0b082c3c1 100644 --- a/test/helper/converter/editor_to_html_test/paragraph_test.exs +++ b/test/helper/converter/editor_to_html_test/paragraph_test.exs @@ -41,13 +41,14 @@ defmodule GroupherServer.Test.Helper.Converter.EditorToHTML.Paragraph do ], "version" => "2.15.0" } - @tag :wip + @tag :wip2 test "invalid paragraph should have invalid hint" do {:ok, editor_string} = Jason.encode(@editor_json) - {:ok, converted} = Parser.to_html(editor_string) + {:error, error} = Parser.to_html(editor_string) - assert converted == - "
[invalid-block] paragraph:text
" + assert error == [ + %{block: "paragraph", field: "text", message: "should be: string", value: []} + ] end end end From ba9b2309a70c0b3859c6e59acededfe92d76528a Mon Sep 17 00:00:00 2001 From: mydearxym Date: Tue, 16 Feb 2021 00:47:24 +0800 Subject: [PATCH 11/23] fix(editor): utils keys_to_atoms edge case --- lib/helper/utils.ex | 2 ++ test/helper/utils_test.exs | 10 ++++++++++ 2 files changed, 12 insertions(+) diff --git a/lib/helper/utils.ex b/lib/helper/utils.ex index fd376d586..13b64b4ff 100644 --- a/lib/helper/utils.ex +++ b/lib/helper/utils.ex @@ -102,6 +102,8 @@ defmodule Helper.Utils do Map.new(json, &reduce_keys_to_atoms/1) end + def keys_to_atoms(string) when is_binary(string), do: string + def reduce_keys_to_atoms({key, val}) when is_map(val), do: {String.to_existing_atom(key), keys_to_atoms(val)} diff --git a/test/helper/utils_test.exs b/test/helper/utils_test.exs index ec6a42c75..642b8c1a4 100644 --- a/test/helper/utils_test.exs +++ b/test/helper/utils_test.exs @@ -26,6 +26,11 @@ defmodule GroupherServer.Test.Helper.UtilsTest do describe "map keys to atom" do test "string keys should covert to atom keys on nested map" do atom_map = %{ + string_array: [ + "line 1", + "line 2", + "line 3" + ], blocks: [ %{ data: %{ @@ -56,6 +61,11 @@ defmodule GroupherServer.Test.Helper.UtilsTest do _ = :labelType string_map = %{ + "string_array" => [ + "line 1", + "line 2", + "line 3" + ], "blocks" => [ %{ "data" => %{ From efa82d5fb24a151f292548416aad986e8ce4e20d Mon Sep 17 00:00:00 2001 From: mydearxym Date: Tue, 16 Feb 2021 00:48:21 +0800 Subject: [PATCH 12/23] fix(editor): error message alert --- .../converter/editor_to_html/validator.ex | 15 ++ .../editor_to_html_test/header_test.exs | 2 +- .../editor_to_html_test/index_test.exs | 164 +----------------- .../editor_to_html_test/list_test.exs | 23 ++- .../editor_to_html_test/paragraph_test.exs | 2 +- 5 files changed, 33 insertions(+), 173 deletions(-) diff --git a/lib/helper/converter/editor_to_html/validator.ex b/lib/helper/converter/editor_to_html/validator.ex index 79a25fb35..b45f9655a 100644 --- a/lib/helper/converter/editor_to_html/validator.ex +++ b/lib/helper/converter/editor_to_html/validator.ex @@ -18,6 +18,7 @@ defmodule Helper.Converter.EditorToHTML.Validator do tursted_atoms = [ # common :text, + :items, # header :level, :eyebrowTitle, @@ -123,6 +124,20 @@ defmodule Helper.Converter.EditorToHTML.Validator do {:ok, :pass} end end) + + {:ok, :pass} + end + + defp validate_block(%{type: "code"}) do + # schema = %{text: [:string]} + # case ValidateBySchema.cast(schema, data) do + # {:error, errors} -> + # format_parse_error("paragraph", errors) + + # _ -> + # {:ok, :pass} + # end + {:ok, :pass} end defp validate_block(%{type: type}), do: raise("undown #{type} block") diff --git a/test/helper/converter/editor_to_html_test/header_test.exs b/test/helper/converter/editor_to_html_test/header_test.exs index 90cc3ca19..c77e15cad 100644 --- a/test/helper/converter/editor_to_html_test/header_test.exs +++ b/test/helper/converter/editor_to_html_test/header_test.exs @@ -124,7 +124,7 @@ defmodule GroupherServer.Test.Helper.Converter.EditorToHTML.Header do }\">footer title content
\n
\n
" end - @tag :wip2 + @tag :wip test "wrong header format data should have invalid hint" do json = Map.merge(@editor_json, %{ diff --git a/test/helper/converter/editor_to_html_test/index_test.exs b/test/helper/converter/editor_to_html_test/index_test.exs index 847b38fc7..e1126db37 100644 --- a/test/helper/converter/editor_to_html_test/index_test.exs +++ b/test/helper/converter/editor_to_html_test/index_test.exs @@ -10,169 +10,6 @@ defmodule GroupherServer.Test.Helper.Converter.EditorToHTML do @real_editor_data ~S({ "time" : 1567250876713, "blocks" : [ - { - "type" : "paragraph", - "data" : { - "text" : "Hey. Meet the new Editor. On this page you can see it in action — try to edit this text." - } - }, - { - "type" : "header", - "data" : { - "text" : "Editor.js", - "level" : 2 - } - }, - { - "type" : "header", - "data" : { - "text" : "Key features", - "level" : 3 - } - }, - { - "type" : "list", - "data" : { - "style" : "unordered", - "items" : [ - "It is a block-styled editor", - "It returns clean data output in JSON", - "Designed to be extendable and pluggable with a simple API" - ] - } - }, - { - "type" : "header", - "data" : { - "text" : "Key features", - "level" : 3 - } - }, - { - "type" : "list", - "data" : { - "style" : "ordered", - "items" : [ - "It is a block-styled editor", - "It returns clean data output in JSON", - "Designed to be extendable and pluggable with a simple API" - ] - } - }, - { - "type" : "header", - "data" : { - "text" : "What does it mean «block-styled editor»", - "level" : 3 - } - }, - { - "type" : "checklist", - "data" : { - "items" : [ - { - "text" : "This is a block-styled editor", - "checked" : true - }, - { - "text" : "Clean output data", - "checked" : false - }, - { - "text" : "Simple and powerful API", - "checked" : true - } - ] - } - }, - { - "type" : "paragraph", - "data" : { - "text" : "Workspace in classic editors is made of a single contenteditable element, used to create different HTML markups. Editor.js workspace consists of separate Blocks: paragraphs, headings, images, lists, quotes, etc. Each of them is an independent contenteditable element (or more complex structure\) provided by Plugin and united by Editor's Core." - } - }, - { - "type" : "paragraph", - "data" : { - "text" : "There are dozens of ready-to-use Blocks and the simple API for creation any Block you need. For example, you can implement Blocks for Tweets, Instagram posts, surveys and polls, CTA-buttons and even games." - } - }, - { - "type" : "header", - "data" : { - "text" : "What does it mean clean data output", - "level" : 3 - } - }, - { - "type" : "paragraph", - "data" : { - "text" : "Classic WYSIWYG-editors produce raw HTML-markup with both content data and content appearance. On the contrary, Editor.js outputs JSON object with data of each Block. You can see an example below" - } - }, - { - "type" : "paragraph", - "data" : { - "text" : "Given data can be used as you want: render with HTML for Web clients, render natively for mobile apps, create markup for Facebook Instant Articles or Google AMP, generate an audio version and so on." - } - }, - { - "type" : "paragraph", - "data" : { - "text" : "Clean data is useful to sanitize, validate and process on the backend." - } - }, - { - "type" : "delimiter", - "data" : {} - }, - { - "type" : "paragraph", - "data" : { - "text" : "We have been working on this project more than three years. Several large media projects help us to test and debug the Editor, to make it's core more stable. At the same time we significantly improved the API. Now, it can be used to create any plugin for any task. Hope you enjoy. 😏" - } - }, - { - "type" : "image", - "data" : { - "file" : { - "url" : "https://codex.so/upload/redactor_images/o_e48549d1855c7fc1807308dd14990126.jpg" - }, - "caption" : "", - "withBorder" : true, - "stretched" : false, - "withBackground" : false - } - }, - { - "type" : "linkTool", - "data" : { - "link" : "https://www.github.com", - "meta" : { - "url" : "https://www.github.com", - "domain" : "www.github.com", - "title" : "Build software better, together", - "description" : "GitHub is where people build software. More than 40 million people use GitHub to discover, fork, and contribute to over 100 million projects.", - "image" : { - "url" : "https://github.githubassets.com/images/modules/open_graph/github-logo.png" - } - } - } - }, - { - "type" : "quote", - "data" : { - "text" : "quote demo text", - "caption" : "desc?", - "alignment" : "left" - } - }, - { - "type" : "delimiter", - "data" : { - "type" : "pen" - } - }, { "type" : "code", "data" : { @@ -237,6 +74,7 @@ defmodule GroupherServer.Test.Helper.Converter.EditorToHTML do end describe "[secure issues]" do + @tag :wip test "code block should avoid potential xss script attack" do {:ok, converted} = Parser.to_html(@real_editor_data) diff --git a/test/helper/converter/editor_to_html_test/list_test.exs b/test/helper/converter/editor_to_html_test/list_test.exs index 9d75a09f9..d4909ca5d 100644 --- a/test/helper/converter/editor_to_html_test/list_test.exs +++ b/test/helper/converter/editor_to_html_test/list_test.exs @@ -36,8 +36,19 @@ defmodule GroupherServer.Test.Helper.Converter.EditorToHTML.List do {:ok, editor_string} = Jason.encode(@editor_json) {:error, err_msg} = Parser.to_html(editor_string) - assert err_msg == - "indent should be: 0 | 1 | 2 | 3 | 4 ; hideLabel should be: boolean" + assert err_msg == [ + %{ + block: "list(checklist)", + field: "hideLabel", + message: "should be: boolean", + value: "invalid" + }, + %{ + block: "list(checklist)", + field: "indent", + message: "should be: 0 | 1 | 2 | 3 | 4" + } + ] end @editor_json %{ @@ -65,11 +76,7 @@ defmodule GroupherServer.Test.Helper.Converter.EditorToHTML.List do @tag :wip test "valid list parse should work" do {:ok, editor_string} = Jason.encode(@editor_json) - # {:ok, converted} = Parser.to_html(editor_string) - Parser.to_html(editor_string) - - assert {:ok, converted} = Parser.to_html(editor_string) - # IO.inspect(converted, label: "->> converted") + assert {:ok, _} = Parser.to_html(editor_string) end @editor_json %{ @@ -94,7 +101,7 @@ defmodule GroupherServer.Test.Helper.Converter.EditorToHTML.List do ], "version" => "2.15.0" } - @tag :wip2 + @tag :wip test "invalid indent field should get error" do {:ok, editor_string} = Jason.encode(@editor_json) {:error, error} = Parser.to_html(editor_string) diff --git a/test/helper/converter/editor_to_html_test/paragraph_test.exs b/test/helper/converter/editor_to_html_test/paragraph_test.exs index 0b082c3c1..9b570719f 100644 --- a/test/helper/converter/editor_to_html_test/paragraph_test.exs +++ b/test/helper/converter/editor_to_html_test/paragraph_test.exs @@ -41,7 +41,7 @@ defmodule GroupherServer.Test.Helper.Converter.EditorToHTML.Paragraph do ], "version" => "2.15.0" } - @tag :wip2 + @tag :wip test "invalid paragraph should have invalid hint" do {:ok, editor_string} = Jason.encode(@editor_json) {:error, error} = Parser.to_html(editor_string) From dc5f0cf22144c9fa9ceffd5f8f5bb0af92198a4f Mon Sep 17 00:00:00 2001 From: mydearxym Date: Tue, 16 Feb 2021 09:59:10 +0800 Subject: [PATCH 13/23] fix(editor): add :lang to tursted_atoms --- lib/helper/converter/editor_to_html/validator.ex | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/helper/converter/editor_to_html/validator.ex b/lib/helper/converter/editor_to_html/validator.ex index b45f9655a..77f5ef368 100644 --- a/lib/helper/converter/editor_to_html/validator.ex +++ b/lib/helper/converter/editor_to_html/validator.ex @@ -28,7 +28,9 @@ defmodule Helper.Converter.EditorToHTML.Validator do :labelType, :indent, :checked, - :label + :label, + # code + :lang ] Enum.each(tursted_atoms, fn atom -> _ = atom end) From 0ea6c34a26a80948aff6ca546375ca12ea5e7bd8 Mon Sep 17 00:00:00 2001 From: mydearxym Date: Tue, 16 Feb 2021 11:15:29 +0800 Subject: [PATCH 14/23] refactor(editor): use string-fmt map instead of atom-fmt --- .../converter/editor_to_html/validator.ex | 76 ++++++------------- lib/helper/validate_by_schema.ex | 4 +- .../editor_to_html_test/list_test.exs | 4 +- 3 files changed, 28 insertions(+), 56 deletions(-) diff --git a/lib/helper/converter/editor_to_html/validator.ex b/lib/helper/converter/editor_to_html/validator.ex index 77f5ef368..f2dd833ef 100644 --- a/lib/helper/converter/editor_to_html/validator.ex +++ b/lib/helper/converter/editor_to_html/validator.ex @@ -9,37 +9,9 @@ defmodule Helper.Converter.EditorToHTML.Validator do @valid_list_label_type ["success", "done", "todo"] @valid_list_indent [0, 1, 2, 3, 4] - # atoms dynamically and atoms are not - # garbage-collected. Therefore, string should not be an untrusted value, such as - # input received from a socket or during a web request. Consider using - # to_existing_atom/1 instead - # keys_to_atoms is using to_existing_atom under the hook, so we have to pre-define the - # trusted atoms - tursted_atoms = [ - # common - :text, - :items, - # header - :level, - :eyebrowTitle, - :footerTitle, - # list - :hideLabel, - :labelType, - :indent, - :checked, - :label, - # code - :lang - ] - - Enum.each(tursted_atoms, fn atom -> _ = atom end) - def is_valid(map) when is_map(map) do - with atom_map <- Utils.keys_to_atoms(map), - true <- is_valid_editorjs_fmt(atom_map) do - blocks = atom_map.blocks - # validate_blocks(blocks) + with true <- is_valid_editorjs_fmt(map) do + blocks = map["blocks"] try do validate_blocks(blocks) @@ -73,8 +45,8 @@ defmodule Helper.Converter.EditorToHTML.Validator do {:ok, :pass} end - defp validate_block(%{type: "paragraph", data: %{text: text} = data}) do - schema = %{text: [:string]} + defp validate_block(%{"type" => "paragraph", "data" => %{"text" => text} = data}) do + schema = %{"text" => [:string]} case ValidateBySchema.cast(schema, data) do {:error, errors} -> @@ -85,12 +57,12 @@ defmodule Helper.Converter.EditorToHTML.Validator do end end - defp validate_block(%{type: "header", data: %{text: text, level: level} = data}) do + defp validate_block(%{"type" => "header", "data" => %{"text" => text, "level" => level} = data}) do schema = %{ - text: [:string], - level: [enum: @valid_header_level], - eyebrowTitle: [:string, required: false], - footerTitle: [:string, required: false] + "text" => [:string], + "level" => [enum: @valid_header_level], + "eyebrowTitle" => [:string, required: false], + "footerTitle" => [:string, required: false] } case ValidateBySchema.cast(schema, data) do @@ -102,18 +74,18 @@ defmodule Helper.Converter.EditorToHTML.Validator do end end - defp validate_block(%{type: "list", data: %{mode: mode, items: items} = data}) + defp validate_block(%{"type" => "list", "data" => %{"mode" => mode, "items" => items} = data}) when mode in @valid_list_mode and is_list(items) do # mode_schema = %{mode: [enum: @valid_list_mode]} # {:ok, _} = ValidateBySchema.cast(mode_schema, data) item_schema = %{ - checked: [:boolean], - hideLabel: [:boolean], - label: [:string], - labelType: [enum: @valid_list_label_type], - indent: [enum: @valid_list_indent], - text: [:string] + "checked" => [:boolean], + "hideLabel" => [:boolean], + "label" => [:string], + "labelType" => [enum: @valid_list_label_type], + "indent" => [enum: @valid_list_indent], + "text" => [:string] } Enum.each(items, fn item -> @@ -130,7 +102,7 @@ defmodule Helper.Converter.EditorToHTML.Validator do {:ok, :pass} end - defp validate_block(%{type: "code"}) do + defp validate_block(%{"type" => "code"}) do # schema = %{text: [:string]} # case ValidateBySchema.cast(schema, data) do # {:error, errors} -> @@ -142,17 +114,17 @@ defmodule Helper.Converter.EditorToHTML.Validator do {:ok, :pass} end - defp validate_block(%{type: type}), do: raise("undown #{type} block") + defp validate_block(%{"type" => type}), do: raise("undown #{type} block") defp validate_block(_), do: raise("undown block") # check if the given map has the right key-value fmt of the editorjs structure defp is_valid_editorjs_fmt(map) when is_map(map) do - Map.has_key?(map, :time) and - Map.has_key?(map, :version) and - Map.has_key?(map, :blocks) and - is_list(map.blocks) and - is_binary(map.version) and - is_integer(map.time) + Map.has_key?(map, "time") and + Map.has_key?(map, "version") and + Map.has_key?(map, "blocks") and + is_list(map["blocks"]) and + is_binary(map["version"]) and + is_integer(map["time"]) end defp format_parse_error(type, error_list) when is_list(error_list) do diff --git a/lib/helper/validate_by_schema.ex b/lib/helper/validate_by_schema.ex index 21449f3ad..ce8a3899e 100644 --- a/lib/helper/validate_by_schema.ex +++ b/lib/helper/validate_by_schema.ex @@ -91,8 +91,8 @@ defmodule Helper.ValidateBySchema do false -> {:error, %{ - field: "#{field |> to_string}", - message: "should be: #{enum |> Enum.join(" | ") |> to_string}" + field: field, + message: "should be: #{enum |> Enum.join(" | ")}" }} end end diff --git a/test/helper/converter/editor_to_html_test/list_test.exs b/test/helper/converter/editor_to_html_test/list_test.exs index d4909ca5d..4dc047398 100644 --- a/test/helper/converter/editor_to_html_test/list_test.exs +++ b/test/helper/converter/editor_to_html_test/list_test.exs @@ -73,7 +73,7 @@ defmodule GroupherServer.Test.Helper.Converter.EditorToHTML.List do ], "version" => "2.15.0" } - @tag :wip + @tag :wip2 test "valid list parse should work" do {:ok, editor_string} = Jason.encode(@editor_json) assert {:ok, _} = Parser.to_html(editor_string) @@ -101,7 +101,7 @@ defmodule GroupherServer.Test.Helper.Converter.EditorToHTML.List do ], "version" => "2.15.0" } - @tag :wip + @tag :wip2 test "invalid indent field should get error" do {:ok, editor_string} = Jason.encode(@editor_json) {:error, error} = Parser.to_html(editor_string) From 50d6108141c9c94630eb335e12468fce6398be68 Mon Sep 17 00:00:00 2001 From: mydearxym Date: Tue, 16 Feb 2021 14:49:33 +0800 Subject: [PATCH 15/23] fix(editor): re-org validate workflow by using config --- .../{validator.ex => validator/index.ex} | 46 ++++++++----------- .../editor_to_html/validator/schema.ex | 22 +++++++++ .../editor_to_html_test/list_test.exs | 4 +- 3 files changed, 42 insertions(+), 30 deletions(-) rename lib/helper/converter/editor_to_html/{validator.ex => validator/index.ex} (82%) create mode 100644 lib/helper/converter/editor_to_html/validator/schema.ex diff --git a/lib/helper/converter/editor_to_html/validator.ex b/lib/helper/converter/editor_to_html/validator/index.ex similarity index 82% rename from lib/helper/converter/editor_to_html/validator.ex rename to lib/helper/converter/editor_to_html/validator/index.ex index f2dd833ef..7196d4674 100644 --- a/lib/helper/converter/editor_to_html/validator.ex +++ b/lib/helper/converter/editor_to_html/validator/index.ex @@ -3,7 +3,10 @@ defmodule Helper.Converter.EditorToHTML.Validator do alias Helper.{Utils, ValidateBySchema} - @valid_header_level [1, 2, 3] + alias Helper.Converter.EditorToHTML.Validator.Schema + + # blocks with no children items + @simple_blocks ["header", "paragraph"] @valid_list_mode ["checklist", "order_list", "unorder_list"] @valid_list_label_type ["success", "done", "todo"] @@ -45,33 +48,9 @@ defmodule Helper.Converter.EditorToHTML.Validator do {:ok, :pass} end - defp validate_block(%{"type" => "paragraph", "data" => %{"text" => text} = data}) do - schema = %{"text" => [:string]} - - case ValidateBySchema.cast(schema, data) do - {:error, errors} -> - format_parse_error("paragraph", errors) - - _ -> - {:ok, :pass} - end - end - - defp validate_block(%{"type" => "header", "data" => %{"text" => text, "level" => level} = data}) do - schema = %{ - "text" => [:string], - "level" => [enum: @valid_header_level], - "eyebrowTitle" => [:string, required: false], - "footerTitle" => [:string, required: false] - } - - case ValidateBySchema.cast(schema, data) do - {:error, errors} -> - format_parse_error("header", errors) - - _ -> - {:ok, :pass} - end + # validate block which have no nested items + defp validate_block(%{"type" => type, "data" => data}) when type in @simple_blocks do + validate_with(type, Schema.get(type), data) end defp validate_block(%{"type" => "list", "data" => %{"mode" => mode, "items" => items} = data}) @@ -117,6 +96,17 @@ defmodule Helper.Converter.EditorToHTML.Validator do defp validate_block(%{"type" => type}), do: raise("undown #{type} block") defp validate_block(_), do: raise("undown block") + # validate with given schema + defp validate_with(block, schema, data) do + case ValidateBySchema.cast(schema, data) do + {:error, errors} -> + format_parse_error(block, errors) + + _ -> + {:ok, :pass} + end + end + # check if the given map has the right key-value fmt of the editorjs structure defp is_valid_editorjs_fmt(map) when is_map(map) do Map.has_key?(map, "time") and diff --git a/lib/helper/converter/editor_to_html/validator/schema.ex b/lib/helper/converter/editor_to_html/validator/schema.ex new file mode 100644 index 000000000..978cf8629 --- /dev/null +++ b/lib/helper/converter/editor_to_html/validator/schema.ex @@ -0,0 +1,22 @@ +defmodule Helper.Converter.EditorToHTML.Validator.Schema do + @moduledoc false + + @valid_header_level [1, 2, 3] + + def get("header") do + %{ + "text" => [:string], + "level" => [enum: @valid_header_level], + "eyebrowTitle" => [:string, required: false], + "footerTitle" => [:string, required: false] + } + end + + def get("paragraph") do + %{"text" => [:string]} + end + + def get(_) do + %{} + end +end diff --git a/test/helper/converter/editor_to_html_test/list_test.exs b/test/helper/converter/editor_to_html_test/list_test.exs index 4dc047398..d4909ca5d 100644 --- a/test/helper/converter/editor_to_html_test/list_test.exs +++ b/test/helper/converter/editor_to_html_test/list_test.exs @@ -73,7 +73,7 @@ defmodule GroupherServer.Test.Helper.Converter.EditorToHTML.List do ], "version" => "2.15.0" } - @tag :wip2 + @tag :wip test "valid list parse should work" do {:ok, editor_string} = Jason.encode(@editor_json) assert {:ok, _} = Parser.to_html(editor_string) @@ -101,7 +101,7 @@ defmodule GroupherServer.Test.Helper.Converter.EditorToHTML.List do ], "version" => "2.15.0" } - @tag :wip2 + @tag :wip test "invalid indent field should get error" do {:ok, editor_string} = Jason.encode(@editor_json) {:error, error} = Parser.to_html(editor_string) From 39ff354a608e13605605ba2735f660962f7c4a04 Mon Sep 17 00:00:00 2001 From: mydearxym Date: Tue, 16 Feb 2021 15:17:21 +0800 Subject: [PATCH 16/23] fix(editor): add list to validator workflow as general --- .../editor_to_html/validator/index.ex | 58 ++++++++++--------- .../editor_to_html/validator/schema.ex | 22 ++++++- 2 files changed, 52 insertions(+), 28 deletions(-) diff --git a/lib/helper/converter/editor_to_html/validator/index.ex b/lib/helper/converter/editor_to_html/validator/index.ex index 7196d4674..21f8ac429 100644 --- a/lib/helper/converter/editor_to_html/validator/index.ex +++ b/lib/helper/converter/editor_to_html/validator/index.ex @@ -7,6 +7,8 @@ defmodule Helper.Converter.EditorToHTML.Validator do # blocks with no children items @simple_blocks ["header", "paragraph"] + # blocks with "mode" and "items" fields + @complex_blocks ["list"] @valid_list_mode ["checklist", "order_list", "unorder_list"] @valid_list_label_type ["success", "done", "todo"] @@ -53,32 +55,11 @@ defmodule Helper.Converter.EditorToHTML.Validator do validate_with(type, Schema.get(type), data) end - defp validate_block(%{"type" => "list", "data" => %{"mode" => mode, "items" => items} = data}) - when mode in @valid_list_mode and is_list(items) do - # mode_schema = %{mode: [enum: @valid_list_mode]} - # {:ok, _} = ValidateBySchema.cast(mode_schema, data) - - item_schema = %{ - "checked" => [:boolean], - "hideLabel" => [:boolean], - "label" => [:string], - "labelType" => [enum: @valid_list_label_type], - "indent" => [enum: @valid_list_indent], - "text" => [:string] - } - - Enum.each(items, fn item -> - case ValidateBySchema.cast(item_schema, item) do - {:error, errors} -> - {:error, message} = format_parse_error("list(#{mode})", errors) - raise %MatchError{term: {:error, message}} - - _ -> - {:ok, :pass} - end - end) - - {:ok, :pass} + # validate block which has mode and items + defp validate_block(%{"type" => type, "data" => %{"mode" => mode, "items" => items} = data}) + when type in @complex_blocks do + [parent: parent_schema, item: item_schema] = Schema.get(type) + validate_with(type, parent_schema, item_schema, data) end defp validate_block(%{"type" => "code"}) do @@ -107,6 +88,31 @@ defmodule Helper.Converter.EditorToHTML.Validator do end end + defp validate_with(block, parent_schema, item_schema, data) do + case ValidateBySchema.cast(parent_schema, data) do + {:error, errors} -> + format_parse_error(block, errors) + + _ -> + {:ok, :pass} + end + + %{"mode" => mode, "items" => items} = data + + Enum.each(items, fn item -> + case ValidateBySchema.cast(item_schema, item) do + {:error, errors} -> + {:error, message} = format_parse_error("#{block}(#{mode})", errors) + raise %MatchError{term: {:error, message}} + + _ -> + {:ok, :pass} + end + end) + + {:ok, :pass} + end + # check if the given map has the right key-value fmt of the editorjs structure defp is_valid_editorjs_fmt(map) when is_map(map) do Map.has_key?(map, "time") and diff --git a/lib/helper/converter/editor_to_html/validator/schema.ex b/lib/helper/converter/editor_to_html/validator/schema.ex index 978cf8629..2c2fa7c5e 100644 --- a/lib/helper/converter/editor_to_html/validator/schema.ex +++ b/lib/helper/converter/editor_to_html/validator/schema.ex @@ -1,8 +1,14 @@ defmodule Helper.Converter.EditorToHTML.Validator.Schema do @moduledoc false + # header @valid_header_level [1, 2, 3] + # list + @valid_list_mode ["checklist", "order_list", "unorder_list"] + @valid_list_label_type ["success", "done", "todo"] + @valid_list_indent [0, 1, 2, 3, 4] + def get("header") do %{ "text" => [:string], @@ -12,8 +18,20 @@ defmodule Helper.Converter.EditorToHTML.Validator.Schema do } end - def get("paragraph") do - %{"text" => [:string]} + def get("paragraph"), do: %{"text" => [:string]} + + def get("list") do + [ + parent: %{"mode" => [enum: @valid_list_mode]}, + item: %{ + "checked" => [:boolean], + "hideLabel" => [:boolean], + "label" => [:string], + "labelType" => [enum: @valid_list_label_type], + "indent" => [enum: @valid_list_indent], + "text" => [:string] + } + ] end def get(_) do From 2ece8eadba1a2c8fa1fa23a3da70e4a572ec7230 Mon Sep 17 00:00:00 2001 From: mydearxym Date: Wed, 17 Feb 2021 22:52:26 +0800 Subject: [PATCH 17/23] refactor(editor): use common schema to validate editor-fmt json --- .../editor_to_html/validator/index.ex | 23 +++----- .../editor_to_html/validator/schema.ex | 8 +++ lib/helper/validate_by_schema.ex | 55 +++++++++++++------ .../editor_to_html_test/index_test.exs | 43 ++++++++++++++- .../editor_to_html_test/list_test.exs | 46 ++++++++-------- 5 files changed, 118 insertions(+), 57 deletions(-) diff --git a/lib/helper/converter/editor_to_html/validator/index.ex b/lib/helper/converter/editor_to_html/validator/index.ex index 21f8ac429..396e44d0d 100644 --- a/lib/helper/converter/editor_to_html/validator/index.ex +++ b/lib/helper/converter/editor_to_html/validator/index.ex @@ -10,14 +10,9 @@ defmodule Helper.Converter.EditorToHTML.Validator do # blocks with "mode" and "items" fields @complex_blocks ["list"] - @valid_list_mode ["checklist", "order_list", "unorder_list"] - @valid_list_label_type ["success", "done", "todo"] - @valid_list_indent [0, 1, 2, 3, 4] - - def is_valid(map) when is_map(map) do - with true <- is_valid_editorjs_fmt(map) do - blocks = map["blocks"] - + def is_valid(data) when is_map(data) do + with {:ok, _} <- validate_editor_fmt(data), + blocks <- Map.get(data, "blocks") do try do validate_blocks(blocks) rescue @@ -30,15 +25,13 @@ defmodule Helper.Converter.EditorToHTML.Validator do e -> format_parse_error() end - else - false -> - {:error, "invalid editor json format"} - - _ -> - {:error, "invalid editor json"} end end + defp validate_editor_fmt(data) do + validate_with("editor", Schema.get("editor"), data) + end + defp validate_blocks([]), do: {:ok, :pass} defp validate_blocks(blocks) do @@ -75,7 +68,7 @@ defmodule Helper.Converter.EditorToHTML.Validator do end defp validate_block(%{"type" => type}), do: raise("undown #{type} block") - defp validate_block(_), do: raise("undown block") + defp validate_block(e), do: raise("undown block: #{e}") # validate with given schema defp validate_with(block, schema, data) do diff --git a/lib/helper/converter/editor_to_html/validator/schema.ex b/lib/helper/converter/editor_to_html/validator/schema.ex index 2c2fa7c5e..2af468549 100644 --- a/lib/helper/converter/editor_to_html/validator/schema.ex +++ b/lib/helper/converter/editor_to_html/validator/schema.ex @@ -9,6 +9,14 @@ defmodule Helper.Converter.EditorToHTML.Validator.Schema do @valid_list_label_type ["success", "done", "todo"] @valid_list_indent [0, 1, 2, 3, 4] + def get("editor") do + %{ + "time" => [:number], + "version" => [:string], + "blocks" => [:list] + } + end + def get("header") do %{ "text" => [:string], diff --git a/lib/helper/validate_by_schema.ex b/lib/helper/validate_by_schema.ex index ce8a3899e..4000f4fd0 100644 --- a/lib/helper/validate_by_schema.ex +++ b/lib/helper/validate_by_schema.ex @@ -8,7 +8,7 @@ defmodule Helper.ValidateBySchema do @doc """ cast data by given schema - e.g: + ## example schema = %{ checked: [:boolean], hideLabel: [:boolean], @@ -22,21 +22,7 @@ defmodule Helper.ValidateBySchema do ValidateBySchema.cast(schema, data) """ def cast(schema, data) do - schema_fields = Map.keys(schema) - - errors_info = - Enum.reduce(schema_fields, [], fn field, acc -> - value = get_in(data, [field]) - field_schema = get_in(schema, [field]) - - case match(field, value, field_schema) do - {:error, error} -> - acc ++ [error] - - _ -> - acc - end - end) + errors_info = cast_errors(schema, data) case errors_info do [] -> {:ok, :pass} @@ -44,6 +30,27 @@ defmodule Helper.ValidateBySchema do end end + defp cast_errors(schema, data) do + schema_fields = Map.keys(schema) + + Enum.reduce(schema_fields, [], fn field, acc -> + value = get_in(data, [field]) + field_schema = get_in(schema, [field]) + + case match(field, value, field_schema) do + {:error, error} -> + acc ++ [error] + + _ -> + acc + end + end) + end + + # 这里试过用 macro 消除重复代码,但是不可行。 + # macro 对于重复定义的 quote 只会覆盖掉,除非每个 quote 中定义的内容不一样 + # 参考: https://elixirforum.com/t/define-multiple-modules-in-macro-only-last-one-gets-created/1654 + # boolean field defp match(field, nil, [:boolean, required: false]), do: done(field, nil) @@ -66,7 +73,9 @@ defmodule Helper.ValidateBySchema do defp match(field, value, [:string, required: false]), do: error(field, value, :string) defp match(field, value, [:string]), do: error(field, value, :string) - # number field + defp match(field, nil, [:string, required: false]), do: done(field, nil) + + # number defp match(field, nil, [:number, required: false]), do: done(field, nil) defp match(field, value, [:number, required: false]) when is_number(value) do @@ -77,6 +86,18 @@ defmodule Helper.ValidateBySchema do defp match(field, value, [:number, required: false]), do: error(field, value, :number) defp match(field, value, [:number]), do: error(field, value, :number) + # list + defp match(field, nil, [:list, required: false]), do: done(field, nil) + + defp match(field, value, [:list, required: false]) when is_list(value) do + done(field, value) + end + + defp match(field, value, [:list]) when is_list(value), do: done(field, value) + defp match(field, value, [:list, required: false]), do: error(field, value, :list) + defp match(field, value, [:list]), do: error(field, value, :list) + + # enum defp match(field, nil, enum: _, required: false), do: done(field, nil) defp match(field, value, enum: enum, required: false) do diff --git a/test/helper/converter/editor_to_html_test/index_test.exs b/test/helper/converter/editor_to_html_test/index_test.exs index e1126db37..09bb27d01 100644 --- a/test/helper/converter/editor_to_html_test/index_test.exs +++ b/test/helper/converter/editor_to_html_test/index_test.exs @@ -41,7 +41,7 @@ defmodule GroupherServer.Test.Helper.Converter.EditorToHTML do assert {:ok, _} = Parser.to_html(editor_string) end - @tag :wip + @tag :wip2 test "invalid editorjs json fmt should raise error" do editor_json = %{ "invalid_time" => 1_567_250_876_713, @@ -50,7 +50,29 @@ defmodule GroupherServer.Test.Helper.Converter.EditorToHTML do } {:ok, editor_string} = Jason.encode(editor_json) - assert {:error, "invalid editor json format"} = Parser.to_html(editor_string) + {:error, error} = Parser.to_html(editor_string) + + assert error == [ + %{block: "editor", field: "time", message: "should be: number", value: nil} + ] + + editor_json = %{ + "time" => "1_567_250_876_713", + "blocks" => [], + "version" => "2.15.0" + } + + {:ok, editor_string} = Jason.encode(editor_json) + {:error, error} = Parser.to_html(editor_string) + + assert error == [ + %{ + block: "editor", + field: "time", + message: "should be: number", + value: "1_567_250_876_713" + } + ] editor_json = %{ "time" => 1_567_250_876_713, @@ -60,7 +82,22 @@ defmodule GroupherServer.Test.Helper.Converter.EditorToHTML do } {:ok, editor_string} = Jason.encode(editor_json) - assert {:error, "invalid editor json format"} = Parser.to_html(editor_string) + {:error, error} = Parser.to_html(editor_string) + + assert error == [ + %{block: "editor", field: "blocks", message: "should be: list", value: "blocks"} + ] + + editor_json = %{ + "time" => 1_567_250_876_713, + "blocks" => [1, 2, 3], + "version" => "2.15.0" + } + + {:ok, editor_string} = Jason.encode(editor_json) + {:error, error} = Parser.to_html(editor_string) + + assert error == "undown block: 1" end test "real-world editor.js data should work" do diff --git a/test/helper/converter/editor_to_html_test/list_test.exs b/test/helper/converter/editor_to_html_test/list_test.exs index d4909ca5d..fe3b9f084 100644 --- a/test/helper/converter/editor_to_html_test/list_test.exs +++ b/test/helper/converter/editor_to_html_test/list_test.exs @@ -19,8 +19,8 @@ defmodule GroupherServer.Test.Helper.Converter.EditorToHTML.List do "items" => [ %{ "checked" => true, - "hideLabel" => "invalid", - "indent" => 5, + "hideLabel" => false, + "indent" => 0, "label" => "label", "labelType" => "success", "text" => "list item" @@ -32,23 +32,11 @@ defmodule GroupherServer.Test.Helper.Converter.EditorToHTML.List do "version" => "2.15.0" } @tag :wip - test "invalid list data parse should raise error message" do + test "valid list parse should work" do {:ok, editor_string} = Jason.encode(@editor_json) - {:error, err_msg} = Parser.to_html(editor_string) - - assert err_msg == [ - %{ - block: "list(checklist)", - field: "hideLabel", - message: "should be: boolean", - value: "invalid" - }, - %{ - block: "list(checklist)", - field: "indent", - message: "should be: 0 | 1 | 2 | 3 | 4" - } - ] + # assert {:ok, converted} = Parser.to_html(editor_string) + {:ok, converted} = Parser.to_html(editor_string) + IO.inspect(converted, label: "list converted") end @editor_json %{ @@ -61,8 +49,8 @@ defmodule GroupherServer.Test.Helper.Converter.EditorToHTML.List do "items" => [ %{ "checked" => true, - "hideLabel" => false, - "indent" => 0, + "hideLabel" => "invalid", + "indent" => 5, "label" => "label", "labelType" => "success", "text" => "list item" @@ -74,9 +62,23 @@ defmodule GroupherServer.Test.Helper.Converter.EditorToHTML.List do "version" => "2.15.0" } @tag :wip - test "valid list parse should work" do + test "invalid list data parse should raise error message" do {:ok, editor_string} = Jason.encode(@editor_json) - assert {:ok, _} = Parser.to_html(editor_string) + {:error, err_msg} = Parser.to_html(editor_string) + + assert err_msg == [ + %{ + block: "list(checklist)", + field: "hideLabel", + message: "should be: boolean", + value: "invalid" + }, + %{ + block: "list(checklist)", + field: "indent", + message: "should be: 0 | 1 | 2 | 3 | 4" + } + ] end @editor_json %{ From e244d89d1093208dc5d8eed2d46da27257f3fff8 Mon Sep 17 00:00:00 2001 From: mydearxym Date: Thu, 18 Feb 2021 00:35:03 +0800 Subject: [PATCH 18/23] refactor(editor): use macros to reduce similar code --- lib/helper/validate_by_schema.ex | 82 ++++++++----------- .../editor_to_html_test/list_test.exs | 2 +- 2 files changed, 33 insertions(+), 51 deletions(-) diff --git a/lib/helper/validate_by_schema.ex b/lib/helper/validate_by_schema.ex index 4000f4fd0..e3acdc6f1 100644 --- a/lib/helper/validate_by_schema.ex +++ b/lib/helper/validate_by_schema.ex @@ -1,3 +1,33 @@ +defmodule Helper.ValidateBySchema.Matchers do + @moduledoc """ + matchers for basic type, support required option + """ + + defmacro __using__(types) do + # can not use Enum.each here, see https://elixirforum.com/t/define-multiple-modules-in-macro-only-last-one-gets-created/1654/4 + for type <- types do + guard_name = if type == :string, do: "binary", else: type |> to_string + + quote do + defp match(field, nil, [unquote(type), required: false]), do: done(field, nil) + + defp match(field, value, [unquote(type), required: false]) + when unquote(:"is_#{guard_name}")(value) do + done(field, value) + end + + defp match(field, value, [unquote(type)]) when unquote(:"is_#{guard_name}")(value), + do: done(field, value) + + defp match(field, value, [unquote(type), required: false]), + do: error(field, value, unquote(type)) + + defp match(field, value, [unquote(type)]), do: error(field, value, unquote(type)) + end + end + end +end + defmodule Helper.ValidateBySchema do @moduledoc """ validate json data by given schema, mostly used in editorjs validator @@ -21,6 +51,8 @@ defmodule Helper.ValidateBySchema do data = %{checked: true, label: "done"} ValidateBySchema.cast(schema, data) """ + use Helper.ValidateBySchema.Matchers, [:string, :number, :list, :boolean] + def cast(schema, data) do errors_info = cast_errors(schema, data) @@ -47,56 +79,6 @@ defmodule Helper.ValidateBySchema do end) end - # 这里试过用 macro 消除重复代码,但是不可行。 - # macro 对于重复定义的 quote 只会覆盖掉,除非每个 quote 中定义的内容不一样 - # 参考: https://elixirforum.com/t/define-multiple-modules-in-macro-only-last-one-gets-created/1654 - - # boolean field - defp match(field, nil, [:boolean, required: false]), do: done(field, nil) - - defp match(field, value, [:boolean, required: false]) when is_boolean(value) do - done(field, value) - end - - defp match(field, value, [:boolean]) when is_boolean(value), do: done(field, value) - defp match(field, value, [:boolean, required: false]), do: error(field, value, :boolean) - defp match(field, value, [:boolean]), do: error(field, value, :boolean) - - # string field - defp match(field, nil, [:string, required: false]), do: done(field, nil) - - defp match(field, value, [:string, required: false]) when is_binary(value) do - done(field, value) - end - - defp match(field, value, [:string]) when is_binary(value), do: done(field, value) - defp match(field, value, [:string, required: false]), do: error(field, value, :string) - defp match(field, value, [:string]), do: error(field, value, :string) - - defp match(field, nil, [:string, required: false]), do: done(field, nil) - - # number - defp match(field, nil, [:number, required: false]), do: done(field, nil) - - defp match(field, value, [:number, required: false]) when is_number(value) do - done(field, value) - end - - defp match(field, value, [:number]) when is_number(value), do: done(field, value) - defp match(field, value, [:number, required: false]), do: error(field, value, :number) - defp match(field, value, [:number]), do: error(field, value, :number) - - # list - defp match(field, nil, [:list, required: false]), do: done(field, nil) - - defp match(field, value, [:list, required: false]) when is_list(value) do - done(field, value) - end - - defp match(field, value, [:list]) when is_list(value), do: done(field, value) - defp match(field, value, [:list, required: false]), do: error(field, value, :list) - defp match(field, value, [:list]), do: error(field, value, :list) - # enum defp match(field, nil, enum: _, required: false), do: done(field, nil) diff --git a/test/helper/converter/editor_to_html_test/list_test.exs b/test/helper/converter/editor_to_html_test/list_test.exs index fe3b9f084..e43a160e4 100644 --- a/test/helper/converter/editor_to_html_test/list_test.exs +++ b/test/helper/converter/editor_to_html_test/list_test.exs @@ -61,7 +61,7 @@ defmodule GroupherServer.Test.Helper.Converter.EditorToHTML.List do ], "version" => "2.15.0" } - @tag :wip + @tag :wip2 test "invalid list data parse should raise error message" do {:ok, editor_string} = Jason.encode(@editor_json) {:error, err_msg} = Parser.to_html(editor_string) From 89c2e2a10576f01e2e558aed0a08b93cf33c44bd Mon Sep 17 00:00:00 2001 From: mydearxym Date: Thu, 18 Feb 2021 00:43:37 +0800 Subject: [PATCH 19/23] refactor(editor): adjust guard name in schema matchers --- lib/helper/validate_by_schema.ex | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/helper/validate_by_schema.ex b/lib/helper/validate_by_schema.ex index e3acdc6f1..1719f84f4 100644 --- a/lib/helper/validate_by_schema.ex +++ b/lib/helper/validate_by_schema.ex @@ -6,17 +6,17 @@ defmodule Helper.ValidateBySchema.Matchers do defmacro __using__(types) do # can not use Enum.each here, see https://elixirforum.com/t/define-multiple-modules-in-macro-only-last-one-gets-created/1654/4 for type <- types do - guard_name = if type == :string, do: "binary", else: type |> to_string + guard_name = if type == :string, do: "is_binary", else: "is_#{to_string(type)}" quote do defp match(field, nil, [unquote(type), required: false]), do: done(field, nil) defp match(field, value, [unquote(type), required: false]) - when unquote(:"is_#{guard_name}")(value) do + when unquote(:"#{guard_name}")(value) do done(field, value) end - defp match(field, value, [unquote(type)]) when unquote(:"is_#{guard_name}")(value), + defp match(field, value, [unquote(type)]) when unquote(:"#{guard_name}")(value), do: done(field, value) defp match(field, value, [unquote(type), required: false]), From 23085c8e6d9e1f8928fd4539adb6f892ae393889 Mon Sep 17 00:00:00 2001 From: mydearxym Date: Thu, 18 Feb 2021 10:48:18 +0800 Subject: [PATCH 20/23] refactor(editor): re-org / rename the schema validator --- .../validator/{schema.ex => editor_schema.ex} | 2 +- .../editor_to_html/validator/index.ex | 33 ++++++---------- .../schema.ex} | 38 ++----------------- lib/helper/validator/schema_matchers.ex | 29 ++++++++++++++ .../editor_to_html_test/header_test.exs | 2 +- 5 files changed, 47 insertions(+), 57 deletions(-) rename lib/helper/converter/editor_to_html/validator/{schema.ex => editor_schema.ex} (93%) rename lib/helper/{validate_by_schema.ex => validator/schema.ex} (58%) create mode 100644 lib/helper/validator/schema_matchers.ex diff --git a/lib/helper/converter/editor_to_html/validator/schema.ex b/lib/helper/converter/editor_to_html/validator/editor_schema.ex similarity index 93% rename from lib/helper/converter/editor_to_html/validator/schema.ex rename to lib/helper/converter/editor_to_html/validator/editor_schema.ex index 2af468549..5e0af0b4a 100644 --- a/lib/helper/converter/editor_to_html/validator/schema.ex +++ b/lib/helper/converter/editor_to_html/validator/editor_schema.ex @@ -1,4 +1,4 @@ -defmodule Helper.Converter.EditorToHTML.Validator.Schema do +defmodule Helper.Converter.EditorToHTML.Validator.EditorSchema do @moduledoc false # header diff --git a/lib/helper/converter/editor_to_html/validator/index.ex b/lib/helper/converter/editor_to_html/validator/index.ex index 396e44d0d..ba52c5ea2 100644 --- a/lib/helper/converter/editor_to_html/validator/index.ex +++ b/lib/helper/converter/editor_to_html/validator/index.ex @@ -1,9 +1,10 @@ defmodule Helper.Converter.EditorToHTML.Validator do @moduledoc false - alias Helper.{Utils, ValidateBySchema} + alias Helper.{Converter, Validator} - alias Helper.Converter.EditorToHTML.Validator.Schema + alias Validator.Schema + alias Converter.EditorToHTML.Validator.EditorSchema # blocks with no children items @simple_blocks ["header", "paragraph"] @@ -22,14 +23,14 @@ defmodule Helper.Converter.EditorToHTML.Validator do e in RuntimeError -> format_parse_error(e) - e -> + _ -> format_parse_error() end end end defp validate_editor_fmt(data) do - validate_with("editor", Schema.get("editor"), data) + validate_with("editor", EditorSchema.get("editor"), data) end defp validate_blocks([]), do: {:ok, :pass} @@ -45,19 +46,19 @@ defmodule Helper.Converter.EditorToHTML.Validator do # validate block which have no nested items defp validate_block(%{"type" => type, "data" => data}) when type in @simple_blocks do - validate_with(type, Schema.get(type), data) + validate_with(type, EditorSchema.get(type), data) end # validate block which has mode and items - defp validate_block(%{"type" => type, "data" => %{"mode" => mode, "items" => items} = data}) + defp validate_block(%{"type" => type, "data" => %{"mode" => _, "items" => _} = data}) when type in @complex_blocks do - [parent: parent_schema, item: item_schema] = Schema.get(type) + [parent: parent_schema, item: item_schema] = EditorSchema.get(type) validate_with(type, parent_schema, item_schema, data) end defp validate_block(%{"type" => "code"}) do # schema = %{text: [:string]} - # case ValidateBySchema.cast(schema, data) do + # case Schema.cast(schema, data) do # {:error, errors} -> # format_parse_error("paragraph", errors) @@ -72,7 +73,7 @@ defmodule Helper.Converter.EditorToHTML.Validator do # validate with given schema defp validate_with(block, schema, data) do - case ValidateBySchema.cast(schema, data) do + case Schema.cast(schema, data) do {:error, errors} -> format_parse_error(block, errors) @@ -82,7 +83,7 @@ defmodule Helper.Converter.EditorToHTML.Validator do end defp validate_with(block, parent_schema, item_schema, data) do - case ValidateBySchema.cast(parent_schema, data) do + case Schema.cast(parent_schema, data) do {:error, errors} -> format_parse_error(block, errors) @@ -93,7 +94,7 @@ defmodule Helper.Converter.EditorToHTML.Validator do %{"mode" => mode, "items" => items} = data Enum.each(items, fn item -> - case ValidateBySchema.cast(item_schema, item) do + case Schema.cast(item_schema, item) do {:error, errors} -> {:error, message} = format_parse_error("#{block}(#{mode})", errors) raise %MatchError{term: {:error, message}} @@ -106,16 +107,6 @@ defmodule Helper.Converter.EditorToHTML.Validator do {:ok, :pass} end - # check if the given map has the right key-value fmt of the editorjs structure - defp is_valid_editorjs_fmt(map) when is_map(map) do - Map.has_key?(map, "time") and - Map.has_key?(map, "version") and - Map.has_key?(map, "blocks") and - is_list(map["blocks"]) and - is_binary(map["version"]) and - is_integer(map["time"]) - end - defp format_parse_error(type, error_list) when is_list(error_list) do {:error, Enum.map(error_list, fn error -> diff --git a/lib/helper/validate_by_schema.ex b/lib/helper/validator/schema.ex similarity index 58% rename from lib/helper/validate_by_schema.ex rename to lib/helper/validator/schema.ex index 1719f84f4..bef9f2969 100644 --- a/lib/helper/validate_by_schema.ex +++ b/lib/helper/validator/schema.ex @@ -1,40 +1,12 @@ -defmodule Helper.ValidateBySchema.Matchers do - @moduledoc """ - matchers for basic type, support required option - """ - - defmacro __using__(types) do - # can not use Enum.each here, see https://elixirforum.com/t/define-multiple-modules-in-macro-only-last-one-gets-created/1654/4 - for type <- types do - guard_name = if type == :string, do: "is_binary", else: "is_#{to_string(type)}" - - quote do - defp match(field, nil, [unquote(type), required: false]), do: done(field, nil) - - defp match(field, value, [unquote(type), required: false]) - when unquote(:"#{guard_name}")(value) do - done(field, value) - end - - defp match(field, value, [unquote(type)]) when unquote(:"#{guard_name}")(value), - do: done(field, value) - - defp match(field, value, [unquote(type), required: false]), - do: error(field, value, unquote(type)) - - defp match(field, value, [unquote(type)]), do: error(field, value, unquote(type)) - end - end - end -end - -defmodule Helper.ValidateBySchema do +defmodule Helper.Validator.Schema do @moduledoc """ validate json data by given schema, mostly used in editorjs validator currently support boolean / string / number / enum """ + use Helper.Validator.Schema.Matchers, [:string, :number, :list, :boolean] + @doc """ cast data by given schema @@ -49,10 +21,8 @@ defmodule Helper.ValidateBySchema do } data = %{checked: true, label: "done"} - ValidateBySchema.cast(schema, data) + Schema.cast(schema, data) """ - use Helper.ValidateBySchema.Matchers, [:string, :number, :list, :boolean] - def cast(schema, data) do errors_info = cast_errors(schema, data) diff --git a/lib/helper/validator/schema_matchers.ex b/lib/helper/validator/schema_matchers.ex new file mode 100644 index 000000000..bcc62c774 --- /dev/null +++ b/lib/helper/validator/schema_matchers.ex @@ -0,0 +1,29 @@ +defmodule Helper.Validator.Schema.Matchers do + @moduledoc """ + matchers for basic type, support required option + """ + + defmacro __using__(types) do + # can not use Enum.each here, see https://elixirforum.com/t/define-multiple-modules-in-macro-only-last-one-gets-created/1654/4 + for type <- types do + guard_name = if type == :string, do: "is_binary", else: "is_#{to_string(type)}" + + quote do + defp match(field, nil, [unquote(type), required: false]), do: done(field, nil) + + defp match(field, value, [unquote(type), required: false]) + when unquote(:"#{guard_name}")(value) do + done(field, value) + end + + defp match(field, value, [unquote(type)]) when unquote(:"#{guard_name}")(value), + do: done(field, value) + + defp match(field, value, [unquote(type), required: false]), + do: error(field, value, unquote(type)) + + defp match(field, value, [unquote(type)]), do: error(field, value, unquote(type)) + end + end + end +end diff --git a/test/helper/converter/editor_to_html_test/header_test.exs b/test/helper/converter/editor_to_html_test/header_test.exs index c77e15cad..90cc3ca19 100644 --- a/test/helper/converter/editor_to_html_test/header_test.exs +++ b/test/helper/converter/editor_to_html_test/header_test.exs @@ -124,7 +124,7 @@ defmodule GroupherServer.Test.Helper.Converter.EditorToHTML.Header do }\">footer title content
\n
\n
" end - @tag :wip + @tag :wip2 test "wrong header format data should have invalid hint" do json = Map.merge(@editor_json, %{ From bad94288da22e9026a463417df09817621444fa6 Mon Sep 17 00:00:00 2001 From: mydearxym Date: Thu, 18 Feb 2021 11:34:44 +0800 Subject: [PATCH 21/23] refactor(editor): adjust parent schema cast logic --- .../editor_to_html/validator/editor_schema.ex | 2 +- .../editor_to_html/validator/index.ex | 5 ++-- .../editor_to_html_test/header_test.exs | 2 +- .../editor_to_html_test/index_test.exs | 2 +- .../editor_to_html_test/list_test.exs | 29 ++++++++++++++++++- 5 files changed, 34 insertions(+), 6 deletions(-) diff --git a/lib/helper/converter/editor_to_html/validator/editor_schema.ex b/lib/helper/converter/editor_to_html/validator/editor_schema.ex index 5e0af0b4a..6c72745df 100644 --- a/lib/helper/converter/editor_to_html/validator/editor_schema.ex +++ b/lib/helper/converter/editor_to_html/validator/editor_schema.ex @@ -30,7 +30,7 @@ defmodule Helper.Converter.EditorToHTML.Validator.EditorSchema do def get("list") do [ - parent: %{"mode" => [enum: @valid_list_mode]}, + parent: %{"mode" => [enum: @valid_list_mode], "items" => [:list]}, item: %{ "checked" => [:boolean], "hideLabel" => [:boolean], diff --git a/lib/helper/converter/editor_to_html/validator/index.ex b/lib/helper/converter/editor_to_html/validator/index.ex index ba52c5ea2..205b84b90 100644 --- a/lib/helper/converter/editor_to_html/validator/index.ex +++ b/lib/helper/converter/editor_to_html/validator/index.ex @@ -50,7 +50,7 @@ defmodule Helper.Converter.EditorToHTML.Validator do end # validate block which has mode and items - defp validate_block(%{"type" => type, "data" => %{"mode" => _, "items" => _} = data}) + defp validate_block(%{"type" => type, "data" => data}) when type in @complex_blocks do [parent: parent_schema, item: item_schema] = EditorSchema.get(type) validate_with(type, parent_schema, item_schema, data) @@ -85,7 +85,8 @@ defmodule Helper.Converter.EditorToHTML.Validator do defp validate_with(block, parent_schema, item_schema, data) do case Schema.cast(parent_schema, data) do {:error, errors} -> - format_parse_error(block, errors) + {:error, message} = format_parse_error(block, errors) + raise %MatchError{term: {:error, message}} _ -> {:ok, :pass} diff --git a/test/helper/converter/editor_to_html_test/header_test.exs b/test/helper/converter/editor_to_html_test/header_test.exs index 90cc3ca19..c77e15cad 100644 --- a/test/helper/converter/editor_to_html_test/header_test.exs +++ b/test/helper/converter/editor_to_html_test/header_test.exs @@ -124,7 +124,7 @@ defmodule GroupherServer.Test.Helper.Converter.EditorToHTML.Header do }\">footer title content
\n
\n
" end - @tag :wip2 + @tag :wip test "wrong header format data should have invalid hint" do json = Map.merge(@editor_json, %{ diff --git a/test/helper/converter/editor_to_html_test/index_test.exs b/test/helper/converter/editor_to_html_test/index_test.exs index 09bb27d01..5c3c9a845 100644 --- a/test/helper/converter/editor_to_html_test/index_test.exs +++ b/test/helper/converter/editor_to_html_test/index_test.exs @@ -41,7 +41,7 @@ defmodule GroupherServer.Test.Helper.Converter.EditorToHTML do assert {:ok, _} = Parser.to_html(editor_string) end - @tag :wip2 + @tag :wip test "invalid editorjs json fmt should raise error" do editor_json = %{ "invalid_time" => 1_567_250_876_713, diff --git a/test/helper/converter/editor_to_html_test/list_test.exs b/test/helper/converter/editor_to_html_test/list_test.exs index e43a160e4..4f6abe275 100644 --- a/test/helper/converter/editor_to_html_test/list_test.exs +++ b/test/helper/converter/editor_to_html_test/list_test.exs @@ -39,6 +39,33 @@ defmodule GroupherServer.Test.Helper.Converter.EditorToHTML.List do IO.inspect(converted, label: "list converted") end + @editor_json %{ + "time" => 1_567_250_876_713, + "blocks" => [ + %{ + "type" => "list", + "data" => %{ + "invalid-mode" => "", + "items" => [] + } + } + ], + "version" => "2.15.0" + } + @tag :wip2 + test "invalid list mode parse should raise error message" do + {:ok, editor_string} = Jason.encode(@editor_json) + {:error, err_msg} = Parser.to_html(editor_string) + + assert err_msg == [ + %{ + block: "list", + field: "mode", + message: "should be: checklist | order_list | unorder_list" + } + ] + end + @editor_json %{ "time" => 1_567_250_876_713, "blocks" => [ @@ -61,7 +88,7 @@ defmodule GroupherServer.Test.Helper.Converter.EditorToHTML.List do ], "version" => "2.15.0" } - @tag :wip2 + @tag :wip test "invalid list data parse should raise error message" do {:ok, editor_string} = Jason.encode(@editor_json) {:error, err_msg} = Parser.to_html(editor_string) From e8d940ef04de42caf9848d86acc0498aba84312e Mon Sep 17 00:00:00 2001 From: mydearxym Date: Thu, 18 Feb 2021 13:08:20 +0800 Subject: [PATCH 22/23] refactor(editor): validate_with logic re-org --- .../editor_to_html/validator/index.ex | 41 ++++++++----------- .../editor_to_html_test/list_test.exs | 4 +- 2 files changed, 20 insertions(+), 25 deletions(-) diff --git a/lib/helper/converter/editor_to_html/validator/index.ex b/lib/helper/converter/editor_to_html/validator/index.ex index 205b84b90..a0a9f1def 100644 --- a/lib/helper/converter/editor_to_html/validator/index.ex +++ b/lib/helper/converter/editor_to_html/validator/index.ex @@ -30,7 +30,15 @@ defmodule Helper.Converter.EditorToHTML.Validator do end defp validate_editor_fmt(data) do - validate_with("editor", EditorSchema.get("editor"), data) + try do + validate_with("editor", EditorSchema.get("editor"), data) + rescue + e in MatchError -> + format_parse_error(e) + + _ -> + format_parse_error() + end end defp validate_blocks([]), do: {:ok, :pass} @@ -69,13 +77,15 @@ defmodule Helper.Converter.EditorToHTML.Validator do end defp validate_block(%{"type" => type}), do: raise("undown #{type} block") + defp validate_block(e), do: raise("undown block: #{e}") # validate with given schema defp validate_with(block, schema, data) do case Schema.cast(schema, data) do {:error, errors} -> - format_parse_error(block, errors) + {:error, message} = format_parse_error(block, errors) + raise %MatchError{term: {:error, message}} _ -> {:ok, :pass} @@ -83,29 +93,14 @@ defmodule Helper.Converter.EditorToHTML.Validator do end defp validate_with(block, parent_schema, item_schema, data) do - case Schema.cast(parent_schema, data) do - {:error, errors} -> - {:error, message} = format_parse_error(block, errors) - raise %MatchError{term: {:error, message}} + with {:ok, _} <- validate_with(block, parent_schema, data), + %{"mode" => mode, "items" => items} <- data do + Enum.each(items, fn item -> + validate_with("#{block}(#{mode})", item_schema, item) + end) - _ -> - {:ok, :pass} + {:ok, :pass} end - - %{"mode" => mode, "items" => items} = data - - Enum.each(items, fn item -> - case Schema.cast(item_schema, item) do - {:error, errors} -> - {:error, message} = format_parse_error("#{block}(#{mode})", errors) - raise %MatchError{term: {:error, message}} - - _ -> - {:ok, :pass} - end - end) - - {:ok, :pass} end defp format_parse_error(type, error_list) when is_list(error_list) do diff --git a/test/helper/converter/editor_to_html_test/list_test.exs b/test/helper/converter/editor_to_html_test/list_test.exs index 4f6abe275..048f06071 100644 --- a/test/helper/converter/editor_to_html_test/list_test.exs +++ b/test/helper/converter/editor_to_html_test/list_test.exs @@ -31,7 +31,7 @@ defmodule GroupherServer.Test.Helper.Converter.EditorToHTML.List do ], "version" => "2.15.0" } - @tag :wip + @tag :wip2 test "valid list parse should work" do {:ok, editor_string} = Jason.encode(@editor_json) # assert {:ok, converted} = Parser.to_html(editor_string) @@ -52,7 +52,7 @@ defmodule GroupherServer.Test.Helper.Converter.EditorToHTML.List do ], "version" => "2.15.0" } - @tag :wip2 + @tag :wip test "invalid list mode parse should raise error message" do {:ok, editor_string} = Jason.encode(@editor_json) {:error, err_msg} = Parser.to_html(editor_string) From 0d43e6a2e7d6aa8a36cf6864d072d5eeee6ec5e9 Mon Sep 17 00:00:00 2001 From: mydearxym Date: Thu, 18 Feb 2021 14:51:28 +0800 Subject: [PATCH 23/23] refactor(editor): adjust xss test && clean up --- lib/helper/converter/editor_to_html/index.ex | 4 -- .../editor_to_html/validator/index.ex | 12 ----- .../editor_to_html_test/index_test.exs | 51 ++++++++++++++++--- .../editor_to_html_test/list_test.exs | 4 +- 4 files changed, 46 insertions(+), 25 deletions(-) diff --git a/lib/helper/converter/editor_to_html/index.ex b/lib/helper/converter/editor_to_html/index.ex index 0e7192fec..30dbf60c2 100644 --- a/lib/helper/converter/editor_to_html/index.ex +++ b/lib/helper/converter/editor_to_html/index.ex @@ -131,9 +131,5 @@ defmodule Helper.Converter.EditorToHTML do "
[unknow block]
" end - defp invalid_hint(part, message) do - "
[invalid-block] #{part}:#{message}
" - end - def string_to_json(string), do: Jason.decode(string) end diff --git a/lib/helper/converter/editor_to_html/validator/index.ex b/lib/helper/converter/editor_to_html/validator/index.ex index a0a9f1def..6775fb937 100644 --- a/lib/helper/converter/editor_to_html/validator/index.ex +++ b/lib/helper/converter/editor_to_html/validator/index.ex @@ -64,18 +64,6 @@ defmodule Helper.Converter.EditorToHTML.Validator do validate_with(type, parent_schema, item_schema, data) end - defp validate_block(%{"type" => "code"}) do - # schema = %{text: [:string]} - # case Schema.cast(schema, data) do - # {:error, errors} -> - # format_parse_error("paragraph", errors) - - # _ -> - # {:ok, :pass} - # end - {:ok, :pass} - end - defp validate_block(%{"type" => type}), do: raise("undown #{type} block") defp validate_block(e), do: raise("undown block: #{e}") diff --git a/test/helper/converter/editor_to_html_test/index_test.exs b/test/helper/converter/editor_to_html_test/index_test.exs index 5c3c9a845..83b8016ef 100644 --- a/test/helper/converter/editor_to_html_test/index_test.exs +++ b/test/helper/converter/editor_to_html_test/index_test.exs @@ -2,19 +2,25 @@ defmodule GroupherServer.Test.Helper.Converter.EditorToHTML do @moduledoc false use GroupherServerWeb.ConnCase, async: true + + alias Helper.Metric alias Helper.Converter.EditorToHTML, as: Parser # alias Helper.Metric # @clazz Metric.Article.class_names(:html) + # "hello Editor.js workspace. is an element <script>alert("hello")</script>" + + # "text" : "" + @clazz Metric.Article.class_names(:html) + @real_editor_data ~S({ "time" : 1567250876713, "blocks" : [ { - "type" : "code", + "type" : "paragraph", "data" : { - "lang" : "js", - "text" : "" + "text": "content" } } ], @@ -113,12 +119,43 @@ defmodule GroupherServer.Test.Helper.Converter.EditorToHTML do describe "[secure issues]" do @tag :wip test "code block should avoid potential xss script attack" do - {:ok, converted} = Parser.to_html(@real_editor_data) + editor_json = %{ + "time" => 1_567_250_876_713, + "blocks" => [ + %{ + "type" => "paragraph", + "data" => %{ + "text" => "" + } + } + ], + "version" => "2.15.0" + } + + {:ok, editor_string} = Jason.encode(editor_json) + {:ok, converted} = Parser.to_html(editor_string) + + assert converted == + "

evel script

" - safe_script = - "
<script>evil scripts</script>
" + editor_json = %{ + "time" => 1_567_250_876_713, + "blocks" => [ + %{ + "type" => "paragraph", + "data" => %{ + "text" => "Editor.js is an element <script>evel script</script>" + } + } + ], + "version" => "2.15.0" + } + + {:ok, editor_string} = Jason.encode(editor_json) + {:ok, converted} = Parser.to_html(editor_string) - assert converted |> String.contains?(safe_script) + assert converted == + "

Editor.js is an element <script>evel script</script>

" end end end diff --git a/test/helper/converter/editor_to_html_test/list_test.exs b/test/helper/converter/editor_to_html_test/list_test.exs index 048f06071..8c7fbda27 100644 --- a/test/helper/converter/editor_to_html_test/list_test.exs +++ b/test/helper/converter/editor_to_html_test/list_test.exs @@ -6,7 +6,7 @@ defmodule GroupherServer.Test.Helper.Converter.EditorToHTML.List do alias Helper.Metric alias Helper.Converter.EditorToHTML, as: Parser - @clazz Metric.Article.class_names(:html) + # @clazz Metric.Article.class_names(:html) describe "[list block unit]" do @editor_json %{ @@ -31,7 +31,7 @@ defmodule GroupherServer.Test.Helper.Converter.EditorToHTML.List do ], "version" => "2.15.0" } - @tag :wip2 + @tag :wip test "valid list parse should work" do {:ok, editor_string} = Jason.encode(@editor_json) # assert {:ok, converted} = Parser.to_html(editor_string)