Skip to content

Commit

Permalink
add runtime_value/runtime_validation validation type re:#29
Browse files Browse the repository at this point in the history
  • Loading branch information
@andreineculau
andreineculau committed Mar 7, 2016
1 parent 0015e47 commit 3c5f97e
Show file tree
Hide file tree
Showing 5 changed files with 560 additions and 8 deletions.
71 changes: 68 additions & 3 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,8 +41,9 @@ default, one can do `{..., "{{_}}": "{{unexpected}}"}` or
are expected beyond the ones defined.

For more complex validations, KATT supports extensible validation types.
An example is the built-in "set" validation type for JSON,
which will ignore the order of an array's items, and just check for existence:
Built-in validation types: `set`, `runtime_value`, `runtime_validation`.

`set` will ignore the order of an array's items, and just check for existence:

```
{
Expand All @@ -53,10 +54,74 @@ which will ignore the order of an array's items, and just check for existence:
}
```

The above example would validate against JSON instances such as
So the above would validate against JSON instances such as
`{"some_array": [1, 3, 2]}`, or `{"some_array": [3, 2, 1]}`,
or even `{"some_array": [4, 3, 2, 1]}` unless we add `{{unexpected}}`.

`runtime_value` would just run code (only `erlang` and `shell` supported for now),
while having access to `ParentKey`, `Actual`, `Unexpected` and `Callbacks`,
and return the expected value and matched against the actual one.

```
{
"rfc1123": {
"{{type}}": "runtime_validation",
"erlang": "list_to_binary(httpd_util:rfc1123_date(calendar:now_to_datetime(erlang:now())))"
}
}
```

or in array format

```
{
"rfc1123": {
"{{type}}": "runtime_validation",
"erlang": ["list_to_binary(",
" httpd_util:rfc1123_date(",
" calendar:now_to_datetime(",
" erlang:now()",
")))"
]
}
}
```

`runtime_validation` would just run code (only `erlang` and `shell` supported for now),
while having access to `ParentKey`, `Actual`, `Unexpected` and `Callbacks`,
and return

* `{pass, [{"Key", "Value"}]}` i.e. validation passed, store new param "Key" with value "Value"
* `{not_equal, {Key, Expected, Actual}}`
* `{not_equal, {Key, Expected, Actual, [{"more", "info"}]}}`

```
{
"rfc1123": {
"{{type}}": "runtime_validation",
"erlang": "Expected = httpd_util:rfc1123_date(calendar:now_to_datetime(erlang:now())), case Actual =:= Expected of true -> {pass, []}; false -> {not_equal, {ParentKey, Expected, Actual}} end"
}
}
```

or in array format

```
{
"rfc1123": {
"{{type}}": "runtime_validation",
"erlang": ["Expected = httpd_util:rfc1123_date(calendar:now_to_datetime(erlang:now())),",
"case Actual =:= Expected of",
" true ->",
" {pass, []};",
" false ->",
" {not_equal, {ParentKey, Expected, Actual}}",
"end"
]
}
}
```

## Examples

A simple example that will make requests to a third party server:
Expand Down
42 changes: 39 additions & 3 deletions src/katt_callbacks_json.erl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
, parse/5
, validate_body/4
, validate_type/7
, parse_json/1
]).

%%%_* Includes =================================================================
Expand Down Expand Up @@ -96,13 +97,15 @@ validate_body( false = _Justcheck


validate_type( true = _JustCheck
, "set"
, Type
, _ParentKey
, _Options
, _Actual
, _Unexpected
, _Callbacks
) ->
) when Type =:= "set" orelse
Type =:= "runtime_value" orelse
Type =:= "runtime_validation" ->
true;
validate_type( true = _JustCheck
, _Type
Expand All @@ -127,6 +130,34 @@ validate_type( false = _JustCheck
, Unexpected
, Callbacks
);
validate_type( false = _JustCheck
, "runtime_value"
, ParentKey
, Options
, Actual
, Unexpected
, Callbacks
) ->
katt_validate_type:validate_type_runtime_value( ParentKey
, Options
, Actual
, Unexpected
, Callbacks
);
validate_type( false = _JustCheck
, "runtime_validation"
, ParentKey
, Options
, Actual
, Unexpected
, Callbacks
) ->
katt_validate_type:validate_type_runtime_validation( ParentKey
, Options
, Actual
, Unexpected
, Callbacks
);
validate_type( false = _JustCheck
, _Type
, _ParentKey
Expand Down Expand Up @@ -160,7 +191,12 @@ normalize_jsx([{_, _}|_] = Items0) ->
|| {Key, Value} <- Items0
]),
Type = proplists:get_value(?TYPE, Items1, struct),
Items = proplists:delete(?TYPE, Items1),
Items = case Type of
struct ->
Items1;
_ ->
proplists:delete(?TYPE, Items1)
end,
{Type, Items};
normalize_jsx([{}] = _Items) ->
{struct, []};
Expand Down
34 changes: 32 additions & 2 deletions src/katt_util.erl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,8 @@
, validate/5
, enumerate/1
, external_http_request/6
, erl_to_list/1
, os_cmd/2
]).

%%%_* Includes =================================================================
Expand Down Expand Up @@ -99,7 +101,7 @@ insert_escape_quotes(Str) when is_list(Str) ->
run_result_to_jsx({error, Reason, Details}) ->
[ {error, true}
, {reason, Reason}
, {details, list_to_binary(io_lib:format("~p", [Details]))}
, {details, list_to_binary(erl_to_list(Details))}
];
run_result_to_jsx({ PassOrFail
, ScenarioFilename
Expand Down Expand Up @@ -141,6 +143,22 @@ external_http_request(Url, Method, Hdrs, Body, Timeout, []) ->
Error
end.

erl_to_list(Term) ->
io_lib:format("~p", [Term]).

os_cmd(Cmd, Env) ->
Opt = [ stream
, exit_status
, use_stdio
, stderr_to_stdout
, in
, eof
, hide
, {env, Env}
],
Port = open_port({spawn, Cmd}, Opt),
os_cmd_result(Port, []).

%%%_* Internal =================================================================

my_float_to_list(X) when is_float(X) ->
Expand Down Expand Up @@ -296,7 +314,7 @@ value_to_jsx(List) when is_list(List) ->
)
end;
value_to_jsx(Value) ->
list_to_binary(io_lib:format("~p", [Value])).
list_to_binary(erl_to_list(Value)).

is_valid(ParentKey, E, A) ->
case validate(ParentKey, E, A) of
Expand Down Expand Up @@ -489,3 +507,15 @@ enumerate(L) ->
lists:zip([ integer_to_list(N)
|| N <- lists:seq(0, length(L) - 1)
], L).

os_cmd_result(Port, Output) ->
receive
{Port, {data, NewOutput}} ->
os_cmd_result(Port, [Output|NewOutput]);
{Port, eof} ->
port_close(Port),
receive
{Port, {exit_status, ExitStatus}} ->
{ExitStatus, lists:flatten(Output)}
end
end.
154 changes: 154 additions & 0 deletions src/katt_validate_type.erl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@
%%%_* Exports ==================================================================
%% API
-export([ validate_type_set/5
, validate_type_runtime_value/5
, validate_type_runtime_validation/5
]).

%%%_* Includes =================================================================
Expand Down Expand Up @@ -54,6 +56,158 @@ validate_type_set( ParentKey
validate_type_set(ParentKey, Options, Actual, _Unexpected, _Callbacks) ->
[{not_equal, {ParentKey, Options, Actual}}].

-spec validate_type_runtime_value( string()
, proplists:proplist()
, proplists:proplist()
, term()
, callbacks()
) -> pass | [validation_failure()].
validate_type_runtime_value( ParentKey
, [{"erlang", Erlang0} | _Options]
, Actual
, Unexpected
, Callbacks
) ->
Erlang = case Erlang0 of
{array, ErlangLines} ->
string:join(lists:map(fun({_, V}) -> V end, ErlangLines), "\n");
_ ->
Erlang0
end ++ ".",
{Error, Expected} =
try
{ok, Tokens, _} = erl_scan:string(Erlang),
{ok, Exprs} = erl_parse:parse_exprs(Tokens),
{value, Expected0, _} = erl_eval:exprs( Exprs
, [ {'ParentKey', ParentKey}
, {'Actual', Actual}
, {'Unexpected', Unexpected}
, {'Callbacks', Callbacks}]
),
{undefined, Expected0}
catch
C:E ->
{ Erlang ++ "~n"
++ katt_util:erl_to_list(C) ++ ":" ++ katt_util:erl_to_list(E)
, undefined
}
end,
case Error of
undefined ->
katt_util:validate(ParentKey, Expected, Actual, Unexpected, Callbacks);
_ ->
{not_equal, {ParentKey, Error, Actual}}
end;
validate_type_runtime_value( ParentKey
, [{"shell", Shell0} | _Options]
, Actual
, Unexpected
, Callbacks
) ->
Shell = case Shell0 of
{array, ShellLines} ->
string:join(lists:map(fun({_, V}) -> V end, ShellLines), "\n");
_ ->
Shell0
end,
try
{0, Erlang} = katt_util:os_cmd( Shell
, [ {"KATT_PARENT_KEY", ParentKey}
, { "KATT_ACTUAL"
, io_lib:format("~p", [Actual])
}
, { "KATT_UNEXPECTED"
, io_lib:format("~p", [Unexpected])
}
]),
validate_type_runtime_value( ParentKey
, [{"erlang", Erlang}]
, Actual
, Unexpected
, Callbacks
)
catch
C:E ->
Error = Shell ++ "~n"
++ katt_util:erl_to_list(C) ++ ":" ++ katt_util:erl_to_list(E),
{not_equal, {ParentKey, Error, Actual}}
end.


-spec validate_type_runtime_validation( string()
, proplists:proplist()
, proplists:proplist()
, term()
, callbacks()
) -> pass | [validation_failure()].
validate_type_runtime_validation( ParentKey
, [{"erlang", Erlang0} | _Options]
, Actual
, Unexpected
, Callbacks
) ->
Erlang = case Erlang0 of
{array, ErlangLines} ->
string:join(lists:map(fun({_, V}) -> V end, ErlangLines), "\n");
_ ->
Erlang0
end ++ ".",
try
{ok, Tokens, _} = erl_scan:string(Erlang),
{ok, Exprs} = erl_parse:parse_exprs(Tokens),
{value, Result, _} = erl_eval:exprs( Exprs
, [ {'ParentKey', ParentKey}
, {'Actual', Actual}
, {'Unexpected', Unexpected}
, {'Callbacks', Callbacks}]
),
Result
catch
C:E ->
Error = { Erlang ++ "~n"
++ katt_util:erl_to_list(C) ++ ":" ++ katt_util:erl_to_list(E)
, undefined
},
{not_equal, {ParentKey, Error, Actual}}
end;
validate_type_runtime_validation( ParentKey
, [{"shell", Shell0} | _Options]
, Actual
, Unexpected
, Callbacks
) ->
Shell = case Shell0 of
{array, ShellLines} ->
string:join(lists:map(fun({_, V}) -> V end, ShellLines), "\n");
_ ->
Shell0
end,
try
{0, Erlang} = katt_util:os_cmd( Shell
, [ {"KATT_PARENT_KEY", ParentKey}
, { "KATT_ACTUAL"
, io_lib:format("~p", [Actual])
}
, { "KATT_UNEXPECTED"
, io_lib:format("~p", [Unexpected])
}
]),
validate_type_runtime_validation( ParentKey
, [{"erlang", Erlang}]
, Actual
, Unexpected
, Callbacks
)
catch
C:E ->
Error = { Shell ++ "~n"
++ katt_util:erl_to_list(C) ++ ":" ++ katt_util:erl_to_list(E)
, undefined
},
{not_equal, {ParentKey, Error, Actual}}
end.


%%%_* Internal =================================================================

-spec validate_set( string()
Expand Down

0 comments on commit 3c5f97e

Please sign in to comment.