Optimize while_op for test #14764

Xreki · 2018-12-06T02:57:08Z

Avoid creating and deleting variables in every iterator when is_test is set.
~~Add in dispensable input ExecutorPrepareContext in while_op, so that the operators of this block can be created in advance.~~ move to another PR.
Fix some missing deps in cmake.
Use async TensorCopy instead of TensorCopySync in reshape_op.
Simplify the CPU kernel of compare_op when numel() of input tensor is 1.

test=develop

… core_opt_while_op test=develop

test=develop

Xreki · 2019-01-04T06:23:31Z

Met following linking error in PR_CI_python35, see building log:

../../operators/distributed/libsendrecvop_rpc.a(grpc_client.cc.o): In function `std::_Function_handler<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> (), std::__future_base::_Task_setter<std::unique_ptr<std::__future_base::_Result<std::unique_ptr<paddle::platform::EnforceNotMet, std::default_delete<paddle::platform::EnforceNotMet> > >, std::__future_base::_Result_base::_Deleter>, std::_Bind_simple<std::reference_wrapper<std::future<std::unique_ptr<paddle::platform::EnforceNotMet, std::default_delete<paddle::platform::EnforceNotMet> > > paddle::framework::ThreadPool::RunAndGetException<paddle::operators::distributed::GRPCClient::AsyncSendVar(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, paddle::platform::DeviceContext const&, paddle::framework::Scope const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, long)::{lambda()#1}>(paddle::operators::distributed::GRPCClient::AsyncSendVar(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, paddle::platform::DeviceContext const&, paddle::framework::Scope const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, long)::{lambda()#1})::{lambda()#1}> ()>, std::unique_ptr<paddle::platform::EnforceNotMet, std::default_delete<paddle::platform::EnforceNotMet> > > >::_M_invoke(std::_Any_data const&)':^M 
grpc_client.cc:(.text+0x28ec): undefined reference to `grpc::GenericStub::PrepareUnaryCall(grpc::ClientContext*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, grpc::ByteBuffer const&, grpc::CompletionQueue*)'^M
../../operators/distributed/libsendrecvop_rpc.a(grpc_client.cc.o): In function `std::_Function_handler<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> (), std::__future_base::_Task_setter<std::unique_ptr<std::__future_base::_Result<std::unique_ptr<paddle::platform::EnforceNotMet, std::default_delete<paddle::platform::EnforceNotMet> > >, std::__future_base::_Result_base::_Deleter>, std::_Bind_simple<std::reference_wrapper<std::future<std::unique_ptr<paddle::platform::EnforceNotMet, std::default_delete<paddle::platform::EnforceNotMet> > > paddle::framework::ThreadPool::RunAndGetException<paddle::operators::distributed::GRPCClient::AsyncPrefetchVar(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, paddle::platform::DeviceContext const&, paddle::framework::Scope const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, long)::{lambda()#1}>(paddle::operators::distributed::GRPCClient::AsyncPrefetchVar(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, paddle::platform::DeviceContext const&, paddle::framework::Scope const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, long)::{lambda()#1})::{lambda()#1}> ()>, std::unique_ptr<paddle::platform::EnforceNotMet, std::default_delete<paddle::platform::EnforceNotMet> > > >::_M_invoke(std::_Any_data const&)':^M
grpc_client.cc:(.text+0x304c): undefined reference to `grpc::GenericStub::PrepareUnaryCall(grpc::ClientContext*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, grpc::ByteBuffer const&, grpc::CompletionQueue*)'^M
../../operators/distributed/libsendrecvop_rpc.a(grpc_client.cc.o): In function `paddle::operators::distributed::GRPCClient::GetChannel(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)':^M
grpc_client.cc:(.text+0x3683): undefined reference to `grpc::InsecureChannelCredentials()'^M
grpc_client.cc:(.text+0x369c): undefined reference to `grpc::CreateCustomChannel(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::shared_ptr<grpc::ChannelCredentials> const&, grpc::ChannelArguments const&)'
../../operators/distributed/libsendrecvop_rpc.a(grpc_client.cc.o): In function `std::_Function_handler<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> (), std::__future_base::_Task_setter<std::unique_ptr<std::__future_base::_Result<std::unique_ptr<paddle::platform::EnforceNotMet, std::default_delete<paddle::platform::EnforceNotMet> > >, std::__future_base::_Result_base::_Deleter>, std::_Bind_simple<std::reference_wrapper<std::future<std::unique_ptr<paddle::platform::EnforceNotMet, std::default_delete<paddle::platform::EnforceNotMet> > > paddle::framework::ThreadPool::RunAndGetException<paddle::operators::distributed::GRPCClient::_AsyncGetVar(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, paddle::platform::DeviceContext const&, paddle::framework::Scope const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, long)::{lambda()#1}>(paddle::operators::distributed::GRPCClient::_AsyncGetVar(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, paddle::platform::DeviceContext const&, paddle::framework::Scope const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char,[ 82%] ^[[34m^[[1mGenerating paddle_fluid.dir/read_op.objdir^[[0m^M
 std::char_traits<char>, std::allocator<char> > const&, long)::{lambda()#1})::{lambda()#1}> ()>, std::unique_ptr<paddle::platform::EnforceNotMet, std::default_delete<paddle::platform::EnforceNotMet> > > >::_M_invoke(std::_Any_data const&)':^M
grpc_client.cc:(.text+0x837e): undefined reference to `grpc::GenericStub::PrepareUnaryCall(grpc::ClientContext*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, grpc::ByteBuffer const&, grpc::CompletionQueue*)'^M
../../operators/distributed/libsendrecvop_rpc.a(grpc_client.cc.o): In function `paddle::operators::distributed::BaseProcessor::Prepare(std::shared_ptr<paddle::operators::distributed::VarHandle>, long)':^M
grpc_client.cc:(.text._ZN6paddle9operators11distributed13BaseProcessor7PrepareESt10shared_ptrINS1_9VarHandleEEl[_ZN6paddle9operators11distributed13BaseProcessor7PrepareESt10shared_ptrINS1_9VarHandleEEl]+0xcb): undefined reference to `grpc::Timepoint2Timespec(std::chrono::time_point<std::chrono::_V2::system_clock, std::chrono::duration<long, std::ratio<1l, 1000000000l> > > const&, gpr_timespec*)'
../../operators/distributed/libsendrecvop_rpc.a(grpc_server.cc.o): In function `paddle::operators::distributed::AsyncGRPCServer::StartServer()':
grpc_server.cc:(.text+0x24b0): undefined reference to `grpc::ServerBuilder::ServerBuilder()'
grpc_server.cc:(.text+0x24b8): undefined reference to `grpc::InsecureServerCredentials()'
grpc_server.cc:(.text+0x24cd): undefined reference to `grpc::ServerBuilder::AddListeningPort(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::shared_ptr<grpc::ServerCredentials>, int*)'
grpc_server.cc:(.text+0x2530): undefined reference to `grpc::ServerBuilder::SetOption(std::unique_ptr<grpc::ServerBuilderOption, std::default_delete<grpc::ServerBuilderOption> >)'
grpc_server.cc:(.text+0x255c): undefined reference to `grpc::ServerBuilder::RegisterService(grpc::Service*)'
grpc_server.cc:(.text+0x25f4): undefined reference to `grpc::ServerBuilder::AddCompletionQueue(bool)'
grpc_server.cc:(.text+0x273c): undefined reference to `grpc::ServerBuilder::BuildAndStart()'
grpc_server.cc:(.text+0x2ca7): undefined reference to `grpc::ServerBuilder::~ServerBuilder()'
grpc_server.cc:(.text+0x3621): undefined reference to `grpc::ServerBuilder::~ServerBuilder()'
collect2: error: ld returned 1 exit status
paddle/fluid/inference/analysis/CMakeFiles/test_analyzer.dir/build.make:522: recipe for target 'paddle/fluid/inference/analysis/test_analyzer' failed
make[2]: *** [paddle/fluid/inference/analysis/test_analyzer] Error 1
CMakeFiles/Makefile2:62669: recipe for target 'paddle/fluid/inference/analysis/CMakeFiles/test_analyzer.dir/all' failed
make[1]: *** [paddle/fluid/inference/analysis/CMakeFiles/test_analyzer.dir/all] Error 2
make[1]: *** Waiting for unfinished jobs....

CLAassistant · 2019-01-08T07:15:11Z

All committers have signed the CLA.

test=develop

Xreki · 2019-01-09T03:50:35Z

The following unittests failed in CI:

[11:18:17][Step 1/1] The following tests FAILED:
[11:18:17][Step 1/1] 	136 - test_analyzer_rnn1 (SEGFAULT)
[11:18:17][Step 1/1] 	137 - test_analyzer_rnn2 (SEGFAULT)
[11:18:17][Step 1/1] 	138 - test_analyzer_dam (SEGFAULT)
[11:18:17][Step 1/1] 	139 - test_analyzer_small_dam (SEGFAULT)
[11:18:17][Step 1/1] 	140 - test_analyzer_ner (SEGFAULT)
[11:18:17][Step 1/1] 	141 - test_analyzer_lac (SEGFAULT)
[11:18:17][Step 1/1] 	142 - test_analyzer_mm_dnn (SEGFAULT)
[11:18:17][Step 1/1] 	143 - test_analyzer_text_classification (SEGFAULT)
[11:18:17][Step 1/1] 	144 - test_analyzer_seq_conv1 (SEGFAULT)
[11:18:17][Step 1/1] 	145 - test_analyzer_seq_pool1 (SEGFAULT)
[11:18:17][Step 1/1] 	146 - test_analyzer_ocr (SEGFAULT)
[11:18:17][Step 1/1] 	147 - test_analyzer_mobilenet_transpose (SEGFAULT)
[11:18:17][Step 1/1] 	148 - test_analyzer_resnet50 (SEGFAULT)
[11:18:17][Step 1/1] 	149 - test_analyzer_mobilenet_depthwise_conv (SEGFAULT)
[11:18:17][Step 1/1] Errors while running CTest
[11:18:17][Step 1/1] 	152 - test_trt_models (SEGFAULT

It is fixed in #15222 .

test=develop

luotao1

LGTM for cmake part.

panyx0718 · 2019-01-10T12:31:55Z

paddle/fluid/operators/controlflow/while_op.cc

+        auto &current_scope = scope.NewScope();
+        step_scopes->push_back(&current_scope);
+        executor.RunPreparedContext(ctx_ptr, &current_scope, false, true, true);
+        if (is_test) {


panyx0718 · 2019-01-10T12:37:47Z

paddle/fluid/operators/controlflow/while_op.cc

-      executor.RunPreparedContext(ctx.get(), &current_scope, false, true, true);
-      if (is_test) {
-        scope.DeleteScope(&current_scope);
+      executor.CreateVariables(*program, &current_scope, block->ID());


why inference doesn't need to create variables but train need?

Paddle/paddle/fluid/framework/executor.cc

Lines 413 to 423 in 5d9edb4

void Executor::RunPreparedContext(ExecutorPrepareContext* ctx, Scope* scope,

bool create_local_scope, bool create_vars,

bool keep_kids) {

PADDLE_ENFORCE_NOT_NULL(scope);

Scope* local_scope = scope;

if (create_vars) {

if (create_local_scope) {

local_scope = &scope->NewScope();

}

CreateVariables(ctx->prog_, local_scope, ctx->block_id_);

}

When create_vars is set to true, variables will be created in RunPreparedContext. Because the step scopes will be used in while_grad_op, so all the step scopes should be saved. So that if there is 100 iterations, CreatedVariables will be called 100 times.

Paddle/paddle/fluid/operators/controlflow/while_op.cc

Lines 84 to 86 in 5d9edb4

if (is_test) {

scope.DeleteScope(&current_scope);

}

For test, the step scopes is deleted at the end of the iteration. In fact, there is no need to new 100 step scopes and call CreateVariables 100 times. Instead, 1 time is enough. This can reduce the overhead a lot, from 17.8619ms to 16.0768ms.

panyx0718 · 2019-01-10T12:40:18Z

paddle/fluid/inference/api/details/prepare_executor.cc

+namespace api {
+namespace details {
+
+static void CheckWhileOpInput(framework::ProgramDesc *program) {


This is not "check", this is "change"

panyx0718 · 2019-01-10T12:42:27Z

paddle/fluid/inference/api/details/prepare_executor.cc

+        framework::Variable *ptr = scope->Var(context_var->Name());
+        framework::InitializeVariable(ptr, context_var->GetType());
+
+        auto *tensor = ptr->GetMutable<framework::LoDTensor>();


why is this LoDTensor, not ExecutorPrepareContext

If we want to use ExecutorPrepareContext directly, we need to add a new type to the var's type list. We need to do so much changes, first here:

Paddle/paddle/fluid/framework/framework.proto

Lines 105 to 135 in 5d9edb4

message VarType {

enum Type {

// Pod Types

BOOL = 0;

INT16 = 1;

INT32 = 2;

INT64 = 3;

FP16 = 4;

FP32 = 5;

FP64 = 6;

// Tensor<size_t> is used in C++.

SIZE_T = 19;

UINT8 = 20;

INT8 = 21;

// Other types that may need additional descriptions

LOD_TENSOR = 7;

SELECTED_ROWS = 8;

FEED_MINIBATCH = 9;

FETCH_LIST = 10;

STEP_SCOPES = 11;

LOD_RANK_TABLE = 12;

LOD_TENSOR_ARRAY = 13;

PLACE_LIST = 14;

READER = 15;

// Any runtime decided variable type is raw

// raw variables should manage their own allocations

// in operators like nccl_op

RAW = 17;

TUPLE = 18;

}

Do you think that is better?

Does RAW work?

I try it and was trapped in complicated cmake dependent errors. The change leads to circle dependent in cmake. I'd like to delete these codes and try to fix this in another PR, is that OK?

panyx0718 · 2019-01-10T12:43:36Z

paddle/fluid/inference/api/api_impl.cc

-  executor_->CreateVariables(*inference_program_,
-                             sub_scope_ ? sub_scope_ : scope_.get(), 0);
+  framework::Scope *scope = sub_scope_ != nullptr ? sub_scope_ : scope_.get();
+  inference::api::details::PrepareExecutor(inference_program_.get(),


should this live in a pass?

In fact, it is not suitable to implement this in a pass, because we need to create ExecutorPrepareContext for the final programs. These ExecutorPrepareContext will be used in Predictor.Run() directly.

In fact, I'm considering implementing this in

Paddle/paddle/fluid/framework/executor.cc

Lines 372 to 374 in 5d9edb4

std::unique_ptr<ExecutorPrepareContext> Executor::Prepare(

const ProgramDesc& program, int block_id,

const std::vector<std::string>& skip_ref_cnt_vars) {

But I'm not sure it is stable enough, so implement in NativePredictor first to verify that.

panyx0718 · 2019-01-10T12:44:21Z

paddle/fluid/inference/api/details/prepare_executor.h

@@ -0,0 +1,33 @@
+/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.


no need for this file?

panyx0718 · 2019-01-14T03:34:47Z

paddle/fluid/inference/api/details/prepare_executor.cc

+        framework::Variable *ptr = scope->Var(context_var->Name());
+        framework::InitializeVariable(ptr, context_var->GetType());
+
+        auto *tensor = ptr->GetMutable<framework::LoDTensor>();


Does RAW work?

panyx0718 · 2019-01-14T08:51:48Z

paddle/fluid/operators/reshape_op.cc

-    framework::TensorCopySync(*in, ctx.GetPlace(), out);
+    framework::TensorCopy(
+        *in, ctx.GetPlace(),
+        ctx.template device_context<platform::DeviceContext>(), out);


@velconia why was this copy sync?

This code is a copied from original implementation, but I guess Sync is NO need here.

…ile_op. test=develop

test=develop

* Revert the modification of while_op in #14764. test=develop * Remove the dependency of GRPC_DEPS. test=develop

Xreki added 5 commits December 4, 2018 17:50

Simplify the compare op for CPU.

7e02fc1

Use asynchronous tensor copy in reshape_op's kernel.

b65760b

Optimize while_op for test, avoiding creating variables every time.

4473895

test=develop

Merge branch 'develop' into core_opt_while_op

9619043

Enable the cache of kernel type and kernel function.

3fda0b1

test=develop

Xreki force-pushed the core_opt_while_op branch 2 times, most recently from 26a1cc8 to f377755 Compare December 10, 2018 06:51

Merge branch 'develop' into core_opt_while_op

c3963f6

test=develop

Xreki force-pushed the core_opt_while_op branch from f377755 to c3963f6 Compare December 10, 2018 08:01

Xreki added 3 commits December 10, 2018 17:54

Enable profiling with gperftools.

aec4e29

Remove flags for testing, and fix the linking error.

4d1b624

test=develop

Merge branch 'develop' into core_opt_while_op

b2aa81a

Xreki force-pushed the core_opt_while_op branch 2 times, most recently from 846b535 to 9a6ee11 Compare December 11, 2018 09:53

Delete the codes of ChooseKernel.

6c76cdd

test=develop

Xreki force-pushed the core_opt_while_op branch from 9a6ee11 to 6c76cdd Compare December 12, 2018 02:37

Merge branch 'develop' into core_opt_while_op

d18af8b

test=develop

Xreki force-pushed the core_opt_while_op branch from f443de4 to d18af8b Compare December 13, 2018 03:02

Xreki added 2 commits December 14, 2018 13:16

Fix bug when preparing ExecutorPrepareContext for while_op.

375454a

Merge branch 'develop' into core_opt_while_op

ac191ba

test=develop

Xreki force-pushed the core_opt_while_op branch from 2ae5cc3 to ac191ba Compare December 14, 2018 06:22

Xreki added 3 commits December 24, 2018 17:30

Merge branch 'develop' of https://github.com/PaddlePaddle/Paddle into…

b26bd96

… core_opt_while_op test=develop

Merge branch 'develop' into core_opt_while_op

c4486fc

test=develop

Merge branch 'develop' into core_opt_while_op

1286fd0

test=develop

Xreki added 2 commits January 8, 2019 07:36

Fix missing depending on grpc libraries.

0944f7f

Merge branch 'develop' into core_opt_while_op

b52b770

test=develop

Xreki force-pushed the core_opt_while_op branch from 9669bbf to b52b770 Compare January 8, 2019 07:39

Xreki added 2 commits January 9, 2019 11:55

Merge branch 'develop' into core_opt_while_op

135c17c

Remove the redundant print.

8476f7c

test=develop

Xreki force-pushed the core_opt_while_op branch from 7880101 to d84fb06 Compare January 9, 2019 12:56

Merge branch 'develop' into core_opt_while_op

ad927fd

test=develop

Xreki force-pushed the core_opt_while_op branch from d84fb06 to ad927fd Compare January 9, 2019 13:04

Xreki requested review from panyx0718, Superjomn, qingqing01, luotao1 and tensor-tang January 10, 2019 03:50

luotao1 reviewed Jan 10, 2019

View reviewed changes

panyx0718 reviewed Jan 10, 2019

View reviewed changes

Follow comments.

3871afa

panyx0718 reviewed Jan 14, 2019

View reviewed changes

Remove the codes related to prepare the ExecutorPrepareContext for wh…

9f38bc3

…ile_op. test=develop

panyx0718 approved these changes Jan 16, 2019

View reviewed changes

Xreki merged commit 568cc2f into PaddlePaddle:develop Jan 16, 2019

Xreki added a commit to Xreki/Paddle that referenced this pull request Jan 16, 2019

Revert the modification of while_op in PaddlePaddle#14764.

b3fce44

test=develop

Xreki mentioned this pull request Jan 16, 2019

Revert the modification of while_op in #14764. #15372

Merged

Xreki added a commit that referenced this pull request Jan 17, 2019

Revert the modification of while_op in #14764. (#15372)

f413b68

* Revert the modification of while_op in #14764. test=develop * Remove the dependency of GRPC_DEPS. test=develop

Xreki mentioned this pull request Feb 20, 2019

Optimize while_op when is_test is true. #15811

Merged

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Optimize while_op for test #14764

Optimize while_op for test #14764

Xreki commented Dec 6, 2018 •

edited

Xreki commented Jan 4, 2019

CLAassistant commented Jan 8, 2019 •

edited

Xreki commented Jan 9, 2019

luotao1 left a comment

panyx0718 Jan 10, 2019

Xreki Jan 11, 2019

panyx0718 Jan 10, 2019

Xreki Jan 11, 2019

panyx0718 Jan 10, 2019

panyx0718 Jan 10, 2019

Xreki Jan 11, 2019

panyx0718 Jan 14, 2019

Xreki Jan 15, 2019

panyx0718 Jan 10, 2019

Xreki Jan 11, 2019

panyx0718 Jan 10, 2019

panyx0718 Jan 14, 2019

panyx0718 Jan 14, 2019

velconia Jan 14, 2019

	void Executor::RunPreparedContext(ExecutorPrepareContext* ctx, Scope* scope,
	bool create_local_scope, bool create_vars,
	bool keep_kids) {
	PADDLE_ENFORCE_NOT_NULL(scope);
	Scope* local_scope = scope;
	if (create_vars) {
	if (create_local_scope) {
	local_scope = &scope->NewScope();
	}
	CreateVariables(ctx->prog_, local_scope, ctx->block_id_);
	}

	message VarType {
	enum Type {
	// Pod Types
	BOOL = 0;
	INT16 = 1;
	INT32 = 2;
	INT64 = 3;
	FP16 = 4;
	FP32 = 5;
	FP64 = 6;
	// Tensor<size_t> is used in C++.
	SIZE_T = 19;
	UINT8 = 20;
	INT8 = 21;

	// Other types that may need additional descriptions
	LOD_TENSOR = 7;
	SELECTED_ROWS = 8;
	FEED_MINIBATCH = 9;
	FETCH_LIST = 10;
	STEP_SCOPES = 11;
	LOD_RANK_TABLE = 12;
	LOD_TENSOR_ARRAY = 13;
	PLACE_LIST = 14;
	READER = 15;
	// Any runtime decided variable type is raw
	// raw variables should manage their own allocations
	// in operators like nccl_op
	RAW = 17;
	TUPLE = 18;
	}

	std::unique_ptr<ExecutorPrepareContext> Executor::Prepare(
	const ProgramDesc& program, int block_id,
	const std::vector<std::string>& skip_ref_cnt_vars) {

		@@ -0,0 +1,33 @@
		/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.

Optimize while_op for test #14764

Optimize while_op for test #14764

Conversation

Xreki commented Dec 6, 2018 • edited

Xreki commented Jan 4, 2019

CLAassistant commented Jan 8, 2019 • edited

Xreki commented Jan 9, 2019

luotao1 left a comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Xreki commented Dec 6, 2018 •

edited

CLAassistant commented Jan 8, 2019 •

edited