Merge remote-tracking branch 'origin/master' into johny-b/585-api-ref…

…erence-sphinx

.gitignore

@@ -3,6 +3,13 @@ __pycache__/
 .idea
 *.egg-info
 
+#   files created by tests and examples
+*.log
+*.png
+.coverage
+
+tests/goth_tests/assets/
+
 dist/
 poetry.lock
 

docs/diagrams/README.md

@@ -0,0 +1,14 @@
+This directory contains [PlantUML](https://plantuml.com) files for UML sequence and class diagrams
+illustrating various aspect of `yapapi` code architecture and execution.
+
+To create diagrams from `.puml` files you need `java` and `plantuml.jar`
+which can be downloaded from https://sourceforge.net/projects/plantuml/files/plantuml.jar/download.
+
+Generate `diagram.png` from the source file `diagram.puml` with:
+```shell script
+java -jar plantuml.jar diagram.puml
+```
+
+See https://plantuml.com/starting for more information on using PlantUML
+(for example, how to generate diagram files in formats other than PNG)
+and on the syntax of `.puml` files.

docs/diagrams/classes.puml

@@ -0,0 +1,60 @@
+@startuml
+title Main yapapi classes
+
+hide empty fields
+
+class Golem {
+  execute_tasks()
+  run_service()
+ }
+
+together {
+  class Executor <<AsyncContextManager>> {
+    submit()
+    add_job()
+    process_batches()
+  }
+  class Cluster <<AsyncContextManager>>
+  ' Cluster -[hidden] Executor
+}
+
+class _Engine <<AsyncContextManager>> {
+  process_batches()
+  start_worker()
+}
+
+class Job {
+  find_offers()
+}
+
+class AgreementsPool
+
+
+
+Golem --|> _Engine
+Golem .. "creates" Executor
+Golem .. "creates" Cluster
+
+Executor -- "1" _Engine
+Cluster -- "1" _Engine
+' Cluster -- "1" Job
+
+_Engine o-- "0..*" Job
+
+Job *-- "1" AgreementsPool
+
+class Payload
+' Executor --> Payload
+' Cluster --> Payload
+Job "0..*" -- "1" Payload
+
+class Service
+class MyService
+MyService --|> Service
+
+Cluster *-- "0..*" Service
+(Cluster, Service) .. ServiceInstance
+
+MyService -[hidden]-- AgreementsPool
+
+@enduml

docs/diagrams/ctrl-c.puml

@@ -0,0 +1,47 @@
+@startuml
+title Handling Ctrl+C
+
+hide footbox
+
+participant "_ _main_ _" as main
+
+participant ":EventLoop" as loop
+
+participant "task:asyncio.Task" as task
+
+activate main
+
+create loop
+main -> loop : get_event_loop()
+
+create task
+main -> task : loop.create_task()
+note right: task runs yapapi code
+
+main -> loop : run_until_complete(task)
+
+activate loop
+loop -> task ++ : <<callback>>
+deactivate
+
+loop -> task ++ : <<callback>>
+deactivate
+
+'user -> loop : Ctrl+C
+
+== Ctrl+C pressed ==
+
+return raise KeyboardInterrupt
+
+main -> main : except KeyboardInterrupt
+main -> task ++ : cancel()
+deactivate
+
+main -> loop ++ : run_until_complete(task)
+loop -> task ++ : throw CancelledError
+task -> task : except CancelledError
+note over task: handle CancelledError\ne.g. peform Executor shutdown
+return
+return
+
+@enduml

docs/diagrams/emit_events_async.puml

@@ -0,0 +1,40 @@
+@startuml
+title Emitting events using AsyncWrapper
+hide footbox
+skinparam BoxPadding 10
+
+box "yapapi 'core'"
+participant ":_Engine" as engine
+participant ":AsyncWrapper" as wrapper
+participant ":asyncio.Queue" as queue
+participant "worker" as worker <<asyncio.Task>>
+end box
+
+box "user code"
+participant "event_consumer" as consumer
+end box
+
+activate engine
+create wrapper
+engine -> wrapper : <<create>>
+
+create queue
+wrapper -> queue : <<create>>
+
+create worker
+wrapper -> worker : <<create_task>>
+activate worker
+
+worker -> queue : get()
+
+engine -> wrapper ++ : emit(event)
+
+wrapper -> queue : put(event)
+return
+
+queue -> worker : return event
+
+worker -> consumer ++ : callback(event)
+|||
+
+@enduml

docs/diagrams/emit_events_summary.puml

@@ -0,0 +1,46 @@
+@startuml
+title Processing events by SummaryLogger
+
+hide footbox
+skinparam BoxPadding 10
+
+box "yapapi 'core'"
+participant ":_Engine" as engine
+participant ":AsyncWrapper" as wrapper
+participant ":asyncio.Queue" as queue
+participant "worker" as worker <<asyncio.Task>>
+end box
+
+
+box "yapapi 'metro area'"
+participant ":SummaryLogger" as consumer
+end box
+
+participant ":logging.Logger" as logger
+
+activate engine
+create wrapper
+engine -> wrapper : <<create>>
+
+create queue
+wrapper -> queue : <<create>>
+
+create worker
+wrapper -> worker : <<create_task>>
+activate worker
+
+worker -> queue : get()
+
+engine -> wrapper ++ : emit(event)
+
+wrapper -> queue : put(event)
+return
+
+queue -> worker : return event
+
+worker -> consumer ++ : callback(event)
+consumer -> consumer ++ : handle()
+consumer -> logger : info()
+|||
+
+@enduml

docs/diagrams/execute_tasks.puml

@@ -0,0 +1,86 @@
+@startuml
+title Golem.execute_tasks()
+
+' Define colors for activation rectangles of
+!$task_1 = "#White"
+!$task_2 = "#White"
+!$task_3 = "#White"
+
+hide footbox
+skinparam BoxPadding 10
+
+box "main task" #ffffee
+participant "Client code"
+participant "golem:Golem" as engine #Turquoise
+participant "executor:Executor" as executor #Violet
+end box
+
+box "worker_starter() task" #ffeeff
+participant "executor:Executor" as executor2 #Violet
+participant "golem:Golem" as engine2 #Turquoise
+participant ":AgreementPool" as pool
+end box
+
+box "worker_task() task" #eeffff
+collections "golem:Golem" as engine3 #Turquoise
+collections "executor:Executor" as executor3 #Violet
+
+participant "batch_generator:AsyncGenerator" as batch_generator
+end box
+
+create engine
+"Client code" -> engine : <<create>>
+"Client code" -> engine ++ $task_1 : execute_tasks(worker)
+
+create executor
+engine -> executor : <<create>>
+engine -> executor ++ $task_1 : submit(worker)
+
+create pool
+executor -> pool : <<create>>
+
+executor -> executor2 ++ $task_2 : <<create_task>>
+note right: worker_starter()
+
+loop executed every 2 seconds if there is unassigned work, each iteration can create new worker_task
+  executor2 -> engine2 ++ $task_2 : start_worker(run_worker)
+
+  engine2 -> pool ++ $task_2: use_agreement(worker_task)
+  deactivate engine2
+  ?<- pool : agr = create_agreement()
+
+  pool -> engine3 ++ $task_3: <<create_task>> \n <<callback>>
+  note right: worker_task(agr)
+  deactivate pool
+end
+
+engine3 ->? : act = new_activity()
+engine3 ->? : ctx = WorkContext()
+
+engine3 -> executor3 ++ $task_3: <<callback>>
+note right: run_worker(act, ctx)
+
+create batch_generator
+executor3 -> batch_generator : <<callback>>
+note over batch_generator: worker(ctx)
+executor3 -> engine3++ : process_batches(batch_generator)
+
+|||
+
+ref over engine3, executor3, batch_generator
+  See the diagram for process_batches()
+end ref
+
+|||
+
+return
+return
+deactivate engine3
+
+legend left
+* Three background boxes represent different asyncio tasks.
+* Participants with the same name represent the same object activated by different asyncio tasks
+* <<create_task>> message represents loop.create_task() call
+endlegend
+
+@enduml

docs/diagrams/process_batches.puml

@@ -0,0 +1,58 @@
+@startuml
+title Creating and executing command batches
+hide footbox
+
+participant "executor:Executor" as executor #Violet
+participant ":Activity" as act
+participant "golem:[[../../yapapi/golem.py Golem]]" as engine #Turquoise
+participant "batch_generator:AsyncGenerator" as batch_generator
+participant ":WorkContext" as ctx
+
+executor -> engine ++ : process_batches(batch_generator)
+
+' batch 1
+engine -> batch_generator : anext()
+activate batch_generator
+batch_generator -> ctx : send_file()
+activate ctx
+deactivate ctx
+batch_generator -> ctx : run()
+activate ctx
+deactivate ctx
+batch_generator -> ctx : commit()
+activate ctx
+return batch_1
+batch_generator --> engine : yield batch_1
+
+engine -> act : send(batch_1)
+activate act
+return batch_results_1
+
+' batch 2
+engine -> batch_generator : asend(batch_results_1)
+batch_generator -> ctx : run()
+activate ctx
+deactivate ctx
+batch_generator -> ctx : download_file()
+activate ctx
+deactivate ctx
+batch_generator -> ctx: commit()
+activate ctx
+return batch_2
+batch_generator --> engine : yield batch_2
+
+engine -> act : send(batch_2)
+activate act
+return batch_results_2
+
+engine -> batch_generator : asend(batch_results_2)
+return StopIteration
+
+return
+
+deactivate executor
+deactivate executor
+
+deactivate engine
+
+@enduml