Change/Fix (#346): update cli-client

Updated code of the cli-client for the current api
and also updated the code in general.
Also fixed some bugs found while testing with the
cli-client.

.github/workflows/build_test.yml

@@ -337,10 +337,95 @@ jobs:
 
 
 
+  cli_api_tests:
+    name: "CLI-API-Tests"
+    needs: [ unit_tests, memory_leak_tests, functional_tests ]
+    runs-on: ubuntu-latest
+    steps:
+      - 
+        name: Checkout repository
+        run: |
+          # use manually clone, because with the "actions/checkout@v3" action the name of the
+          # branch can not be read by the git commands, which is necessary for the build-script
+          git clone https://github.com/kitsudaiki/${GITHUB_REPOSITORY#*/}.git
+          cd ${GITHUB_REPOSITORY#*/}
+          git checkout ${GITHUB_REF#refs/heads/}
+          git submodule init
+          git submodule update --recursive
+      - 
+        name: Get artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: result
+          path: /tmp/build_result
+      - 
+        name: Install go
+        run: |
+          wget -c https://go.dev/dl/go1.22.5.linux-amd64.tar.gz
+          sudo tar -C /usr/local/ -xzf go1.22.5.linux-amd64.tar.gz
+      - 
+        name: Install protobuf
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y protobuf-compiler golang-goprotobuf-dev
+      - 
+        name: Build local docker-image
+        run: |
+          cd ${GITHUB_REPOSITORY#*/}
+          chmod +x /tmp/build_result/Hanami
+          mkdir -p builds/binaries
+          cp /tmp/build_result/Hanami ./builds/binaries/
+          # IMPORTANT: use the Dockerfile instead of the Earthfile here, 
+          # because the Dockerfile is also used in the final build-process.
+          # So using the Dockerfile here to ensure, the final build after 
+          # merge doesn't fail.
+          docker build -t kitsudaiki/hanami:test .
+      - 
+        name: Download mnist-files
+        run: |
+          cd /tmp/
+          wget https://storage.googleapis.com/cvdf-datasets/mnist/train-images-idx3-ubyte.gz
+          wget https://storage.googleapis.com/cvdf-datasets/mnist/train-labels-idx1-ubyte.gz
+          wget https://storage.googleapis.com/cvdf-datasets/mnist/t10k-images-idx3-ubyte.gz
+          wget https://storage.googleapis.com/cvdf-datasets/mnist/t10k-labels-idx1-ubyte.gz
+          gzip -d train-images-idx3-ubyte.gz
+          gzip -d train-labels-idx1-ubyte.gz
+          gzip -d t10k-images-idx3-ubyte.gz
+          gzip -d t10k-labels-idx1-ubyte.gz
+      - 
+        name: Start Hanami-docker-container
+        run: |
+          cd ${GITHUB_REPOSITORY#*/}
+          sudo cp -r ./example_configs/hanami /etc/
+          docker run -d -v \
+            ./example_configs/hanami:/etc/hanami \
+            -p 127.0.0.1:11418:11418 \
+            --name=hanami \
+            --env HANAMI_ADMIN_USER_ID=asdf \
+            --env HANAMI_ADMIN_USER_NAME=asdf \
+            --env HANAMI_ADMIN_PASSWORD=asdfasdf \
+            kitsudaiki/hanami:test
+      - 
+        name: Sleep for 15 seconds
+        uses: jakejarvis/wait-action@master
+        with:
+          time: '15s'
+      - 
+        name: Run test
+        run: |
+          export PATH=$PATH:/usr/local/go/bin
+          cd ${GITHUB_REPOSITORY#*/}/testing/go_cli_api
+          ./cli_test.sh
+      - 
+        name: Get log of the docker-container, even if the test failed
+        if: always()
+        run: docker logs hanami
+
+
   docker_build:
     name: "Build Docker-images"
     if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/develop' || github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') }}
-    needs: [ sdk_api_tests ]
+    needs: [ sdk_api_tests, cli_api_tests ]
     runs-on: ubuntu-22.04
     strategy:
       fail-fast: false

.gitignore

@@ -8,6 +8,7 @@ gitIdentityChange.sh
 *.png~
 *.dtmp
 src/cli/hanamictl/hanamictl
+testing/go_cli_api/hanamictl
 testing/ansible_deploy/.vagrant/
 .git-hooks/post-checkout
 .git-hooks/post-commit
@@ -26,3 +27,6 @@ src/sdk/python/sdk_test_env
 *.proto3.pb.go
 src/sdk/python/hanami_sdk/build
 src/sdk/python/hanami_sdk/hanami_sdk.egg-info
+*.tar.gz
+*.tgz
+

src/Hanami/src/api/http/task/create_request_task_v1_0.cpp

@@ -78,6 +78,26 @@ CreateRequestTaskV1M0::CreateRequestTaskV1M0()
 
     registerOutputField("name", SAKURA_STRING_TYPE).setComment("Name of the new created task.");
 
+    registerOutputField("current_cycle", SAKURA_INT_TYPE)
+        .setComment("Current cycle of the current task.");
+
+    registerOutputField("total_number_of_cycles", SAKURA_INT_TYPE)
+        .setComment("Total number of cycles requred by the task.");
+
+    registerOutputField("state", SAKURA_STRING_TYPE)
+        .setComment("Actual state of the task (queued, active, aborted or finished).");
+
+    registerOutputField("queue_timestamp", SAKURA_STRING_TYPE)
+        .setComment(
+            "Timestamp in UTC when the task entered the queued state, "
+            "which is basicall the timestamp when the task was created");
+
+    registerOutputField("start_timestamp", SAKURA_STRING_TYPE)
+        .setComment("Timestamp in UTC when the task entered the active state.");
+
+    registerOutputField("end_timestamp", SAKURA_STRING_TYPE)
+        .setComment("Timestamp in UTC when the task was finished.");
+
     //----------------------------------------------------------------------------------------------
     //
     //----------------------------------------------------------------------------------------------
@@ -177,6 +197,12 @@ CreateRequestTaskV1M0::runTask(BlossomIO& blossomIO,
     // create output
     blossomIO.output["uuid"] = newTask->uuid.toString();
     blossomIO.output["name"] = name;
+    blossomIO.output["current_cycle"] = newTask->progress.currentCyle;
+    blossomIO.output["total_number_of_cycles"] = newTask->progress.totalNumberOfCycles;
+    blossomIO.output["queue_timestamp"] = serializeTimePoint(newTask->progress.queuedTimeStamp);
+    blossomIO.output["state"] = "queued";
+    blossomIO.output["start_timestamp"] = "-";
+    blossomIO.output["end_timestamp"] = "-";
 
     return true;
 }
@@ -261,11 +287,11 @@ CreateRequestTaskV1M0::createResultTarget(DataSetFileHandle& fileHandle,
     if (targetFilePath.at(targetFilePath.size() - 1) != '/') {
         targetFilePath.append("/");
     }
-    targetFilePath.append(name + "_result_" + datasetUuid);
+    targetFilePath.append(name + datasetUuid);
 
     // create new database-entry
     DataSetTable::DataSetDbEntry dbEntry;
-    dbEntry.name = name + "_result";
+    dbEntry.name = name;
     dbEntry.ownerId = userContext.userId;
     dbEntry.projectId = userContext.projectId;
     dbEntry.uuid = datasetUuid;

src/Hanami/src/api/http/task/create_train_task_v1_0.cpp

@@ -77,6 +77,26 @@ CreateTrainTaskV1M0::CreateTrainTaskV1M0()
 
     registerOutputField("name", SAKURA_STRING_TYPE).setComment("Name of the new created task.");
 
+    registerOutputField("current_cycle", SAKURA_INT_TYPE)
+        .setComment("Current cycle of the current task.");
+
+    registerOutputField("total_number_of_cycles", SAKURA_INT_TYPE)
+        .setComment("Total number of cycles requred by the task.");
+
+    registerOutputField("state", SAKURA_STRING_TYPE)
+        .setComment("Actual state of the task (queued, active, aborted or finished).");
+
+    registerOutputField("queue_timestamp", SAKURA_STRING_TYPE)
+        .setComment(
+            "Timestamp in UTC when the task entered the queued state, "
+            "which is basicall the timestamp when the task was created");
+
+    registerOutputField("start_timestamp", SAKURA_STRING_TYPE)
+        .setComment("Timestamp in UTC when the task entered the active state.");
+
+    registerOutputField("end_timestamp", SAKURA_STRING_TYPE)
+        .setComment("Timestamp in UTC when the task was finished.");
+
     //----------------------------------------------------------------------------------------------
     //
     //----------------------------------------------------------------------------------------------
@@ -176,6 +196,12 @@ CreateTrainTaskV1M0::runTask(BlossomIO& blossomIO,
     // create output
     blossomIO.output["uuid"] = newTask->uuid.toString();
     blossomIO.output["name"] = taskName;
+    blossomIO.output["current_cycle"] = newTask->progress.currentCyle;
+    blossomIO.output["total_number_of_cycles"] = newTask->progress.totalNumberOfCycles;
+    blossomIO.output["queue_timestamp"] = serializeTimePoint(newTask->progress.queuedTimeStamp);
+    blossomIO.output["state"] = "queued";
+    blossomIO.output["start_timestamp"] = "-";
+    blossomIO.output["end_timestamp"] = "-";
 
     return true;
 }

src/Hanami/src/api/http/task/show_task_v1_0.cpp

@@ -47,6 +47,10 @@ ShowTaskV1M0::ShowTaskV1M0() : Blossom("Show information of a specific task.")
     // output
     //----------------------------------------------------------------------------------------------
 
+    registerOutputField("uuid", SAKURA_STRING_TYPE).setComment("UUID of the task.");
+
+    registerOutputField("name", SAKURA_STRING_TYPE).setComment("Name of the task.");
+
     registerOutputField("current_cycle", SAKURA_INT_TYPE)
         .setComment("Current cycle of the current task.");
 
@@ -109,9 +113,12 @@ ShowTaskV1M0::runTask(BlossomIO& blossomIO,
         return false;
     }
 
-    const TaskProgress progress = cluster->getProgress(taskUuid);
+    const Task* task = cluster->getTask(taskUuid);
+    TaskProgress progress = task->progress;
 
     // get basic information
+    blossomIO.output["uuid"] = taskUuid;
+    blossomIO.output["name"] = task->name;
     blossomIO.output["current_cycle"] = progress.currentCyle;
     blossomIO.output["total_number_of_cycles"] = progress.totalNumberOfCycles;
     blossomIO.output["queue_timestamp"] = serializeTimePoint(progress.queuedTimeStamp);

src/Hanami/src/core/cluster/cluster.cpp

@@ -257,6 +257,26 @@ Cluster::getProgress(const std::string& taskUuid)
     return progress;
 }
 
+/**
+ * @brief get task-progress
+ *
+ * @param taskUuid UUID of the task
+ *
+ * @return task-progress
+ */
+const Task*
+Cluster::getTask(const std::string& taskUuid)
+{
+    std::lock_guard<std::mutex> guard(m_taskMutex);
+
+    const auto it = m_taskMap.find(taskUuid);
+    if (it != m_taskMap.end()) {
+        return &it->second;
+    }
+
+    return nullptr;
+}
+
 /**
  * @brief remove task from queue of abort the task, if actual in progress
  *

src/Hanami/src/core/cluster/cluster.h

@@ -72,6 +72,7 @@ class Cluster
     Task* addNewTask();
     Task* getCurrentTask();
     const TaskProgress getProgress(const std::string& taskUuid);
+    const Task* getTask(const std::string& taskUuid);
     bool removeTask(const std::string& taskUuid);
     bool isFinish(const std::string& taskUuid);
     void getAllProgress(std::map<std::string, TaskProgress>& result);

src/Hanami/src/core/cluster/states/cycle_finish_state.cpp

@@ -72,6 +72,11 @@ CycleFinish_State::processEvent()
 
     // to go next state of finish the task to goal is reached
     if (currentCycle == numberOfCycles) {
+        // in case of a request, write remainging results of the write-buffer into the final dataset
+        if (actualTask->type == REQUEST_TASK) {
+            RequestInfo* info = &std::get<RequestInfo>(actualTask->info);
+            info->results.clear();
+        }
         m_cluster->goToNextState(FINISH_TASK);
     }
     else {

src/Hanami/src/core/processing/logical_host.cpp

@@ -153,8 +153,10 @@ handleClientOutput(Cluster& cluster)
                 DataSetFileHandle* fileHandle = &info->results[name];
                 for (const OutputNeuron& outputNeuron : outputInterface.outputNeurons) {
                     // TODO: handle return value
+                    // std::cout<<outputNeuron.outputVal<<std::endl;
                     appendValueToDataSet(*fileHandle, outputNeuron.outputVal, error);
                 }
+                // std::cout << "-------------------------------------" << std::endl;
             }
         }
     }