Skip to content
Permalink
Browse files
Feat: Add code sample for video streaming action recognition. (#667)
  • Loading branch information
bingatgoogle committed Sep 14, 2021
1 parent d01f0ff commit 3cf1a4562999371e9e51adbcd5766937cb388d8c
@@ -0,0 +1,114 @@
/*
* Copyright 2019 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package beta.video;

// [START video_streaming_automl_action_recognition_beta]

import com.google.api.gax.rpc.BidiStream;
import com.google.cloud.videointelligence.v1p3beta1.LabelAnnotation;
import com.google.cloud.videointelligence.v1p3beta1.LabelFrame;
import com.google.cloud.videointelligence.v1p3beta1.StreamingAnnotateVideoRequest;
import com.google.cloud.videointelligence.v1p3beta1.StreamingAnnotateVideoResponse;
import com.google.cloud.videointelligence.v1p3beta1.StreamingAutomlActionRecognitionConfig;
import com.google.cloud.videointelligence.v1p3beta1.StreamingFeature;
import com.google.cloud.videointelligence.v1p3beta1.StreamingVideoAnnotationResults;
import com.google.cloud.videointelligence.v1p3beta1.StreamingVideoConfig;
import com.google.cloud.videointelligence.v1p3beta1.StreamingVideoIntelligenceServiceClient;
import com.google.protobuf.ByteString;
import io.grpc.StatusRuntimeException;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.concurrent.TimeoutException;

class StreamingAutoMlActionRecognition {

// Perform streaming video action recognition
static void streamingAutoMlActionRecognition(String filePath, String projectId, String modelId)
throws IOException, TimeoutException, StatusRuntimeException {

try (StreamingVideoIntelligenceServiceClient client =
StreamingVideoIntelligenceServiceClient.create()) {

Path path = Paths.get(filePath);
byte[] data = Files.readAllBytes(path);
// Set the chunk size to 5MB (recommended less than 10MB).
int chunkSize = 5 * 1024 * 1024;
int numChunks = (int) Math.ceil((double) data.length / chunkSize);

String modelPath =
String.format("projects/%s/locations/us-central1/models/%s", projectId, modelId);

System.out.println(modelPath);

StreamingAutomlActionRecognitionConfig streamingAutomlActionRecognitionConfig =
StreamingAutomlActionRecognitionConfig.newBuilder().setModelName(modelPath).build();

StreamingVideoConfig streamingVideoConfig =
StreamingVideoConfig.newBuilder()
.setFeature(StreamingFeature.STREAMING_AUTOML_ACTION_RECOGNITION)
.setAutomlActionRecognitionConfig(streamingAutomlActionRecognitionConfig)
.build();

BidiStream<StreamingAnnotateVideoRequest, StreamingAnnotateVideoResponse> call =
client.streamingAnnotateVideoCallable().call();

// The first request must **only** contain the video configuration:
call.send(
StreamingAnnotateVideoRequest.newBuilder().setVideoConfig(streamingVideoConfig).build());

// Subsequent requests must **only** contain the video data.
// Send the requests in chunks
for (int i = 0; i < numChunks; i++) {
call.send(
StreamingAnnotateVideoRequest.newBuilder()
.setInputContent(
ByteString.copyFrom(
Arrays.copyOfRange(data, i * chunkSize, i * chunkSize + chunkSize)))
.build());
}

// Tell the service you are done sending data
call.closeSend();

for (StreamingAnnotateVideoResponse response : call) {
if (response.hasError()) {
System.out.println(response.getError().getMessage());
break;
}

StreamingVideoAnnotationResults annotationResults = response.getAnnotationResults();

for (LabelAnnotation annotation : annotationResults.getLabelAnnotationsList()) {
String entity = annotation.getEntity().getDescription();

// There is only one frame per annotation
LabelFrame labelFrame = annotation.getFrames(0);
double offset =
labelFrame.getTimeOffset().getSeconds() + labelFrame.getTimeOffset().getNanos() / 1e9;
float confidence = labelFrame.getConfidence();

System.out.format("At %fs segment: %s (%f)\n", offset, entity, confidence);
}
}
System.out.println("Video streamed successfully.");
}
}
}
// [END video_streaming_automl_action_recognition_beta]
@@ -0,0 +1,80 @@
/*
* Copyright 2019 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package beta.video;

import static com.google.common.truth.Truth.assertThat;

import io.grpc.Status;
import io.grpc.StatusRuntimeException;
import java.io.ByteArrayOutputStream;
import java.io.PrintStream;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;

/** Integration (system) tests for {@link StreamingAutoMlActionRecognition}. */
@RunWith(JUnit4.class)
@SuppressWarnings("checkstyle:abbreviationaswordinname")
public class StreamingAutoMlActionRecognitionIT {

private static String PROJECT_ID = System.getenv().get("GOOGLE_CLOUD_PROJECT");
private static String MODEL_ID = "2787930479481847808";

private ByteArrayOutputStream bout;
private PrintStream out;
private PrintStream originalPrintStream;

@Before
public void setUp() {
bout = new ByteArrayOutputStream();
out = new PrintStream(bout);
originalPrintStream = System.out;
System.setOut(out);
}

@After
public void tearDown() {
// restores print statements in the original method
System.out.flush();
System.setOut(originalPrintStream);
}

@Test
public void testStreamingAutoMlActionRecognition() {
// Bad Gateway sporadically occurs
int tryCount = 0;
int maxTries = 3;
while (tryCount < maxTries) {
try {
StreamingAutoMlActionRecognition.streamingAutoMlActionRecognition(
"resources/cat.mp4", PROJECT_ID, MODEL_ID);
assertThat(bout.toString()).contains("Video streamed successfully.");

break;
} catch (StatusRuntimeException ex) {
if (ex.getStatus().getCode() == Status.Code.UNAVAILABLE) {
assertThat(ex.getMessage()).contains("Bad Gateway");
tryCount++;
}
} catch (Exception e) {
e.printStackTrace();
}
}
}
}

0 comments on commit 3cf1a45

Please sign in to comment.