Skip to content

Commit

Permalink
RSDK-7470: Add GetProperties and CaptureAllFromCamera (#312)
Browse files Browse the repository at this point in the history
  • Loading branch information
@ethanlook
ethanlook committed Jun 20, 2024
1 parent 63029c3 commit 9a286e8
Show file tree
Hide file tree
Showing 4 changed files with 247 additions and 106 deletions.
234 changes: 129 additions & 105 deletions src/services/vision/client.spec.ts
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
// @vitest-environment happy-dom

import { beforeEach, describe, expect, it, vi, type Mock } from 'vitest';
import { beforeEach, describe, expect, it, vi } from 'vitest';
import {
Classification as PBClassification,
Detection as PBDetection,
Expand All @@ -17,68 +17,121 @@ const visionClientName = 'test-vision';

let vision: VisionClient;

const classification: Classification = {
className: 'face',
confidence: 0.995_482_683_181_762_7,
};
const pbClassification = (() => {
const pb = new PBClassification();
pb.setClassName(classification.className);
pb.setConfidence(classification.confidence);
return pb;
})();

const detection: Detection = {
xMin: 251,
yMin: 225,
xMax: 416,
yMax: 451,
confidence: 0.995_482_683_181_762_7,
className: 'face',
};
const pbDetection = (() => {
const pb = new PBDetection();
pb.setClassName(detection.className);
pb.setConfidence(detection.confidence);
pb.setXMin(detection.xMin);
pb.setXMax(detection.xMax);
pb.setYMin(detection.yMin);
pb.setYMax(detection.yMax);
return pb;
})();

const pco: PointCloudObject = {
pointCloud: 'This is a PointCloudObject',
geometries: undefined,
};
const pbPCO = (() => {
const pb = new PBPointCloudObject();
pb.setPointCloud(pco.pointCloud);
return pb;
})();

describe('VisionClient Tests', () => {
beforeEach(() => {
RobotClient.prototype.createServiceClient = vi
.fn()
.mockImplementation(() => new VisionServiceClient(visionClientName));

VisionServiceClient.prototype.getDetections = vi
.fn()
.mockImplementation((_req, _md, cb) => {
cb(null, {
getDetectionsList: () => [pbDetection],
});
});
VisionServiceClient.prototype.getDetectionsFromCamera = vi
.fn()
.mockImplementation((_req, _md, cb) => {
cb(null, {
getDetectionsList: () => [pbDetection],
});
});
VisionServiceClient.prototype.getClassifications = vi
.fn()
.mockImplementation((_req, _md, cb) => {
cb(null, {
getClassificationsList: () => [pbClassification],
});
});
VisionServiceClient.prototype.getClassificationsFromCamera = vi
.fn()
.mockImplementation((_req, _md, cb) => {
cb(null, {
getClassificationsList: () => [pbClassification],
});
});
VisionServiceClient.prototype.getObjectPointClouds = vi
.fn()
.mockImplementation((_req, _md, cb) => {
cb(null, {
getObjectsList: () => [pbPCO],
});
});
VisionServiceClient.prototype.getProperties = vi
.fn()
.mockImplementation((_req, _md, cb) => {
cb(null, {
getClassificationsSupported: () => true,
getDetectionsSupported: () => true,
getObjectPointCloudsSupported: () => true,
});
});
VisionServiceClient.prototype.captureAllFromCamera = vi
.fn()
.mockImplementation((_req, _md, cb) => {
cb(null, {
getImage: () => undefined,
getClassificationsList: () => [pbClassification],
getDetectionsList: () => [pbDetection],
getObjectsList: () => [pbPCO],
});
});

vision = new VisionClient(new RobotClient('host'), visionClientName);
});

describe('Detection Tests', () => {
const testDetection: Detection = {
xMin: 251,
yMin: 225,
xMax: 416,
yMax: 451,
confidence: 0.995_482_683_181_762_7,
className: 'face',
};
let detection: Mock<[], PBDetection>;
const encodeDetection = (det: Detection) => {
const pbDetection = new PBDetection();
pbDetection.setClassName(det.className);
pbDetection.setConfidence(det.confidence);
pbDetection.setXMin(det.xMin);
pbDetection.setXMax(det.xMax);
pbDetection.setYMin(det.yMin);
pbDetection.setYMax(det.yMax);
return pbDetection;
};

beforeEach(() => {
VisionServiceClient.prototype.getDetections = vi
.fn()
.mockImplementation((_req, _md, cb) => {
cb(null, {
getDetectionsList: () => [detection()],
});
});

VisionServiceClient.prototype.getDetectionsFromCamera = vi
.fn()
.mockImplementation((_req, _md, cb) => {
cb(null, {
getDetectionsList: () => [detection()],
});
});
});

it('returns detections from a camera', async () => {
detection = vi.fn(() => encodeDetection(testDetection));

const expected = [testDetection];
const expected = [detection];

await expect(
vision.getDetectionsFromCamera('camera')
).resolves.toStrictEqual(expected);
});

it('returns detections from an image', async () => {
detection = vi.fn(() => encodeDetection(testDetection));

const expected = [testDetection];
const expected = [detection];

await expect(
vision.getDetections(new Uint8Array(), 1, 1, 'image/jpeg')
Expand All @@ -87,50 +140,16 @@ describe('VisionClient Tests', () => {
});

describe('Classification Tests', () => {
const testClassification: Classification = {
className: 'face',
confidence: 0.995_482_683_181_762_7,
};
let classification: Mock<[], PBClassification>;
const encodeClassification = (cls: Classification) => {
const pbClassification = new PBClassification();
pbClassification.setClassName(cls.className);
pbClassification.setConfidence(cls.confidence);
return pbClassification;
};

beforeEach(() => {
VisionServiceClient.prototype.getClassifications = vi
.fn()
.mockImplementation((_req, _md, cb) => {
cb(null, {
getClassificationsList: () => [classification()],
});
});

VisionServiceClient.prototype.getClassificationsFromCamera = vi
.fn()
.mockImplementation((_req, _md, cb) => {
cb(null, {
getClassificationsList: () => [classification()],
});
});
});

it('returns classifications from a camera', async () => {
classification = vi.fn(() => encodeClassification(testClassification));

const expected = [testClassification];
const expected = [classification];

await expect(
vision.getClassificationsFromCamera('camera', 1)
).resolves.toStrictEqual(expected);
});

it('returns classifications from an image', async () => {
classification = vi.fn(() => encodeClassification(testClassification));

const expected = [testClassification];
const expected = [classification];

await expect(
vision.getClassifications(new Uint8Array(), 1, 1, 'image/jpeg', 1)
Expand All @@ -139,35 +158,40 @@ describe('VisionClient Tests', () => {
});

describe('Object Point Cloud Tests', () => {
const testPCO: PointCloudObject = {
pointCloud: 'This is a PointCloudObject',
geometries: undefined,
};
let pointCloudObject: Mock<[], PBPointCloudObject>;
const encodePCO = (pco: PointCloudObject) => {
const pbPCO = new PBPointCloudObject();
pbPCO.setPointCloud(pco.pointCloud);
return pbPCO;
};

beforeEach(() => {
VisionServiceClient.prototype.getObjectPointClouds = vi
.fn()
.mockImplementation((_req, _md, cb) => {
cb(null, {
getObjectsList: () => [pointCloudObject()],
});
});
});

it('returns a PointCloudObject from a camera', async () => {
pointCloudObject = vi.fn(() => encodePCO(testPCO));

const expected = [testPCO];
const expected = [pco];

await expect(
vision.getObjectPointClouds('camera')
).resolves.toStrictEqual(expected);
});
});

describe('Properties', () => {
it('returns properties', async () => {
await expect(vision.getProperties()).resolves.toStrictEqual({
classificationsSupported: true,
detectionsSupported: true,
objectPointCloudsSupported: true,
});
});
});

describe('Capture All', () => {
it('returns captured values', async () => {
await expect(
vision.captureAllFromCamera('camera', {
returnImage: true,
returnClassifications: true,
returnDetections: true,
returnObjectPointClouds: true,
})
).resolves.toStrictEqual({
image: undefined,
classifications: [classification],
detections: [detection],
objectPointClouds: [pco],
});
});
});
});
61 changes: 61 additions & 0 deletions src/services/vision/client.ts
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,13 @@
import { Struct } from 'google-protobuf/google/protobuf/struct_pb';
import pb from '../../gen/service/vision/v1/vision_pb';
import { VisionServiceClient } from '../../gen/service/vision/v1/vision_pb_service';
import commonPB from '../../gen/common/v1/common_pb';
import type { MimeType } from '../../main';
import type { RobotClient } from '../../robot';
import type { Options, StructType } from '../../types';
import { doCommandFromClient, promisify } from '../../utils';
import type { Vision } from './vision';
import type { CaptureAllOptions } from './types';

/**
* A gRPC-web client for a Vision service.
Expand Down Expand Up @@ -142,6 +144,65 @@ export class VisionClient implements Vision {
return response.getObjectsList().map((x) => x.toObject());
}

async getProperties(extra = {}) {
const { service } = this;
const request = new pb.GetPropertiesRequest();
request.setName(this.name);
request.setExtra(Struct.fromJavaScript(extra));

this.options.requestLogger?.(request);

const response = await promisify<
pb.GetPropertiesRequest,
pb.GetPropertiesResponse
>(service.getProperties.bind(service), request);
return {
classificationsSupported: response.getClassificationsSupported(),
detectionsSupported: response.getDetectionsSupported(),
objectPointCloudsSupported: response.getObjectPointCloudsSupported(),
};
}

async captureAllFromCamera(
cameraName: string,
{
returnImage,
returnClassifications,
returnDetections,
returnObjectPointClouds,
}: CaptureAllOptions,
extra = {}
) {
const { service } = this;
const request = new pb.CaptureAllFromCameraRequest();
request.setName(this.name);
request.setCameraName(cameraName);
request.setReturnImage(returnImage);
request.setReturnClassifications(returnClassifications);
request.setReturnDetections(returnDetections);
request.setReturnObjectPointClouds(returnObjectPointClouds);
request.setExtra(Struct.fromJavaScript(extra));

this.options.requestLogger?.(request);

const response = await promisify<
pb.CaptureAllFromCameraRequest,
pb.CaptureAllFromCameraResponse
>(service.captureAllFromCamera.bind(service), request);
return {
image: response.getImage()?.toObject(),
classifications: response
.getClassificationsList()
.map((classification: pb.Classification) => classification.toObject()),
detections: response
.getDetectionsList()
.map((detection: pb.Detection) => detection.toObject()),
objectPointClouds: response
.getObjectsList()
.map((pbObject: commonPB.PointCloudObject) => pbObject.toObject()),
};
}

async doCommand(command: StructType): Promise<StructType> {
const { service } = this;
return doCommandFromClient(service, this.name, command, this.options);
Expand Down
Loading

0 comments on commit 9a286e8

Please sign in to comment.