diff --git a/cpp/OcvYoloDetection/triton_server/Dockerfile b/cpp/OcvYoloDetection/triton_server/Dockerfile index 1381ae3c..853f5815 100644 --- a/cpp/OcvYoloDetection/triton_server/Dockerfile +++ b/cpp/OcvYoloDetection/triton_server/Dockerfile @@ -60,7 +60,7 @@ RUN --mount=type=tmpfs,target=/tmp \ mv main /model-gen/yolo/gen-engine-608; \ mv libyolo608layerplugin.so /plugins/libyolo608layerplugin.so -COPY models/yolo-608.config.pbtxt /models/yolo-608/1/config.pbtxt +COPY models/yolo-608.config.pbtxt /models/yolo-608/config.pbtxt COPY docker-entrypoint.sh /opt/tritonserver diff --git a/python/ClipDetection/COPYING b/python/ClipDetection/COPYING new file mode 100644 index 00000000..19dc35b2 --- /dev/null +++ b/python/ClipDetection/COPYING @@ -0,0 +1,175 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. \ No newline at end of file diff --git a/python/ClipDetection/Dockerfile b/python/ClipDetection/Dockerfile new file mode 100644 index 00000000..5e7ad9cf --- /dev/null +++ b/python/ClipDetection/Dockerfile @@ -0,0 +1,65 @@ +# syntax=docker/dockerfile:experimental + +############################################################################# +# NOTICE # +# # +# This software (or technical data) was produced for the U.S. Government # +# under contract, and is subject to the Rights in Data-General Clause # +# 52.227-14, Alt. IV (DEC 2007). # +# # +# Copyright 2023 The MITRE Corporation. All Rights Reserved. # +############################################################################# + +############################################################################# +# Copyright 2023 The MITRE Corporation # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); # +# you may not use this file except in compliance with the License. # +# You may obtain a copy of the License at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, # +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +############################################################################# + +ARG MODELS_REGISTRY=openmpf/ +ARG BUILD_REGISTRY +ARG BUILD_TAG=latest +FROM ${MODELS_REGISTRY}openmpf_clip_detection_models:7.2.0 as models +FROM ${BUILD_REGISTRY}openmpf_python_executor_ssb:${BUILD_TAG} + +COPY --from=models /models/ViT-B-32.pt /models/ViT-B-32.pt + +RUN --mount=type=tmpfs,target=/var/cache/apt \ + --mount=type=tmpfs,target=/var/lib/apt/lists \ + --mount=type=tmpfs,target=/tmp \ + apt-get update; \ + DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y wget + +RUN pip3 install --upgrade pip + +RUN pip3 install ftfy regex tqdm + +RUN --mount=type=tmpfs,target=/tmp \ + mkdir /tmp/CLIP; \ + wget -O- 'https://github.com/openai/CLIP/tarball/master' \ + | tar --extract --gzip --directory /tmp/CLIP; \ + cd /tmp/CLIP/*; \ + pip3 install . 'torchvision==0.8.2' + +ARG RUN_TESTS=false + +RUN --mount=target=.,readwrite \ + install-component.sh; \ + if [ "${RUN_TESTS,,}" == true ]; then python tests/test_clip.py; fi + +LABEL org.label-schema.license="Apache 2.0" \ + org.label-schema.name="OpenMPF CLIP Detection" \ + org.label-schema.schema-version="1.0" \ + org.label-schema.url="https://openmpf.github.io" \ + org.label-schema.vcs-url="https://github.com/openmpf/openmpf-components" \ + org.label-schema.vendor="MITRE" \ No newline at end of file diff --git a/python/ClipDetection/LICENSE b/python/ClipDetection/LICENSE new file mode 100644 index 00000000..ce6902f8 --- /dev/null +++ b/python/ClipDetection/LICENSE @@ -0,0 +1,32 @@ +/****************************************************************************** +* Copyright 2023 The MITRE Corporation * +* * +* Licensed under the Apache License, Version 2.0 (the "License"); * +* you may not use this file except in compliance with the License. * +* You may obtain a copy of the License at * +* * +* http://www.apache.org/licenses/LICENSE-2.0 * +* * +* Unless required by applicable law or agreed to in writing, software * +* distributed under the License is distributed on an "AS IS" BASIS, * +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * +* See the License for the specific language governing permissions and * +* limitations under the License. * +******************************************************************************/ + +This project contains content developed by The MITRE Corporation. If this code +is used in a deployment or embedded within another project, it is requested +that you send an email to opensource@mitre.org in order to let us know where +this software is being used. + + + +This software makes use of a data model derived from third party software: + +-------------------------------------------------------------------------- + +The TensorFlow implementation of the Contrastive Language-Image +Pre-Training (CLIP) model used by this component was developed by OpenAI: +http://www.github.com/openai/CLIP + +The OpenAI CLIP model is licensed under the MIT License. \ No newline at end of file diff --git a/python/ClipDetection/NOTICE b/python/ClipDetection/NOTICE new file mode 100644 index 00000000..3b180c00 --- /dev/null +++ b/python/ClipDetection/NOTICE @@ -0,0 +1,7 @@ +# NOTICE + +This software (or technical data) was produced for the U.S. Government +under contract, and is subject to the Rights in Data-General Clause +552.227-14, Alt. IV (DEC 2007). + +Copyright 2023 The MITRE Corporation. All Rights Reserved. \ No newline at end of file diff --git a/python/ClipDetection/README.md b/python/ClipDetection/README.md new file mode 100644 index 00000000..75d40b37 --- /dev/null +++ b/python/ClipDetection/README.md @@ -0,0 +1,64 @@ +# Overview + +This repository contains source code for the OpenMPF CLIP detection component. CLIP (Contrastive Language-Image Pre-Training) was developed by OpenAI and published in 2021. https://arxiv.org/abs/2103.00020 + +# Job Properties + +The following are the properties that can be specified for the component. Each property has a default value and so none of them necessarily need to be specified for processing jobs. + +- `NUMBER_OF_CLASSIFICATIONS`: Specifies how many of the top classifications you want to return. The default value is set to 1, and so you'll only see the classification with the greatest confidence. + +- `CLASSIFICATION_PATH`: If specified, this allows the user to give the component a file path to their own list of classifications in a CSV file, if the COCO or ImageNet class lists aren't of interest. See below for the formatting that's required for that file. + +- `CLASSIFICATION_LIST`: Specifies whether the user wants to use the COCO or ImageNet classification list, by specifying 'coco' or 'imagenet', respectively. By default, this is set to 'coco'. Also this property is overridden if a `CLASSIFICATION_PATH` is given. + +- `TEMPLATE_PATH`: If specified, this allows the user to give the component a file path to their own list of templates. See below for the formatting that's required for that file. The OpenAI developers admitted that the process of developing templates was a lot of trial and error, so feel free to come up with your own! + +- `NUMBER_OF_TEMPLATES`: There are three template files that are included in the component, with the number of templates in each being 1, 7, and 80. The one template is a basic template, while the 7 and 80 come from the OpenAI team when trying to [improve performance](https://github.com/openai/CLIP/blob/main/notebooks/Prompt_Engineering_for_ImageNet.ipynb) on the ImageNet dataset. The default value is 80, while 1 and 7 are the only other valid inputs. Also this property is overridden if a `TEMPLATE_PATH` is specified. + +- `ENABLE_CROPPING`: A boolean toggle to specify if the image is to be cropped into 144 images of size 224x224 which cover all areas of the original. By default, this is set to true. This technique is described Section 7 of the paper "[Going deeper with convolutions](https://arxiv.org/abs/1409.4842)" from Szegedy, et al. + +- `ENABLE_TRITON`: A boolean toggle to specify whether the component should use a Triton inference server to process the image job. By default this is set to false. + +- `INCLUDE_FEATURES`: A boolean toggle to specify whether the `FEATURE` detection property is included with each detection. By default, this is set to false. + +- `TRITON_SERVER`: Specifies the Triton server `:` to use for inferencing. By default, this is set to 'clip-detection-server:8001'. + +## Detection Properties + +Returned `ImageLocation` objects have the following members in their `detection_properties`: + +| Property Key | Description +|----------------------------------|---------------------------------------------------------------------------------------------------------------------------------------- +| `CLASSIFICATION` | The classification returned from the CLIP model with the highest similarity. +| `CLASSIFICATION CONFIDENCE LIST` | A list of the highest confidences returned by the model. The number of confidences is specified by `NUMBER_OF_CLASSIFICATIONS`. +| `CLASSIFICATION LIST` | A list of the classes with highest confidence returned by the model. The number of classes is specified by `NUMBER_OF_CLASSIFICATIONS`. +| `FEATURE` | When `INCLUDE_FEATURES` is true, this detection property is set to the value of the base64-encoded version of the image feature vector. + +# Custom Templates + +When tuning the CLIP model, it is important to have appropriate templates for what you're trying to classify. In order to write the file, put one template on each line. Use a pair of brackets, {}, where the potential classifications need to be placed. See below for example templates. +``` +A photograph of a {}. +A {} in an open field. +``` + +# Custom Classifications + +The need for custom classifications arose when training on the ImageNet classifications, where any different class can have many equivalent names. For example, one of the classes is "great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias". We found the model to be most performant when given a single representative class title. For this case, 'great white shark' makes the most sense. The `imagenet_classification_list.csv` file gives representative titles for each class, adapted from .ipynb files on the CLIP GitHub page. + +As for the format of the CSV file, it has two columns. The first being the representative name, and the second being the full name of the class. The representative name is what goes inside the brackets, {}, of the templates, and the full name is what will be used when displaying results. Below are a couple of examples of rows from the ImageNet classifications. Note that in the first example, quotes are put around the full classification name so they're easier to read and so that those commas aren't confused for the separator. + +``` +tench,"tench, Tinca tinca" +kite (bird of prey),kite +magpie,magpie +``` + +# Known Issues + +When building on a host without a GPU, the following warning is expected: +``` +UserWarning: CUDA initialization: Found no NVIDIA driver on your system. Please check that you have an NVIDIA GPU and installed a driver from http://www.nvidia.com/Download/index.aspx (Triggered internally at /pytorch/c10/cuda/CUDAFunctions.cpp:100.) + return torch._C._cuda_getDeviceCount() > 0 +``` \ No newline at end of file diff --git a/python/ClipDetection/clip_component/__init__.py b/python/ClipDetection/clip_component/__init__.py new file mode 100644 index 00000000..635f36e5 --- /dev/null +++ b/python/ClipDetection/clip_component/__init__.py @@ -0,0 +1,27 @@ +############################################################################# +# NOTICE # +# # +# This software (or technical data) was produced for the U.S. Government # +# under contract, and is subject to the Rights in Data-General Clause # +# 52.227-14, Alt. IV (DEC 2007). # +# # +# Copyright 2023 The MITRE Corporation. All Rights Reserved. # +############################################################################# + +############################################################################# +# Copyright 2023 The MITRE Corporation # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); # +# you may not use this file except in compliance with the License. # +# You may obtain a copy of the License at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, # +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +############################################################################# + +from .clip_component import ClipComponent \ No newline at end of file diff --git a/python/ClipDetection/clip_component/clip_component.py b/python/ClipDetection/clip_component/clip_component.py new file mode 100644 index 00000000..9a6df34f --- /dev/null +++ b/python/ClipDetection/clip_component/clip_component.py @@ -0,0 +1,404 @@ +############################################################################# +# NOTICE # +# # +# This software (or technical data) was produced for the U.S. Government # +# under contract, and is subject to the Rights in Data-General Clause # +# 52.227-14, Alt. IV (DEC 2007). # +# # +# Copyright 2023 The MITRE Corporation. All Rights Reserved. # +############################################################################# + +############################################################################# +# Copyright 2023 The MITRE Corporation # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); # +# you may not use this file except in compliance with the License. # +# You may obtain a copy of the License at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, # +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +############################################################################# + +import logging +import os +import csv +from pkg_resources import resource_filename + +from PIL import Image +import cv2 +import base64 +import numpy as np +import torch +import torchvision.transforms as T +import torchvision.transforms.functional as TF +import clip + +import tritonclient.grpc as grpcclient +from tritonclient.utils import InferenceServerException, triton_to_np_dtype + +import mpf_component_api as mpf +import mpf_component_util as mpf_util + +logger = logging.getLogger('ClipComponent') +device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu') + +class ClipComponent(mpf_util.ImageReaderMixin): + detection_type = 'CLASS' + + def __init__(self): + self._wrapper = ClipWrapper() + + def get_detections_from_image_reader(self, image_job, image_reader): + try: + logger.info("received image job: %s", image_job) + image = image_reader.get_image() + detections = self._wrapper.get_classifications(image, image_job.job_properties) + logger.info(f"Job complete. Found {len(detections)} detections.") + return detections + + except Exception: + logger.exception(f"Failed to complete job {image_job.job_name} due to the following exception:") + raise + +class ClipWrapper(object): + def __init__(self): + logger.info("Loading model...") + model, _ = clip.load('ViT-B/32', device=device, download_root='/models') + logger.info("Model loaded.") + self._model = model + + self._classification_path = '' + self._template_path = '' + self._classification_list = '' + + self._templates = None + self._class_mapping = None + self._text_features = None + + self._inferencing_server = None + self._triton_server_url = None + + def get_classifications(self, image, job_properties): + image = Image.fromarray(cv2.cvtColor(image, cv2.COLOR_BGR2RGB)) + kwargs = self._parse_properties(job_properties) + image_width, image_height = image.size + + self._check_template_list(kwargs['template_path'], kwargs['num_templates']) + self._check_class_list(kwargs['classification_path'], kwargs['classification_list']) + + image = ImagePreprocessor(kwargs['enable_cropping']).preprocess(image).to(device) + + if kwargs['enable_triton']: + if self._inferencing_server is None or kwargs['triton_server'] != self._triton_server_url: + self._inferencing_server = CLIPInferencingServer(kwargs['triton_server']) + self._triton_server_url = kwargs['triton_server'] + + results = self._inferencing_server.get_responses(image) + image_tensors= torch.Tensor(np.copy(results)).to(device=device) + image_features = torch.mean(image_tensors, 0) + else: + with torch.no_grad(): + image_features = self._model.encode_image(image).float() + image_features = torch.mean(image_features, 0).unsqueeze(0) + + with torch.no_grad(): + image_features /= image_features.norm(dim=-1, keepdim=True) + + similarity = (100.0 * image_features @ self._text_features).softmax(dim=-1).to(device) + similarity = torch.mean(similarity, 0) + values, indices = similarity.topk(kwargs['num_classifications']) + + classification_list = '; '.join([self._class_mapping[list(self._class_mapping.keys())[int(index)]] for index in indices]) + classification_confidence_list = '; '.join([str(value.item()) for value in values]) + + detection_properties = { + "CLASSIFICATION": classification_list.split('; ')[0], + "CLASSIFICATION CONFIDENCE LIST": classification_confidence_list, + "CLASSIFICATION LIST": classification_list + } + + if kwargs['include_features']: + detection_properties['FEATURE'] = base64.b64encode(image_features.cpu().numpy()).decode() + + return [ + mpf.ImageLocation( + x_left_upper = 0, + y_left_upper = 0, + width = image_width, + height = image_height, + confidence = float(classification_confidence_list.split('; ')[0]), + detection_properties = detection_properties + ) + ] + + def _parse_properties(self, job_properties): + classification_list = self._get_prop(job_properties, "CLASSIFICATION_LIST", 'coco', ['coco', 'imagenet']) + classification_path = self._get_prop(job_properties, "CLASSIFICATION_PATH", '') + enable_cropping = self._get_prop(job_properties, "ENABLE_CROPPING", True) + enable_triton = self._get_prop(job_properties, "ENABLE_TRITON", False) + include_features = self._get_prop(job_properties, "INCLUDE_FEATURES", False) + num_classifications = self._get_prop(job_properties, "NUMBER_OF_CLASSIFICATIONS", 1) + num_templates = self._get_prop(job_properties, "NUMBER_OF_TEMPLATES", 80, [1, 7, 80]) + template_path = self._get_prop(job_properties, "TEMPLATE_PATH", '') + triton_server = self._get_prop(job_properties, "TRITON_SERVER", 'clip-detection-server:8001') + + return dict( + classification_list = classification_list, + classification_path = classification_path, + enable_cropping = enable_cropping, + enable_triton = enable_triton, + include_features = include_features, + num_classifications = num_classifications, + num_templates = num_templates, + template_path = template_path, + triton_server = triton_server + ) + + @staticmethod + def _get_prop(job_properties, key, default_value, accep_values=[]): + prop = mpf_util.get_property(job_properties, key, default_value) + if (accep_values != []) and (prop not in accep_values): + raise mpf.DetectionException( + f"Property {key} not in list of acceptible values: {accep_values}", + mpf.DetectionError.INVALID_PROPERTY + ) + return prop + + def _check_template_list(self, template_path, number_of_templates): + if template_path != '': + if (not os.path.exists(template_path)): + raise mpf.DetectionException( + f"The path {template_path} is not valid", + mpf.DetectionError.COULD_NOT_OPEN_DATAFILE + ) + elif self._template_path != template_path: + self._template_path = template_path + + try: + logger.info("Updating templates...") + self._templates = self._get_templates_from_file(template_path) + logger.info("Templates updated.") + except: + raise mpf.DetectionException( + f"Could not read templates from {template_path}", + mpf.DetectionError.COULD_NOT_READ_DATAFILE + ) + + elif self._templates == None or number_of_templates != len(self._templates): + if number_of_templates == 80: + template_filename = 'eighty_templates.txt' + elif number_of_templates == 7: + template_filename = 'seven_templates.txt' + elif number_of_templates == 1: + template_filename = 'one_template.txt' + + template_path = os.path.realpath(resource_filename(__name__, 'data/' + template_filename)) + logger.info("Updating templates...") + self._templates = self._get_templates_from_file(template_path) + logger.info("Templates updated.") + + def _check_class_list(self, classification_path, classification_list): + if classification_path != "": + if (not os.path.exists(classification_path)): + raise mpf.DetectionException( + f"The path {classification_path} is not valid", + mpf.DetectionError.COULD_NOT_OPEN_DATAFILE + ) + else: + if self._classification_list != classification_list.lower(): + self._classification_list = classification_list.lower() + classification_path = os.path.realpath(resource_filename(__name__, f'data/{self._classification_list}_classification_list.csv')) + + if self._classification_path != classification_path: + self._classification_path = classification_path + + try: + logger.info("Updating classifications...") + self._class_mapping = self._get_mapping_from_classifications(classification_path) + logger.info("Classifications updated.") + except Exception: + raise mpf.DetectionException( + f"Could not read classifications from {classification_path}", + mpf.DetectionError.COULD_NOT_READ_DATAFILE + ) + + with torch.no_grad(): + logger.info("Creating text embeddings...") + text_features = [] + for label in self._class_mapping.keys(): + text_phrases = [template.format(label) for template in self._templates] + text_tokens = clip.tokenize(text_phrases).to(device) + text_embeddings = self._model.encode_text(text_tokens) + text_embeddings /= text_embeddings.norm(dim=-1, keepdim=True) + text_embedding = text_embeddings.mean(dim=0) + text_embedding /= text_embedding.norm() + text_features.append(text_embedding) + self._text_features = torch.stack(text_features, dim=1).float().to(device) + logger.info("Text embeddings created.") + + @staticmethod + def _get_mapping_from_classifications(classification_path): + with open(classification_path) as csvfile: + mapping = {} + csvreader = csv.reader(csvfile) + for row in csvreader: + mapping[row[0].strip()] = row[1].strip() + + return mapping + + @staticmethod + def _get_templates_from_file(template_path): + with open(template_path) as f: + return [line.strip() for line in f.readlines()] + +class CLIPInferencingServer(object): + ''' + Class that handles Triton inferencing if enabled. + ''' + def __init__(self, triton_server): + self._model_name = 'ip_clip_512' + self._input_name = None + self._output_name = None + self._dtype = None + + try: + self._triton_client = grpcclient.InferenceServerClient(url=triton_server, verbose=False) + except InferenceServerException as e: + logger.exception("Client creation failed.") + raise + + # Check if triton server is alive and ready + self._check_triton_server() + + try: + model_metadata = self._triton_client.get_model_metadata(model_name=self._model_name) + except InferenceServerException as e: + logger.exception("Failed to retrieve model metadata.") + raise + + self._parse_model(model_metadata) + + def _parse_model(self, model_metadata): + input_metadata = model_metadata.inputs[0] + output_metadata = model_metadata.outputs[0] + + self._input_name = input_metadata.name + self._output_name = output_metadata.name + self._dtype = input_metadata.datatype + + def _get_inputs_outputs(self, images): + inputs = [grpcclient.InferInput(self._input_name, images.shape, self._dtype)] + inputs[0].set_data_from_numpy(images) + + outputs = [grpcclient.InferRequestedOutput(self._output_name, class_count=0)] + + yield inputs, outputs + + def get_responses(self, images): + images = np.array(images.cpu()) + images = images.astype(triton_to_np_dtype(self._dtype)) + + responses = [] + try: + for inputs, outputs in self._get_inputs_outputs(images): + responses.append(self._triton_client.infer(model_name=self._model_name, inputs=inputs, outputs=outputs)) + except Exception: + raise mpf.DetectionException( + f"Inference failed.", + mpf.DetectionError.NETWORK_ERROR + ) + + results = [] + for response in responses: + result = response.as_numpy(self._output_name) + results.append(result) + return results + + def _check_triton_server(self): + if not self._triton_client.is_server_live(): + raise mpf.DetectionException( + "Server is not live.", + mpf.DetectionError.NETWORK_ERROR + ) + + if not self._triton_client.is_server_ready(): + raise mpf.DetectionException( + "Server is not ready.", + mpf.DetectionError.NETWORK_ERROR + ) + + if not self._triton_client.is_model_ready(self._model_name): + raise mpf.DetectionException( + f"Model {self._model_name} is not ready.", + mpf.DetectionError.NETWORK_ERROR + ) + +class ImagePreprocessor(object): + ''' + Class that handles the preprocessing of images before being sent through the CLIP model. + Values from T.Normalize() taken from OpenAI's code for CLIP, https://github.com/openai/CLIP/blob/main/clip/clip.py#L85 + ''' + def __init__(self, enable_cropping): + if enable_cropping: + self.preprocess = self.crop + else: + self.preprocess = self.resize_pad + + def crop(self, image): + return T.Compose([ + self._resize_images, + self._get_crops, + T.Lambda(lambda crops: torch.stack([T.ToTensor()(crop) for crop in crops])), + T.Normalize((0.48145466, 0.4578275, 0.40821073), (0.26862954, 0.26130258, 0.27577711)) + ])(image) + + def resize_pad(self, image): + width, height = image.width, image.height + resize_ratio = 224 / max(width, height) + new_w, new_h = (int(width * resize_ratio), int(height * resize_ratio)) + + if new_w < new_h: + left = (224 - new_w) // 2 + right = (225 - new_w) // 2 + padding = (left, 0, right, 0) + else: + top = (224 - new_h) // 2 + bottom = (225 - new_h) // 2 + padding = (0, top, 0, bottom) + + new_img = T.Compose([ + T.Resize(size=(new_h, new_w)), + T.Pad(padding=padding, padding_mode='edge'), + T.ToTensor(), + T.Normalize((0.48145466, 0.4578275, 0.40821073), (0.26862954, 0.26130258, 0.27577711)) + ])(image) + + return new_img.unsqueeze(0) + + @staticmethod + def _resize_images(img): + resized_images = [] + for size in [256, 288, 320, 352]: + resized_image = TF.resize(img, size) + width, height = resized_image.size + resized_images.append(TF.center_crop(resized_image, size)) + resized_images.append(TF.crop(resized_image, 0, 0, size, size)) + resized_images.append(TF.crop(resized_image, height-size, width-size, size, size)) + return resized_images + + @staticmethod + def _get_crops(imgs): + crops = () + for img in imgs: + five_crops = TF.five_crop(img, 224) + resized = TF.resize(img, 224) + crops += five_crops + (resized, TF.hflip(resized)) + tuple([TF.hflip(fcrop) for fcrop in five_crops]) + return crops + + +EXPORT_MPF_COMPONENT = ClipComponent \ No newline at end of file diff --git a/python/ClipDetection/clip_component/data/coco_classification_list.csv b/python/ClipDetection/clip_component/data/coco_classification_list.csv new file mode 100644 index 00000000..e67ec89f --- /dev/null +++ b/python/ClipDetection/clip_component/data/coco_classification_list.csv @@ -0,0 +1,80 @@ +person,person +bicycle,bicycle +car,car +motorcycle,motorcycle +airplane,airplane +bus,bus +train,train +truck,truck +boat,boat +traffic light,traffic light +fire hydrant,fire hydrant +stop sign,stop sign +parking meter,parking meter +bench,bench +bird,bird +cat,cat +dog,dog +horse,horse +sheep,sheep +cow,cow +elephant,elephant +bear,bear +zebra,zebra +giraffe,giraffe +backpack,backpack +umbrella,umbrella +handbag,handbag +tie,tie +suitcase,suitcase +frisbee,frisbee +skis,skis +snowboard,snowboard +sports ball,sports ball +kite,kite +baseball bat,baseball bat +baseball glove,baseball glove +skateboard,skateboard +surfboard,surfboard +tennis racket,tennis racket +bottle,bottle +wine glass,wine glass +cup,cup +fork,fork +knife,knife +spoon,spoon +bowl,bowl +banana,banana +apple,apple +sandwich,sandwich +orange,orange +broccoli,broccoli +carrot,carrot +hot dog,hot dog +pizza,pizza +donut,donut +cake,cake +chair,chair +couch,couch +potted plant,potted plant +bed,bed +dining table,dining table +toilet,toilet +tv,tv +laptop,laptop +mouse,mouse +remote,remote +keyboard,keyboard +cell phone,cell phone +microwave,microwave +oven,oven +toaster,toaster +sink,sink +refrigerator,refrigerator +book,book +clock,clock +vase,vase +scissors,scissors +teddy bear,teddy bear +hair drier,hair drier +toothbrush,toothbrush diff --git a/python/ClipDetection/clip_component/data/eighty_templates.txt b/python/ClipDetection/clip_component/data/eighty_templates.txt new file mode 100644 index 00000000..526e39c0 --- /dev/null +++ b/python/ClipDetection/clip_component/data/eighty_templates.txt @@ -0,0 +1,80 @@ +a bad photo of a {}. +a photo of many {}. +a sculpture of a {}. +a photo of the hard to see {}. +a low resolution photo of the {}. +a rendering of a {}. +graffiti of a {}. +a bad photo of the {}. +a cropped photo of the {}. +a tattoo of a {}. +the embroidered {}. +a photo of a hard to see {}. +a bright photo of a {}. +a photo of a clean {}. +a photo of a dirty {}. +a dark photo of the {}. +a drawing of a {}. +a photo of my {}. +the plastic {}. +a photo of the cool {}. +a close-up photo of a {}. +a black and white photo of the {}. +a painting of the {}. +a painting of a {}. +a pixelated photo of the {}. +a sculpture of the {}. +a bright photo of the {}. +a cropped photo of a {}. +a plastic {}. +a photo of the dirty {}. +a jpeg corrupted photo of a {}. +a blurry photo of the {}. +a photo of the {}. +a good photo of the {}. +a rendering of the {}. +a {} in a video game. +a photo of one {}. +a doodle of a {}. +a close-up photo of the {}. +a photo of a {}. +the origami {}. +the {} in a video game. +a sketch of a {}. +a doodle of the {}. +a origami {}. +a low resolution photo of a {}. +the toy {}. +a rendition of the {}. +a photo of the clean {}. +a photo of a large {}. +a rendition of a {}. +a photo of a nice {}. +a photo of a weird {}. +a blurry photo of a {}. +a cartoon {}. +art of a {}. +a sketch of the {}. +a embroidered {}. +a pixelated photo of a {}. +itap of the {}. +a jpeg corrupted photo of the {}. +a good photo of a {}. +a plushie {}. +a photo of the nice {}. +a photo of the small {}. +a photo of the weird {}. +the cartoon {}. +art of the {}. +a drawing of the {}. +a photo of the large {}. +a black and white photo of a {}. +the plushie {}. +a dark photo of a {}. +itap of a {}. +graffiti of the {}. +a toy {}. +itap of my {}. +a photo of a cool {}. +a photo of a small {}. +a tattoo of the {}. \ No newline at end of file diff --git a/python/ClipDetection/clip_component/data/imagenet_classification_list.csv b/python/ClipDetection/clip_component/data/imagenet_classification_list.csv new file mode 100644 index 00000000..78c8537b --- /dev/null +++ b/python/ClipDetection/clip_component/data/imagenet_classification_list.csv @@ -0,0 +1,1000 @@ +tench,"tench, Tinca tinca" +goldfish,"goldfish, Carassius auratus" +great white shark,"great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias" +tiger shark,"tiger shark, Galeocerdo cuvieri" +hammerhead shark,"hammerhead, hammerhead shark" +electric ray,"electric ray, crampfish, numbfish, torpedo" +stingray,stingray +rooster,cock +hen,hen +ostrich,"ostrich, Struthio camelus" +brambling,"brambling, Fringilla montifringilla" +goldfinch,"goldfinch, Carduelis carduelis" +house finch,"house finch, linnet, Carpodacus mexicanus" +junco,"junco, snowbird" +indigo bunting,"indigo bunting, indigo finch, indigo bird, Passerina cyanea" +American robin,"robin, American robin, Turdus migratorius" +bulbul,bulbul +jay,jay +magpie,magpie +chickadee,chickadee +American dipper,"water ouzel, dipper" +kite (bird of prey),kite +bald eagle,"bald eagle, American eagle, Haliaeetus leucocephalus" +vulture,vulture +great grey owl,"great grey owl, great gray owl, Strix nebulosa" +fire salamander,"European fire salamander, Salamandra salamandra" +smooth newt,"common newt, Triturus vulgaris" +newt,eft +spotted salamander,"spotted salamander, Ambystoma maculatum" +axolotl,"axolotl, mud puppy, Ambystoma mexicanum" +American bullfrog,"bullfrog, Rana catesbeiana" +tree frog,"tree frog, tree-frog" +tailed frog,"tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui" +loggerhead sea turtle,"loggerhead, loggerhead turtle, Caretta caretta" +leatherback sea turtle,"leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea" +mud turtle,mud turtle +terrapin,terrapin +box turtle,"box turtle, box tortoise" +banded gecko,banded gecko +green iguana,"common iguana, iguana, Iguana iguana" +Carolina anole,"American chameleon, anole, Anolis carolinensis" +desert grassland whiptail lizard,"whiptail, whiptail lizard" +agama,agama +frilled-necked lizard,"frilled lizard, Chlamydosaurus kingi" +alligator lizard,alligator lizard +Gila monster,"Gila monster, Heloderma suspectum" +European green lizard,"green lizard, Lacerta viridis" +chameleon,"African chameleon, Chamaeleo chamaeleon" +Komodo dragon,"Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis" +Nile crocodile,"African crocodile, Nile crocodile, Crocodylus niloticus" +American alligator,"American alligator, Alligator mississipiensis" +triceratops,triceratops +worm snake,"thunder snake, worm snake, Carphophis amoenus" +ring-necked snake,"ringneck snake, ring-necked snake, ring snake" +eastern hog-nosed snake,"hognose snake, puff adder, sand viper" +smooth green snake,"green snake, grass snake" +kingsnake,"king snake, kingsnake" +garter snake,"garter snake, grass snake" +water snake,water snake +vine snake,vine snake +night snake,"night snake, Hypsiglena torquata" +boa constrictor,"boa constrictor, Constrictor constrictor" +African rock python,"rock python, rock snake, Python sebae" +Indian cobra,"Indian cobra, Naja naja" +green mamba,green mamba +sea snake,sea snake +Saharan horned viper,"horned viper, cerastes, sand viper, horned asp, Cerastes cornutus" +eastern diamondback rattlesnake,"diamondback, diamondback rattlesnake, Crotalus adamanteus" +sidewinder rattlesnake,"sidewinder, horned rattlesnake, Crotalus cerastes" +trilobite,trilobite +harvestman,"harvestman, daddy longlegs, Phalangium opilio" +scorpion,scorpion +yellow garden spider,"black and gold garden spider, Argiope aurantia" +barn spider,"barn spider, Araneus cavaticus" +European garden spider,"garden spider, Aranea diademata" +southern black widow,"black widow, Latrodectus mactans" +tarantula,tarantula +wolf spider,"wolf spider, hunting spider" +tick,tick +centipede,centipede +black grouse,black grouse +ptarmigan,ptarmigan +ruffed grouse,"ruffed grouse, partridge, Bonasa umbellus" +prairie grouse,"prairie chicken, prairie grouse, prairie fowl" +peafowl,peacock +quail,quail +partridge,partridge +african grey parrot,"African grey, African gray, Psittacus erithacus" +macaw,macaw +sulphur-crested cockatoo,"sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita" +lorikeet,lorikeet +coucal,coucal +bee eater,bee eater +hornbill,hornbill +hummingbird,hummingbird +jacamar,jacamar +toucan,toucan +duck,drake +red-breasted merganser,"red-breasted merganser, Mergus serrator" +goose,goose +black swan,"black swan, Cygnus atratus" +tusker,tusker +echidna,"echidna, spiny anteater, anteater" +platypus,"platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus" +wallaby,"wallaby, brush kangaroo" +koala,"koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus" +wombat,wombat +jellyfish,jellyfish +sea anemone,"sea anemone, anemone" +brain coral,brain coral +flatworm,"flatworm, platyhelminth" +nematode,"nematode, nematode worm, roundworm" +conch,conch +snail,snail +slug,slug +sea slug,"sea slug, nudibranch" +chiton,"chiton, coat-of-mail shell, sea cradle, polyplacophore" +chambered nautilus,"chambered nautilus, pearly nautilus, nautilus" +Dungeness crab,"Dungeness crab, Cancer magister" +rock crab,"rock crab, Cancer irroratus" +fiddler crab,fiddler crab +red king crab,"king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica" +American lobster,"American lobster, Northern lobster, Maine lobster, Homarus americanus" +spiny lobster,"spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish" +crayfish,"crayfish, crawfish, crawdad, crawdaddy" +hermit crab,hermit crab +isopod,isopod +white stork,"white stork, Ciconia ciconia" +black stork,"black stork, Ciconia nigra" +spoonbill,spoonbill +flamingo,flamingo +little blue heron,"little blue heron, Egretta caerulea" +great egret,"American egret, great white heron, Egretta albus" +bittern bird,bittern +crane bird,crane +limpkin,"limpkin, Aramus pictus" +common gallinule,"European gallinule, Porphyrio porphyrio" +American coot,"American coot, marsh hen, mud hen, water hen, Fulica americana" +bustard,bustard +ruddy turnstone,"ruddy turnstone, Arenaria interpres" +dunlin,"red-backed sandpiper, dunlin, Erolia alpina" +common redshank,"redshank, Tringa totanus" +dowitcher,dowitcher +oystercatcher,"oystercatcher, oyster catcher" +pelican,pelican +king penguin,"king penguin, Aptenodytes patagonica" +albatross,"albatross, mollymawk" +grey whale,"grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus" +killer whale,"killer whale, killer, orca, grampus, sea wolf, Orcinus orca" +dugong,"dugong, Dugong dugon" +sea lion,sea lion +Chihuahua,Chihuahua +Japanese Chin,Japanese spaniel +Maltese,"Maltese dog, Maltese terrier, Maltese" +Pekingese,"Pekinese, Pekingese, Peke" +Shih Tzu,Shih-Tzu +King Charles Spaniel,Blenheim spaniel +Papillon,papillon +toy terrier,toy terrier +Rhodesian Ridgeback,Rhodesian ridgeback +Afghan Hound,"Afghan hound, Afghan" +Basset Hound,"basset, basset hound" +Beagle,beagle +Bloodhound,"bloodhound, sleuthhound" +Bluetick Coonhound,bluetick +Black and Tan Coonhound,black-and-tan coonhound +Treeing Walker Coonhound,"Walker hound, Walker foxhound" +English foxhound,English foxhound +Redbone Coonhound,redbone +borzoi,"borzoi, Russian wolfhound" +Irish Wolfhound,Irish wolfhound +Italian Greyhound,Italian greyhound +Whippet,whippet +Ibizan Hound,"Ibizan hound, Ibizan Podenco" +Norwegian Elkhound,"Norwegian elkhound, elkhound" +Otterhound,"otterhound, otter hound" +Saluki,"Saluki, gazelle hound" +Scottish Deerhound,"Scottish deerhound, deerhound" +Weimaraner,Weimaraner +Staffordshire Bull Terrier,"Staffordshire bullterrier, Staffordshire bull terrier" +American Staffordshire Terrier,"American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier" +Bedlington Terrier,Bedlington terrier +Border Terrier,Border terrier +Kerry Blue Terrier,Kerry blue terrier +Irish Terrier,Irish terrier +Norfolk Terrier,Norfolk terrier +Norwich Terrier,Norwich terrier +Yorkshire Terrier,Yorkshire terrier +Wire Fox Terrier,wire-haired fox terrier +Lakeland Terrier,Lakeland terrier +Sealyham Terrier,"Sealyham terrier, Sealyham" +Airedale Terrier,"Airedale, Airedale terrier" +Cairn Terrier,"cairn, cairn terrier" +Australian Terrier,Australian terrier +Dandie Dinmont Terrier,"Dandie Dinmont, Dandie Dinmont terrier" +Boston Terrier,"Boston bull, Boston terrier" +Miniature Schnauzer,miniature schnauzer +Giant Schnauzer,giant schnauzer +Standard Schnauzer,standard schnauzer +Scottish Terrier,"Scotch terrier, Scottish terrier, Scottie" +Tibetan Terrier,"Tibetan terrier, chrysanthemum dog" +Australian Silky Terrier,"silky terrier, Sydney silky" +Soft-coated Wheaten Terrier,soft-coated wheaten terrier +West Highland White Terrier,West Highland white terrier +Lhasa Apso,"Lhasa, Lhasa apso" +Flat-Coated Retriever,flat-coated retriever +Curly-coated Retriever,curly-coated retriever +Golden Retriever,golden retriever +Labrador Retriever,Labrador retriever +Chesapeake Bay Retriever,Chesapeake Bay retriever +German Shorthaired Pointer,German short-haired pointer +Vizsla,"vizsla, Hungarian pointer" +English Setter,English setter +Irish Setter,"Irish setter, red setter" +Gordon Setter,Gordon setter +Brittany dog,Brittany spaniel +Clumber Spaniel,"clumber, clumber spaniel" +English Springer Spaniel,"English springer, English springer spaniel" +Welsh Springer Spaniel,Welsh springer spaniel +Cocker Spaniel,"cocker spaniel, English cocker spaniel, cocker" +Sussex Spaniel,Sussex spaniel +Irish Water Spaniel,Irish water spaniel +Kuvasz,kuvasz +Schipperke,schipperke +Groenendael dog,groenendael +Malinois,malinois +Briard,briard +Australian Kelpie,kelpie +Komondor,komondor +Old English Sheepdog,"Old English sheepdog, bobtail" +Shetland Sheepdog,"Shetland sheepdog, Shetland sheep dog, Shetland" +collie,collie +Border Collie,Border collie +Bouvier des Flandres dog,"Bouvier des Flandres, Bouviers des Flandres" +Rottweiler,Rottweiler +German Shepherd Dog,"German shepherd, German shepherd dog, German police dog, alsatian" +Dobermann,"Doberman, Doberman pinscher" +Miniature Pinscher,miniature pinscher +Greater Swiss Mountain Dog,Greater Swiss Mountain dog +Bernese Mountain Dog,Bernese mountain dog +Appenzeller Sennenhund,Appenzeller +Entlebucher Sennenhund,EntleBucher +Boxer,boxer +Bullmastiff,bull mastiff +Tibetan Mastiff,Tibetan mastiff +French Bulldog,French bulldog +Great Dane,Great Dane +St. Bernard,"Saint Bernard, St Bernard" +husky,"Eskimo dog, husky" +Alaskan Malamute,"malamute, malemute, Alaskan malamute" +Siberian Husky,Siberian husky +Dalmatian,"dalmatian, coach dog, carriage dog" +Affenpinscher,"affenpinscher, monkey pinscher, monkey dog" +Basenji,basenji +pug,"pug, pug-dog" +Leonberger,Leonberg +Newfoundland dog,"Newfoundland, Newfoundland dog" +Great Pyrenees dog,Great Pyrenees +Samoyed,"Samoyed, Samoyede" +Pomeranian,Pomeranian +Chow Chow,"chow, chow chow" +Keeshond,keeshond +brussels griffon,Brabancon griffon +Pembroke Welsh Corgi,"Pembroke, Pembroke Welsh corgi" +Cardigan Welsh Corgi,"Cardigan, Cardigan Welsh corgi" +Toy Poodle,toy poodle +Miniature Poodle,miniature poodle +Standard Poodle,standard poodle +Mexican hairless dog (xoloitzcuintli),Mexican hairless +grey wolf,"timber wolf, grey wolf, gray wolf, Canis lupus" +Alaskan tundra wolf,"white wolf, Arctic wolf, Canis lupus tundrarum" +red wolf or maned wolf,"red wolf, maned wolf, Canis rufus, Canis niger" +coyote,"coyote, prairie wolf, brush wolf, Canis latrans" +dingo,"dingo, warrigal, warragal, Canis dingo" +dhole,"dhole, Cuon alpinus" +African wild dog,"African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus" +hyena,"hyena, hyaena" +red fox,"red fox, Vulpes vulpes" +kit fox,"kit fox, Vulpes macrotis" +Arctic fox,"Arctic fox, white fox, Alopex lagopus" +grey fox,"grey fox, gray fox, Urocyon cinereoargenteus" +tabby cat,"tabby, tabby cat" +tiger cat,tiger cat +Persian cat,Persian cat +Siamese cat,"Siamese cat, Siamese" +Egyptian Mau,Egyptian cat +cougar,"cougar, puma, catamount, mountain lion, painter, panther, Felis concolor" +lynx,"lynx, catamount" +leopard,"leopard, Panthera pardus" +snow leopard,"snow leopard, ounce, Panthera uncia" +jaguar,"jaguar, panther, Panthera onca, Felis onca" +lion,"lion, king of beasts, Panthera leo" +tiger,"tiger, Panthera tigris" +cheetah,"cheetah, chetah, Acinonyx jubatus" +brown bear,"brown bear, bruin, Ursus arctos" +American black bear,"American black bear, black bear, Ursus americanus, Euarctos americanus" +polar bear,"ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus" +sloth bear,"sloth bear, Melursus ursinus, Ursus ursinus" +mongoose,mongoose +meerkat,"meerkat, mierkat" +tiger beetle,tiger beetle +ladybug,"ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle" +ground beetle,"ground beetle, carabid beetle" +longhorn beetle,"long-horned beetle, longicorn, longicorn beetle" +leaf beetle,"leaf beetle, chrysomelid" +dung beetle,dung beetle +rhinoceros beetle,rhinoceros beetle +weevil,weevil +fly,fly +bee,bee +ant,"ant, emmet, pismire" +grasshopper,"grasshopper, hopper" +cricket insect,cricket +stick insect,"walking stick, walkingstick, stick insect" +cockroach,"cockroach, roach" +praying mantis,"mantis, mantid" +cicada,"cicada, cicala" +leafhopper,leafhopper +lacewing,"lacewing, lacewing fly" +dragonfly,"dragonfly, darning needle, devil's darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk" +damselfly,damselfly +red admiral butterfly,admiral +ringlet butterfly,"ringlet, ringlet butterfly" +monarch butterfly,"monarch, monarch butterfly, milkweed butterfly, Danaus plexippus" +small white butterfly,cabbage butterfly +sulphur butterfly,"sulphur butterfly, sulfur butterfly" +gossamer-winged butterfly,"lycaenid, lycaenid butterfly" +starfish,"starfish, sea star" +sea urchin,sea urchin +sea cucumber,"sea cucumber, holothurian" +cottontail rabbit,"wood rabbit, cottontail, cottontail rabbit" +hare,hare +Angora rabbit,"Angora, Angora rabbit" +hamster,hamster +porcupine,"porcupine, hedgehog" +fox squirrel,"fox squirrel, eastern fox squirrel, Sciurus niger" +marmot,marmot +beaver,beaver +guinea pig,"guinea pig, Cavia cobaya" +common sorrel horse,sorrel +zebra,zebra +pig,"hog, pig, grunter, squealer, Sus scrofa" +wild boar,"wild boar, boar, Sus scrofa" +warthog,warthog +hippopotamus,"hippopotamus, hippo, river horse, Hippopotamus amphibius" +ox,ox +water buffalo,"water buffalo, water ox, Asiatic buffalo, Bubalus bubalis" +bison,bison +ram (adult male sheep),"ram, tup" +bighorn sheep,"bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis" +Alpine ibex,"ibex, Capra ibex" +hartebeest,hartebeest +impala (antelope),"impala, Aepyceros melampus" +gazelle,gazelle +arabian camel,"Arabian camel, dromedary, Camelus dromedarius" +llama,llama +weasel,weasel +mink,mink +European polecat,"polecat, fitch, foulmart, foumart, Mustela putorius" +black-footed ferret,"black-footed ferret, ferret, Mustela nigripes" +otter,otter +skunk,"skunk, polecat, wood pussy" +badger,badger +armadillo,armadillo +three-toed sloth,"three-toed sloth, ai, Bradypus tridactylus" +orangutan,"orangutan, orang, orangutang, Pongo pygmaeus" +gorilla,"gorilla, Gorilla gorilla" +chimpanzee,"chimpanzee, chimp, Pan troglodytes" +gibbon,"gibbon, Hylobates lar" +siamang,"siamang, Hylobates syndactylus, Symphalangus syndactylus" +guenon,"guenon, guenon monkey" +patas monkey,"patas, hussar monkey, Erythrocebus patas" +baboon,baboon +macaque,macaque +langur,langur +black-and-white colobus,"colobus, colobus monkey" +proboscis monkey,"proboscis monkey, Nasalis larvatus" +marmoset,marmoset +white-headed capuchin,"capuchin, ringtail, Cebus capucinus" +howler monkey,"howler monkey, howler" +titi monkey,"titi, titi monkey" +Geoffroy's spider monkey,"spider monkey, Ateles geoffroyi" +common squirrel monkey,"squirrel monkey, Saimiri sciureus" +ring-tailed lemur,"Madagascar cat, ring-tailed lemur, Lemur catta" +indri,"indri, indris, Indri indri, Indri brevicaudatus" +Asian elephant,"Indian elephant, Elephas maximus" +African bush elephant,"African elephant, Loxodonta africana" +red panda,"lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens" +giant panda,"giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca" +snoek fish,"barracouta, snoek" +eel,eel +silver salmon,"coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch" +rock beauty fish,"rock beauty, Holocanthus tricolor" +clownfish,anemone fish +sturgeon,sturgeon +gar fish,"gar, garfish, garpike, billfish, Lepisosteus osseus" +lionfish,lionfish +pufferfish,"puffer, pufferfish, blowfish, globefish" +abacus,abacus +abaya,abaya +academic gown,"academic gown, academic robe, judge's robe" +accordion,"accordion, piano accordion, squeeze box" +acoustic guitar,acoustic guitar +aircraft carrier,"aircraft carrier, carrier, flattop, attack aircraft carrier" +airliner,airliner +airship,"airship, dirigible" +altar,altar +ambulance,ambulance +amphibious vehicle,"amphibian, amphibious vehicle" +analog clock,analog clock +apiary,"apiary, bee house" +apron,apron +trash can,"ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin" +assault rifle,"assault rifle, assault gun" +backpack,"backpack, back pack, knapsack, packsack, rucksack, haversack" +bakery,"bakery, bakeshop, bakehouse" +balance beam,"balance beam, beam" +balloon,balloon +ballpoint pen,"ballpoint, ballpoint pen, ballpen, Biro" +Band-Aid,Band Aid +banjo,banjo +baluster / handrail,"bannister, banister, balustrade, balusters, handrail" +barbell,barbell +barber chair,barber chair +barbershop,barbershop +barn,barn +barometer,barometer +barrel,"barrel, cask" +wheelbarrow,"barrow, garden cart, lawn cart, wheelbarrow" +baseball,baseball +basketball,basketball +bassinet,bassinet +bassoon,bassoon +swimming cap,"bathing cap, swimming cap" +bath towel,bath towel +bathtub,"bathtub, bathing tub, bath, tub" +station wagon,"beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon" +lighthouse,"beacon, lighthouse, beacon light, pharos" +beaker,beaker +military hat (bearskin or shako),"bearskin, busby, shako" +beer bottle,beer bottle +beer glass,beer glass +bell tower,"bell cote, bell cot" +baby bib,bib +tandem bicycle,"bicycle-built-for-two, tandem bicycle, tandem" +bikini,"bikini, two-piece" +ring binder,"binder, ring-binder" +binoculars,"binoculars, field glasses, opera glasses" +birdhouse,birdhouse +boathouse,boathouse +bobsleigh,"bobsled, bobsleigh, bob" +bolo tie,"bolo tie, bolo, bola tie, bola" +poke bonnet,"bonnet, poke bonnet" +bookcase,bookcase +bookstore,"bookshop, bookstore, bookstall" +bottle cap,bottlecap +hunting bow,bow +bow tie,"bow tie, bow-tie, bowtie" +brass memorial plaque,"brass, memorial tablet, plaque" +bra,"brassiere, bra, bandeau" +breakwater,"breakwater, groin, groyne, mole, bulwark, seawall, jetty" +breastplate,"breastplate, aegis, egis" +broom,broom +bucket,"bucket, pail" +buckle,buckle +bulletproof vest,bulletproof vest +high-speed train,"bullet train, bullet" +butcher shop,"butcher shop, meat market" +taxicab,"cab, hack, taxi, taxicab" +cauldron,"caldron, cauldron" +candle,"candle, taper, wax light" +cannon,cannon +canoe,canoe +can opener,"can opener, tin opener" +cardigan,cardigan +car mirror,car mirror +carousel,"carousel, carrousel, merry-go-round, roundabout, whirligig" +tool kit,"carpenter's kit, tool kit" +cardboard box / carton,carton +car wheel,car wheel +automated teller machine,"cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM" +cassette,cassette +cassette player,cassette player +castle,castle +catamaran,catamaran +CD player,CD player +cello,"cello, violoncello" +mobile phone,"cellular telephone, cellular phone, cellphone, cell, mobile phone" +chain,chain +chain-link fence,chainlink fence +chain mail,"chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour" +chainsaw,"chain saw, chainsaw" +storage chest,chest +chiffonier,"chiffonier, commode" +bell or wind chime,"chime, bell, gong" +china cabinet,"china cabinet, china closet" +Christmas stocking,Christmas stocking +church,"church, church building" +movie theater,"cinema, movie theater, movie theatre, movie house, picture palace" +cleaver,"cleaver, meat cleaver, chopper" +cliff dwelling,cliff dwelling +cloak,cloak +clogs,"clog, geta, patten, sabot" +cocktail shaker,cocktail shaker +coffee mug,coffee mug +coffeemaker,coffeepot +spiral or coil,"coil, spiral, volute, whorl, helix" +combination lock,combination lock +computer keyboard,"computer keyboard, keypad" +candy store,"confectionery, confectionary, candy store" +container ship,"container ship, containership, container vessel" +convertible,convertible +corkscrew,"corkscrew, bottle screw" +cornet,"cornet, horn, trumpet, trump" +cowboy boot,cowboy boot +cowboy hat,"cowboy hat, ten-gallon hat" +cradle,cradle +construction crane,crane +crash helmet,crash helmet +crate,crate +infant bed,"crib, cot" +Crock Pot,Crock Pot +croquet ball,croquet ball +crutch,crutch +cuirass,cuirass +dam,"dam, dike, dyke" +desk,desk +desktop computer,desktop computer +rotary dial telephone,"dial telephone, dial phone" +diaper,"diaper, nappy, napkin" +digital clock,digital clock +digital watch,digital watch +dining table,"dining table, board" +dishcloth,"dishrag, dishcloth" +dishwasher,"dishwasher, dish washer, dishwashing machine" +disc brake,"disk brake, disc brake" +dock,"dock, dockage, docking facility" +dog sled,"dogsled, dog sled, dog sleigh" +dome,dome +doormat,"doormat, welcome mat" +drilling rig,"drilling platform, offshore rig" +drum,"drum, membranophone, tympan" +drumstick,drumstick +dumbbell,dumbbell +Dutch oven,Dutch oven +electric fan,"electric fan, blower" +electric guitar,electric guitar +electric locomotive,electric locomotive +entertainment center,entertainment center +envelope,envelope +espresso machine,espresso maker +face powder,face powder +feather boa,"feather boa, boa" +filing cabinet,"file, file cabinet, filing cabinet" +fireboat,fireboat +fire truck,"fire engine, fire truck" +fire screen,"fire screen, fireguard" +flagpole,"flagpole, flagstaff" +flute,"flute, transverse flute" +folding chair,folding chair +football helmet,football helmet +forklift,forklift +fountain,fountain +fountain pen,fountain pen +four-poster bed,four-poster +freight car,freight car +French horn,"French horn, horn" +frying pan,"frying pan, frypan, skillet" +fur coat,fur coat +garbage truck,"garbage truck, dustcart" +gas mask or respirator,"gasmask, respirator, gas helmet" +gas pump,"gas pump, gasoline pump, petrol pump, island dispenser" +goblet,goblet +go-kart,go-kart +golf ball,golf ball +golf cart,"golfcart, golf cart" +gondola,gondola +gong,"gong, tam-tam" +gown,gown +grand piano,"grand piano, grand" +greenhouse,"greenhouse, nursery, glasshouse" +radiator grille,"grille, radiator grille" +grocery store,"grocery store, grocery, food market, market" +guillotine,guillotine +hair clip,hair slide +hair spray,hair spray +half-track,half track +hammer,hammer +hamper,hamper +hair dryer,"hand blower, blow dryer, blow drier, hair dryer, hair drier" +hand-held computer,"hand-held computer, hand-held microcomputer" +handkerchief,"handkerchief, hankie, hanky, hankey" +hard disk drive,"hard disc, hard disk, fixed disk" +harmonica,"harmonica, mouth organ, harp, mouth harp" +harp,harp +combine harvester,"harvester, reaper" +hatchet,hatchet +holster,holster +home theater,"home theater, home theatre" +honeycomb,honeycomb +hook,"hook, claw" +hoop skirt,"hoopskirt, crinoline" +gymnastic horizontal bar,"horizontal bar, high bar" +horse-drawn vehicle,"horse cart, horse-cart" +hourglass,hourglass +iPod,iPod +clothes iron,"iron, smoothing iron" +carved pumpkin,jack-o'-lantern +jeans,"jean, blue jean, denim" +jeep,"jeep, landrover" +T-shirt,"jersey, T-shirt, tee shirt" +jigsaw puzzle,jigsaw puzzle +rickshaw,"jinrikisha, ricksha, rickshaw" +joystick,joystick +kimono,kimono +knee pad,knee pad +knot,knot +lab coat,"lab coat, laboratory coat" +ladle,ladle +lampshade,"lampshade, lamp shade" +laptop computer,"laptop, laptop computer" +lawn mower,"lawn mower, mower" +lens cap,"lens cap, lens cover" +letter opener,"letter opener, paper knife, paperknife" +library,library +lifeboat,lifeboat +lighter,"lighter, light, igniter, ignitor" +limousine,"limousine, limo" +ocean liner,"liner, ocean liner" +lipstick,"lipstick, lip rouge" +slip-on shoe,Loafer +lotion,lotion +music speaker,"loudspeaker, speaker, speaker unit, loudspeaker system, speaker system" +loupe magnifying glass,"loupe, jeweler's loupe" +sawmill,"lumbermill, sawmill" +magnetic compass,magnetic compass +messenger bag,"mailbag, postbag" +mailbox,"mailbox, letter box" +tights,maillot +one-piece bathing suit,"maillot, tank suit" +manhole cover,manhole cover +maraca,maraca +marimba,"marimba, xylophone" +mask,mask +matchstick,matchstick +maypole,maypole +maze,"maze, labyrinth" +measuring cup,measuring cup +medicine cabinet,"medicine chest, medicine cabinet" +megalith,"megalith, megalithic structure" +microphone,"microphone, mike" +microwave oven,"microwave, microwave oven" +military uniform,military uniform +milk can,milk can +minibus,minibus +miniskirt,"miniskirt, mini" +minivan,minivan +missile,missile +mitten,mitten +mixing bowl,mixing bowl +mobile home,"mobile home, manufactured home" +ford model t,Model T +modem,modem +monastery,monastery +monitor,monitor +moped,moped +mortar and pestle,mortar +graduation cap,mortarboard +mosque,mosque +mosquito net,mosquito net +vespa,"motor scooter, scooter" +mountain bike,"mountain bike, all-terrain bike, off-roader" +tent,mountain tent +computer mouse,"mouse, computer mouse" +mousetrap,mousetrap +moving van,moving van +muzzle,muzzle +metal nail,nail +neck brace,neck brace +necklace,necklace +baby pacifier,nipple +notebook computer,"notebook, notebook computer" +obelisk,obelisk +oboe,"oboe, hautboy, hautbois" +ocarina,"ocarina, sweet potato" +odometer,"odometer, hodometer, mileometer, milometer" +oil filter,oil filter +pipe organ,"organ, pipe organ" +oscilloscope,"oscilloscope, scope, cathode-ray oscilloscope, CRO" +overskirt,overskirt +bullock cart,oxcart +oxygen mask,oxygen mask +product packet / packaging,packet +paddle,"paddle, boat paddle" +paddle wheel,"paddlewheel, paddle wheel" +padlock,padlock +paintbrush,paintbrush +pajamas,"pajama, pyjama, pj's, jammies" +palace,palace +pan flute,"panpipe, pandean pipe, syrinx" +paper towel,paper towel +parachute,"parachute, chute" +parallel bars,"parallel bars, bars" +park bench,park bench +parking meter,parking meter +railroad car,"passenger car, coach, carriage" +patio,"patio, terrace" +payphone,"pay-phone, pay-station" +pedestal,"pedestal, plinth, footstall" +pencil case,"pencil box, pencil case" +pencil sharpener,pencil sharpener +perfume,"perfume, essence" +Petri dish,Petri dish +photocopier,photocopier +plectrum,"pick, plectrum, plectron" +Pickelhaube,pickelhaube +picket fence,"picket fence, paling" +pickup truck,"pickup, pickup truck" +pier,pier +piggy bank,"piggy bank, penny bank" +pill bottle,pill bottle +pillow,pillow +ping-pong ball,ping-pong ball +pinwheel,pinwheel +pirate ship,"pirate, pirate ship" +drink pitcher,"pitcher, ewer" +block plane,"plane, carpenter's plane, woodworking plane" +planetarium,planetarium +plastic bag,plastic bag +plate rack,plate rack +farm plow,"plow, plough" +plunger,"plunger, plumber's helper" +Polaroid camera,"Polaroid camera, Polaroid Land camera" +pole,pole +police van,"police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria" +poncho,poncho +pool table,"pool table, billiard table, snooker table" +soda bottle,"pop bottle, soda bottle" +plant pot,"pot, flowerpot" +potter's wheel,potter's wheel +power drill,power drill +prayer rug,"prayer rug, prayer mat" +printer,printer +prison,"prison, prison house" +missile,"projectile, missile" +projector,projector +hockey puck,"puck, hockey puck" +punching bag,"punching bag, punch bag, punching ball, punchball" +purse,purse +quill,"quill, quill pen" +quilt,"quilt, comforter, comfort, puff" +race car,"racer, race car, racing car" +racket,"racket, racquet" +radiator,radiator +radio,"radio, wireless" +radio telescope,"radio telescope, radio reflector" +rain barrel,rain barrel +recreational vehicle,"recreational vehicle, RV, R.V." +fishing casting reel,reel +reflex camera,reflex camera +refrigerator,"refrigerator, icebox" +remote control,"remote control, remote" +restaurant,"restaurant, eating house, eating place, eatery" +revolver,"revolver, six-gun, six-shooter" +rifle,rifle +rocking chair,"rocking chair, rocker" +rotisserie,rotisserie +eraser,"rubber eraser, rubber, pencil eraser" +rugby ball,rugby ball +ruler measuring stick,"rule, ruler" +sneaker,running shoe +safe,safe +safety pin,safety pin +salt shaker,"saltshaker, salt shaker" +sandal,sandal +sarong,sarong +saxophone,"sax, saxophone" +scabbard,scabbard +weighing scale,"scale, weighing machine" +school bus,school bus +schooner,schooner +scoreboard,scoreboard +CRT monitor,"screen, CRT screen" +screw,screw +screwdriver,screwdriver +seat belt,"seat belt, seatbelt" +sewing machine,sewing machine +shield,"shield, buckler" +shoe store,"shoe shop, shoe-shop, shoe store" +shoji screen / room divider,shoji +shopping basket,shopping basket +shopping cart,shopping cart +shovel,shovel +shower cap,shower cap +shower curtain,shower curtain +ski,ski +balaclava ski mask,ski mask +sleeping bag,sleeping bag +slide rule,"slide rule, slipstick" +sliding door,sliding door +slot machine,"slot, one-armed bandit" +snorkel,snorkel +snowmobile,snowmobile +snowplow,"snowplow, snowplough" +soap dispenser,soap dispenser +soccer ball,soccer ball +sock,sock +solar thermal collector,"solar dish, solar collector, solar furnace" +sombrero,sombrero +soup bowl,soup bowl +keyboard space bar,space bar +space heater,space heater +space shuttle,space shuttle +spatula,spatula +motorboat,speedboat +spider web,"spider web, spider's web" +spindle,spindle +sports car,"sports car, sport car" +spotlight,"spotlight, spot" +stage,stage +steam locomotive,steam locomotive +through arch bridge,steel arch bridge +steel drum,steel drum +stethoscope,stethoscope +scarf,stole +stone wall,stone wall +stopwatch,"stopwatch, stop watch" +stove,stove +strainer,strainer +tram,"streetcar, tram, tramcar, trolley, trolley car" +stretcher,stretcher +couch,"studio couch, day bed" +stupa,"stupa, tope" +submarine,"submarine, pigboat, sub, U-boat" +suit,"suit, suit of clothes" +sundial,sundial +sunglasses,sunglass +sunglasses,"sunglasses, dark glasses, shades" +sunscreen,"sunscreen, sunblock, sun blocker" +suspension bridge,suspension bridge +mop,"swab, swob, mop" +sweatshirt,sweatshirt +swim trunks / shorts,"swimming trunks, bathing trunks" +swing,swing +electrical switch,"switch, electric switch, electrical switch" +syringe,syringe +table lamp,table lamp +tank,"tank, army tank, armored combat vehicle, armoured combat vehicle" +tape player,tape player +teapot,teapot +teddy bear,"teddy, teddy bear" +television,"television, television system" +tennis ball,tennis ball +thatched roof,"thatch, thatched roof" +front curtain,"theater curtain, theatre curtain" +thimble,thimble +threshing machine,"thresher, thrasher, threshing machine" +throne,throne +tile roof,tile roof +toaster,toaster +tobacco shop,"tobacco shop, tobacconist shop, tobacconist" +toilet seat,toilet seat +torch,torch +totem pole,totem pole +tow truck,"tow truck, tow car, wrecker" +toy store,toyshop +tractor,tractor +semi-trailer truck,"trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi" +tray,tray +trench coat,trench coat +tricycle,"tricycle, trike, velocipede" +trimaran,trimaran +tripod,tripod +triumphal arch,triumphal arch +trolleybus,"trolleybus, trolley coach, trackless trolley" +trombone,trombone +hot tub,"tub, vat" +turnstile,turnstile +typewriter keyboard,typewriter keyboard +umbrella,umbrella +unicycle,"unicycle, monocycle" +upright piano,"upright, upright piano" +vacuum cleaner,"vacuum, vacuum cleaner" +vase,vase +vaulted or arched ceiling,vault +velvet fabric,velvet +vending machine,vending machine +vestment,vestment +viaduct,viaduct +violin,"violin, fiddle" +volleyball,volleyball +waffle iron,waffle iron +wall clock,wall clock +wallet,"wallet, billfold, notecase, pocketbook" +wardrobe,"wardrobe, closet, press" +military aircraft,"warplane, military plane" +sink,"washbasin, handbasin, washbowl, lavabo, wash-hand basin" +washing machine,"washer, automatic washer, washing machine" +water bottle,water bottle +water jug,water jug +water tower,water tower +whiskey jug,whiskey jug +whistle,whistle +hair wig,wig +window screen,window screen +window shade,window shade +Windsor tie,Windsor tie +wine bottle,wine bottle +airplane wing,wing +wok,wok +wooden spoon,wooden spoon +wool,"wool, woolen, woollen" +split-rail fence,"worm fence, snake fence, snake-rail fence, Virginia fence" +shipwreck,wreck +sailboat,yawl +yurt,yurt +website,"web site, website, internet site, site" +comic book,comic book +crossword,"crossword puzzle, crossword" +traffic or street sign,street sign +traffic light,"traffic light, traffic signal, stoplight" +dust jacket,"book jacket, dust cover, dust jacket, dust wrapper" +menu,menu +plate,plate +guacamole,guacamole +consomme,consomme +hot pot,"hot pot, hotpot" +trifle,trifle +ice cream,"ice cream, icecream" +popsicle,"ice lolly, lolly, lollipop, popsicle" +baguette,French loaf +bagel,"bagel, beigel" +pretzel,pretzel +cheeseburger,cheeseburger +hot dog,"hotdog, hot dog, red hot" +mashed potatoes,mashed potato +cabbage,head cabbage +broccoli,broccoli +cauliflower,cauliflower +zucchini,"zucchini, courgette" +spaghetti squash,spaghetti squash +acorn squash,acorn squash +butternut squash,butternut squash +cucumber,"cucumber, cuke" +artichoke,"artichoke, globe artichoke" +bell pepper,bell pepper +cardoon,cardoon +mushroom,mushroom +Granny Smith apple,Granny Smith +strawberry,strawberry +orange,orange +lemon,lemon +fig,fig +pineapple,"pineapple, ananas" +banana,banana +jackfruit,"jackfruit, jak, jack" +cherimoya (custard apple),custard apple +pomegranate,pomegranate +hay,hay +carbonara,carbonara +chocolate syrup,"chocolate sauce, chocolate syrup" +dough,dough +meatloaf,"meat loaf, meatloaf" +pizza,"pizza, pizza pie" +pot pie,potpie +burrito,burrito +red wine,red wine +espresso,espresso +tea cup,cup +eggnog,eggnog +mountain,alp +bubble,bubble +cliff,"cliff, drop, drop-off" +coral reef,coral reef +geyser,geyser +lakeshore,"lakeside, lakeshore" +promontory,"promontory, headland, head, foreland" +sandbar,"sandbar, sand bar" +beach,"seashore, coast, seacoast, sea-coast" +valley,"valley, vale" +volcano,volcano +baseball player,"ballplayer, baseball player" +bridegroom,"groom, bridegroom" +scuba diver,scuba diver +rapeseed,rapeseed +daisy,daisy +yellow lady's slipper,"yellow lady's slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum" +corn,corn +acorn,acorn +rose hip,"hip, rose hip, rosehip" +horse chestnut seed,"buckeye, horse chestnut, conker" +coral fungus,coral fungus +agaric,agaric +gyromitra,gyromitra +stinkhorn mushroom,"stinkhorn, carrion fungus" +earth star fungus,earthstar +hen of the woods mushroom,"hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa" +bolete,bolete +corn cob,"ear, spike, capitulum" +toilet paper,"toilet tissue, toilet paper, bathroom tissue" diff --git a/python/ClipDetection/clip_component/data/one_template.txt b/python/ClipDetection/clip_component/data/one_template.txt new file mode 100644 index 00000000..95ce86b9 --- /dev/null +++ b/python/ClipDetection/clip_component/data/one_template.txt @@ -0,0 +1 @@ +A photo of a {}. \ No newline at end of file diff --git a/python/ClipDetection/clip_component/data/seven_templates.txt b/python/ClipDetection/clip_component/data/seven_templates.txt new file mode 100644 index 00000000..2ba4ff13 --- /dev/null +++ b/python/ClipDetection/clip_component/data/seven_templates.txt @@ -0,0 +1,7 @@ +itap of a {}. +a bad photo of the {}. +an origami {}. +a photo of the large {}. +a {} in a video game. +art of the {}. +a photo of the small {}. \ No newline at end of file diff --git a/python/ClipDetection/plugin-files/descriptor/descriptor.json b/python/ClipDetection/plugin-files/descriptor/descriptor.json new file mode 100644 index 00000000..97fe74dd --- /dev/null +++ b/python/ClipDetection/plugin-files/descriptor/descriptor.json @@ -0,0 +1,186 @@ +{ + "componentName": "ClipDetection", + "componentVersion": "7.2", + "middlewareVersion": "7.2", + "sourceLanguage": "python", + "batchLibrary": "ClipDetection", + "environmentVariables": [], + "algorithm": { + "name": "CLIP", + "description": "CLIP classification.", + "detectionType": "CLASS", + "actionType": "DETECTION", + "outputChangedCounter": 1, + "requiresCollection": { + "states": [] + }, + "providesCollection": { + "states": [ + "DETECTION", + "DETECTION_CLASS", + "DETECTION_CLASS_CLIP" + ], + "properties": [ + { + "name": "NUMBER_OF_CLASSIFICATIONS", + "description": "The number of classifications, N, to be returned. The N highest confidence classifications found by the network will be returned with their associated confidence values. The value must be greater than 0, and less than the size of the model output layer.", + "type": "INT", + "defaultValue": "1" + }, + { + "name": "NUMBER_OF_TEMPLATES", + "description": "The number of templates to be used in the text encoder. The current acceptable values are 7 and 80.", + "type": "INT", + "defaultValue": "80" + }, + { + "name": "CLASSIFICATION_LIST", + "description": "Specifies the classification list that will be tokenized for the text encoder (supports 'imagenet' and 'coco'). By default, the COCO classifications will be used.", + "type": "STRING", + "defaultValue": "coco" + }, + { + "name": "CLASSIFICATION_PATH", + "description": "Optionally specifies a path to a custom csv file containing two names for each classification: one is the full name to display and the other to enter into the CLIP text encoding.", + "type": "STRING", + "defaultValue": "" + }, + { + "name": "TEMPLATE_PATH", + "description": "Optionally specifies a path to a custom text file containing templates for use in the CLIP model. Include a single {} where each classification is to be inserted.", + "type": "STRING", + "defaultValue": "" + }, + { + "name": "ENABLE_CROPPING", + "description": "If true, the image will be cropped into 144 images of size 224x224. The results from each of these images is averaged to get the results. Not available for use on CPU.", + "type": "BOOLEAN", + "defaultValue": "true" + }, + { + "name": "ENABLE_TRITON", + "description": "If true, inferencing will be performed via a configured Triton inference server.", + "type": "BOOLEAN", + "defaultValue": "false" + }, + { + "name": "INCLUDE_FEATURES", + "description": "If true, the detection will have a detection property, FEATURE, which contains the base64-encoded version of the feature vector.", + "type": "BOOLEAN", + "defaultValue": "false" + }, + { + "name": "TRITON_SERVER", + "description": "Triton server : to use for inferencing.", + "type": "STRING", + "defaultValue": "clip-detection-server:8001" + } + ] + } + }, + "actions": [ + { + "name": "CLIP COCO CLASSIFICATION ACTION", + "description": "Runs CLIP classification on the COCO dataset classes.", + "algorithm": "CLIP", + "properties": [] + }, + { + "name": "CLIP TRITON COCO CLASSIFICATION ACTION", + "description": "Runs CLIP classification on the COCO dataset classes using a Triton Inferencing Server.", + "algorithm": "CLIP", + "properties": [ + { + "name": "ENABLE_TRITON", + "value": "true" + } + ] + }, + { + "name": "CLIP IMAGENET CLASSIFICATION ACTION", + "description": "Runs CLIP classification on the ImageNet dataset classes.", + "algorithm": "CLIP", + "properties": [ + { + "name": "CLASSIFICATION_LIST", + "value": "imagenet" + } + ] + }, + { + "name": "CLIP TRITON IMAGENET CLASSIFICATION ACTION", + "description": "Runs CLIP classification on the ImageNet dataset classes using a Triton Inferencing Server.", + "algorithm": "CLIP", + "properties": [ + { + "name": "ENABLE_TRITON", + "value": "true" + }, + { + "name": "CLASSIFICATION_LIST", + "value": "imagenet" + } + ] + } + ], + "tasks": [ + { + "name": "CLIP COCO CLASSIFICATION TASK", + "description": "Runs CLIP classification on the COCO dataset classes.", + "actions": [ + "CLIP COCO CLASSIFICATION ACTION" + ] + }, + { + "name": "CLIP TRITON COCO CLASSIFICATION TASK", + "description": "Runs CLIP classification on the COCO dataset classes using a Triton Inferencing Server.", + "actions": [ + "CLIP TRITON COCO CLASSIFICATION ACTION" + ] + }, + { + "name": "CLIP IMAGENET CLASSIFICATION TASK", + "description": "Runs CLIP classification on the ImageNet dataset classes.", + "actions": [ + "CLIP IMAGENET CLASSIFICATION ACTION" + ] + }, + { + "name": "CLIP TRITON IMAGENET CLASSIFICATION TASK", + "description": "Runs CLIP classification on the ImageNet dataset classes using a Triton Inferencing Server.", + "actions": [ + "CLIP TRITON IMAGENET CLASSIFICATION ACTION" + ] + } + ], + "pipelines": [ + { + "name": "CLIP COCO CLASSIFICATION PIPELINE", + "description": "Runs CLIP classification on the COCO dataset classes.", + "tasks": [ + "CLIP COCO CLASSIFICATION TASK" + ] + }, + { + "name": "CLIP TRITON COCO CLASSIFICATION PIPELINE", + "description": "Runs CLIP classification on the COCO dataset classes using a Triton Inferencing Server.", + "tasks": [ + "CLIP TRITON COCO CLASSIFICATION TASK" + ] + }, + { + "name": "CLIP IMAGENET CLASSIFICATION PIPELINE", + "description": "Runs CLIP classification on the ImageNet dataset classes.", + "tasks": [ + "CLIP IMAGENET CLASSIFICATION TASK" + ] + }, + { + "name": "CLIP TRITON IMAGENET CLASSIFICATION PIPELINE", + "description": "Runs CLIP classification on the ImageNet dataset classes using a Triton Inferencing Server.", + "tasks": [ + "CLIP TRITON IMAGENET CLASSIFICATION TASK" + ] + } + ] +} \ No newline at end of file diff --git a/python/ClipDetection/pyproject.toml b/python/ClipDetection/pyproject.toml new file mode 100644 index 00000000..52c60148 --- /dev/null +++ b/python/ClipDetection/pyproject.toml @@ -0,0 +1,29 @@ +############################################################################# +# NOTICE # +# # +# This software (or technical data) was produced for the U.S. Government # +# under contract, and is subject to the Rights in Data-General Clause # +# 52.227-14, Alt. IV (DEC 2007). # +# # +# Copyright 2023 The MITRE Corporation. All Rights Reserved. # +############################################################################# + +############################################################################# +# Copyright 2023 The MITRE Corporation # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); # +# you may not use this file except in compliance with the License. # +# You may obtain a copy of the License at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, # +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +############################################################################# + +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" diff --git a/python/ClipDetection/setup.cfg b/python/ClipDetection/setup.cfg new file mode 100644 index 00000000..32fe1564 --- /dev/null +++ b/python/ClipDetection/setup.cfg @@ -0,0 +1,44 @@ +############################################################################# +# NOTICE # +# # +# This software (or technical data) was produced for the U.S. Government # +# under contract, and is subject to the Rights in Data-General Clause # +# 52.227-14, Alt. IV (DEC 2007). # +# # +# Copyright 2023 The MITRE Corporation. All Rights Reserved. # +############################################################################# + +############################################################################# +# Copyright 2023 The MITRE Corporation # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); # +# you may not use this file except in compliance with the License. # +# You may obtain a copy of the License at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, # +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +############################################################################# + +[metadata] +name = ClipDetection +version = 7.2 + +[options] +packages = clip_component +install_requires = + mpf_component_api>=7.2 + mpf_component_util>=7.2 + tritonclient[grpc] + +[options.entry_points] +mpf.exported_component = + component = clip_component.clip_component:ClipComponent + +[options.package_data] +clip_component = data/imagenet_classification_list.csv, data/coco_classification_list.csv, data/eighty_templates.txt, data/seven_templates.txt, data/one_template.txt + diff --git a/python/ClipDetection/tests/data/NOTICE b/python/ClipDetection/tests/data/NOTICE new file mode 100644 index 00000000..9e5de8ce --- /dev/null +++ b/python/ClipDetection/tests/data/NOTICE @@ -0,0 +1,14 @@ +# dog.jpg +# Public Domain + +# collie.jpg +# Public Domain + +# riot.jpg +# Public Domain + +# violence_classes.csv +# Custom created file for testing CLASSIFICATION_PATH + +# violence_templates.txt +# Custom created file for testing TEMPLATE_PATH \ No newline at end of file diff --git a/python/ClipDetection/tests/data/collie.jpg b/python/ClipDetection/tests/data/collie.jpg new file mode 100644 index 00000000..07479908 Binary files /dev/null and b/python/ClipDetection/tests/data/collie.jpg differ diff --git a/python/ClipDetection/tests/data/dog.jpg b/python/ClipDetection/tests/data/dog.jpg new file mode 100644 index 00000000..28ed320d Binary files /dev/null and b/python/ClipDetection/tests/data/dog.jpg differ diff --git a/python/ClipDetection/tests/data/riot.jpg b/python/ClipDetection/tests/data/riot.jpg new file mode 100644 index 00000000..39dcd210 Binary files /dev/null and b/python/ClipDetection/tests/data/riot.jpg differ diff --git a/python/ClipDetection/tests/data/violence_classes.csv b/python/ClipDetection/tests/data/violence_classes.csv new file mode 100644 index 00000000..b5adadaa --- /dev/null +++ b/python/ClipDetection/tests/data/violence_classes.csv @@ -0,0 +1,4 @@ +peaceful,peaceful scene +safe,safe scene +violent,violent scene +dangerous,dangerous scene \ No newline at end of file diff --git a/python/ClipDetection/tests/data/violence_templates.txt b/python/ClipDetection/tests/data/violence_templates.txt new file mode 100644 index 00000000..b3330db8 --- /dev/null +++ b/python/ClipDetection/tests/data/violence_templates.txt @@ -0,0 +1,3 @@ +a {} scene. +photo of a {} scene. +people in a {} environment. \ No newline at end of file diff --git a/python/ClipDetection/tests/test_clip.py b/python/ClipDetection/tests/test_clip.py new file mode 100644 index 00000000..f29a6bc8 --- /dev/null +++ b/python/ClipDetection/tests/test_clip.py @@ -0,0 +1,91 @@ +############################################################################# +# NOTICE # +# # +# This software (or technical data) was produced for the U.S. Government # +# under contract, and is subject to the Rights in Data-General Clause # +# 52.227-14, Alt. IV (DEC 2007). # +# # +# Copyright 2023 The MITRE Corporation. All Rights Reserved. # +############################################################################# + +############################################################################# +# Copyright 2023 The MITRE Corporation # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); # +# you may not use this file except in compliance with the License. # +# You may obtain a copy of the License at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, # +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +############################################################################# + +import sys +import os +import logging + +# Add clip_component to path. +sys.path.append(os.path.join(os.path.dirname(__file__), '..')) +from clip_component.clip_component import ClipComponent + +import unittest +import mpf_component_api as mpf + +logging.basicConfig(level=logging.DEBUG) + +class TestClip(unittest.TestCase): + + def test_image_file(self): + job = mpf.ImageJob( + job_name='test-image', + data_uri=self._get_test_file('dog.jpg'), + job_properties=dict( + NUMBER_OF_CLASSIFICATIONS = 3, + NUMBER_OF_TEMPLATES = 1, + CLASSIFICATION_LIST = 'coco', + ENABLE_CROPPING='False', + INCLUDE_FEATURES = 'True' + ), + media_properties={}, + feed_forward_location=None + ) + component = ClipComponent() + result = list(component.get_detections_from_image(job))[0] + + self.assertEqual(job.job_properties["NUMBER_OF_CLASSIFICATIONS"], len(self._output_to_list(result.detection_properties["CLASSIFICATION LIST"]))) + self.assertTrue("dog" in self._output_to_list(result.detection_properties["CLASSIFICATION LIST"])) + self.assertEqual("dog", result.detection_properties["CLASSIFICATION"]) + self.assertTrue(result.detection_properties["FEATURE"] is not None) + + def test_image_file_custom(self): + job = mpf.ImageJob( + job_name='test-image', + data_uri=self._get_test_file('riot.jpg'), + job_properties=dict( + NUMBER_OF_CLASSIFICATIONS = 4, + CLASSIFICATION_PATH = self._get_test_file("violence_classes.csv"), + TEMPLATE_PATH = self._get_test_file("violence_templates.txt") + ), + media_properties={}, + feed_forward_location=None + ) + result = list(ClipComponent().get_detections_from_image(job))[0] + self.assertEqual(job.job_properties["NUMBER_OF_CLASSIFICATIONS"], len(self._output_to_list(result.detection_properties["CLASSIFICATION LIST"]))) + self.assertTrue("violent scene" in self._output_to_list(result.detection_properties["CLASSIFICATION LIST"])) + self.assertEqual("violent scene", result.detection_properties["CLASSIFICATION"]) + + @staticmethod + def _get_test_file(filename): + return os.path.join(os.path.dirname(__file__), 'data', filename) + + @staticmethod + def _output_to_list(output): + return [elt.strip() for elt in output.split('; ')] + + +if __name__ == '__main__': + unittest.main(verbosity=2) \ No newline at end of file diff --git a/python/ClipDetection/tests/test_clip_triton.py b/python/ClipDetection/tests/test_clip_triton.py new file mode 100644 index 00000000..80113644 --- /dev/null +++ b/python/ClipDetection/tests/test_clip_triton.py @@ -0,0 +1,71 @@ +############################################################################# +# NOTICE # +# # +# This software (or technical data) was produced for the U.S. Government # +# under contract, and is subject to the Rights in Data-General Clause # +# 52.227-14, Alt. IV (DEC 2007). # +# # +# Copyright 2023 The MITRE Corporation. All Rights Reserved. # +############################################################################# + +############################################################################# +# Copyright 2023 The MITRE Corporation # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); # +# you may not use this file except in compliance with the License. # +# You may obtain a copy of the License at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, # +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +############################################################################# + +import sys +import os +import logging + +# Add clip_component to path. +sys.path.append(os.path.join(os.path.dirname(__file__), '..')) +from clip_component.clip_component import ClipComponent + +import unittest +import mpf_component_api as mpf + +logging.basicConfig(level=logging.DEBUG) + +class TestClip(unittest.TestCase): + + def test_image_file(self): + job = mpf.ImageJob( + job_name='test-image', + data_uri=self._get_test_file('collie.jpg'), + job_properties=dict( + NUMBER_OF_CLASSIFICATIONS = 10, + NUMBER_OF_TEMPLATES = 80, + CLASSIFICATION_LIST = 'imagenet', + ENABLE_CROPPING='False', + ENABLE_TRITON='True', + TRITON_SERVER='clip-detection-server:8001' + ), + media_properties={}, + feed_forward_location=None + ) + result = list(ClipComponent().get_detections_from_image(job))[0] + self.assertTrue("collie" in self._output_to_list(result.detection_properties["CLASSIFICATION LIST"]) or "Border collie" in self._output_to_list(result.detection_properties["CLASSIFICATION LIST"])) + + + @staticmethod + def _get_test_file(filename): + return os.path.join(os.path.dirname(__file__), 'data', filename) + + @staticmethod + def _output_to_list(output): + return [elt.strip() for elt in output.split('; ')] + + +if __name__ == '__main__': + unittest.main(verbosity=2) \ No newline at end of file diff --git a/python/ClipDetection/triton_server/Dockerfile b/python/ClipDetection/triton_server/Dockerfile new file mode 100644 index 00000000..909eff3f --- /dev/null +++ b/python/ClipDetection/triton_server/Dockerfile @@ -0,0 +1,47 @@ +# syntax=docker/dockerfile:experimental + +############################################################################# +# NOTICE # +# # +# This software (or technical data) was produced for the U.S. Government # +# under contract, and is subject to the Rights in Data-General Clause # +# 52.227-14, Alt. IV (DEC 2007). # +# # +# Copyright 2023 The MITRE Corporation. All Rights Reserved. # +############################################################################# + +############################################################################# +# Copyright 2023 The MITRE Corporation # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); # +# you may not use this file except in compliance with the License. # +# You may obtain a copy of the License at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, # +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +############################################################################# + +ARG MODELS_REGISTRY=openmpf/ + +FROM ${MODELS_REGISTRY}openmpf_clip_detection_models:7.2.0 as models + +FROM nvcr.io/nvidia/tritonserver:22.04-py3 as openmpf_triton_server + +COPY --from=models /models/model.pt /models/ip_clip_512/1/model.pt +COPY models/ip_clip_512.config.pbtxt /models/ip_clip_512/config.pbtxt + +RUN apt-get update; \ + apt-get -y upgrade; \ + rm -rf /var/lib/apt/lists/* + +LABEL org.label-schema.license="Apache 2.0" \ + org.label-schema.name="OpenMPF CLIP Detection Triton Server" \ + org.label-schema.schema-version="1.0" \ + org.label-schema.url="https://openmpf.github.io" \ + org.label-schema.vcs-url="https://github.com/openmpf/openmpf-components" \ + org.label-schema.vendor="MITRE" \ No newline at end of file diff --git a/python/ClipDetection/triton_server/models/ip_clip_512.config.pbtxt b/python/ClipDetection/triton_server/models/ip_clip_512.config.pbtxt new file mode 100644 index 00000000..bdc0d103 --- /dev/null +++ b/python/ClipDetection/triton_server/models/ip_clip_512.config.pbtxt @@ -0,0 +1,27 @@ +name: "ip_clip_512" +backend: "pytorch" +max_batch_size: 2048 +input [ + { + name: "image_input" + data_type: TYPE_FP32 + dims: [3, 224, 224] + } +] +output [ + { + name: "feature_vector__0" + data_type: TYPE_FP32 + dims: [512] + } +] +parameters [ + { + key: "INFERENCE_MODE" + value: {string_value: "true"} + }, + { + key: "ENABLE_NVFUSER" + value: {string_value: "true"} + } +] \ No newline at end of file