diff --git a/.azure-pipelines/azure-pipelines-demo.yml b/.azure-pipelines/azure-pipelines-demo.yml new file mode 100644 index 00000000000..b6eb669b2f6 --- /dev/null +++ b/.azure-pipelines/azure-pipelines-demo.yml @@ -0,0 +1,51 @@ +# Starter pipeline +# Start with a minimal pipeline that you can customize to build and deploy your code. +# Add steps that build, run tests, deploy, and more: +# https://aka.ms/yaml + +trigger: none +pr: +- wenxin-test + +pool: docker-agent + +variables: + IMAGE_NAME: 'neural-compressor' + IMAGE_TAG: 'py38' + +stages: +- stage: Build + displayName: Build and push stage + jobs: + - job: Build + displayName: Build + steps: + - script: | + echo ${BUILD_SOURCESDIRECTORY} + sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + displayName: 'Clean workspace' + + - checkout: self + displayName: "Checkout out Repo" + + - script: | + if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then + docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + fi + docker images | grep -i ${IMAGE_NAME} + if [[ $? -ne 0 ]]; then + echo "NO Such Image ${IMAGE_NAME}" + exit 1 + fi + displayName: "Build Devel Images" + + - script: | + docker stop $(docker ps -aq) + docker rm -vf $(docker ps -aq) || true + displayName: 'Clean Docker' + + - script: | + docker run --disable-content-trust --privileged --name="pr" --hostname="pr-host" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd /neural_compressor && python -m pip install --no-cache-dir -r requirements.txt && python setup.py sdist bdist_wheel && pip install dist/neural_compressor-*.whl && pip list && pip install intel-tensorflow && python test/strategy/test_bayesian.py" + displayName: 'Build Binary and Run UT' + + diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml new file mode 100644 index 00000000000..4cb89626c89 --- /dev/null +++ b/.azure-pipelines/code-scan.yml @@ -0,0 +1,210 @@ +trigger: none +pr: + - master +pool: suyue-test + +variables: + IMAGE_NAME: "code-scan" + IMAGE_TAG: "1.0" + CODE_SCAN_PATH: ".azure-pipelines/scripts/codeScan" + CODE_SCAN_LOG_PATH: ".azure-pipelines/scripts/codeScan/scanLog" + CURRENT_PATH: $(Build.SourcesDirectory) + CURRENT_STATUS: true + ER: "1212" + +stages: + - stage: BanditCodeScan + displayName: Bandit Code Scan + dependsOn: [] + jobs: + - job: Bandit + displayName: Bandit + condition: eq(variables['CURRENT_STATUS'], 'true') + steps: + - script: | + echo ${BUILD_SOURCESDIRECTORY} + sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + echo y | docker system prune + displayName: "Clean workspace" + - checkout: self + displayName: "Checkout out Repo" + - script: | + if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then + docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + fi + docker images | grep -i ${IMAGE_NAME} + if [[ $? -ne 0 ]]; then + echo "NO Such Image ${IMAGE_NAME}" + exit 1 + fi + displayName: "Build Devel Images" + - script: | + docker stop $(docker ps -aq) + docker rm -vf $(docker ps -aq) || true + displayName: "Clean Docker" + + - task: Bash@3 + inputs: + targetType: "inline" + script: | + docker run --disable-content-trust --privileged --name="bandit" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} bash /neural-compressor/$(CODE_SCAN_PATH)/bandit/bandit.sh $(CURRENT_STATUS) + echo "------$(CURRENT_STATUS)-------" + echo $(CURRENT_STATUS) + + echo "---$(ER)----$(QR)------" + - task: PublishPipelineArtifact@1 + inputs: + targetPath: $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-bandit.log + artifact: bandit + publishLocation: "pipeline" + + # - script: | + # if [ $(CURRENT_STATUS) -ne 0 ] ; then + # exit 1 + # fi + # exit 0 + + + + # - stage: PylintCodeScan + # displayName: Pylint Code Scan + # dependsOn: [] + # jobs: + # - job: Pylint + # displayName: Pylint + # steps: + # - script: | + # if [ $(CURRENT_STATUS) -ne 0 ] ; then + # exit 1 + # fi + # exit 0 + # - script: | + # echo ${BUILD_SOURCESDIRECTORY} + # sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + # echo y | docker system prune + # displayName: "Clean workspace" + # - checkout: self + # displayName: "Checkout out Repo" + # - script: | + # if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then + # docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + # fi + # docker images | grep -i ${IMAGE_NAME} + # if [[ $? -ne 0 ]]; then + # echo "NO Such Image ${IMAGE_NAME}" + # exit 1 + # fi + # displayName: "Build Devel Images" + # - script: | + # docker stop $(docker ps -aq) + # docker rm -vf $(docker ps -aq) || true + # displayName: "Clean Docker" + + # - task: Bash@3 + # inputs: + # targetType: "inline" + # script: | + # docker run --disable-content-trust --privileged --name="pylint" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} bash /neural-compressor/$(CODE_SCAN_PATH)/pylint/pylint.sh + + # - task: PublishPipelineArtifact@1 + # inputs: + # targetPath: $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-pylint.json + # artifact: pylint + # publishLocation: "pipeline" + + # - script: | + # if [ $(CURRENT_STATUS) -ne 0 ] ; then + # exit 1 + # fi + # exit 0 + + # - stage: PyspellingCodeScan + # displayName: Pyspelling Code Scan + # dependsOn: [] + # jobs: + # - job: Pyspelling + # displayName: Pyspelling + # steps: + # - script: | + # if [ $(CURRENT_STATUS) -ne 0 ] ; then + # exit 1 + # fi + # exit 0 + # - script: | + # echo ${BUILD_SOURCESDIRECTORY} + # sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + # echo y | docker system prune + # displayName: "Clean workspace" + # - checkout: self + # displayName: "Checkout out Repo" + # - script: | + # if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then + # docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + # fi + # docker images | grep -i ${IMAGE_NAME} + # if [[ $? -ne 0 ]]; then + # echo "NO Such Image ${IMAGE_NAME}" + # exit 1 + # fi + # displayName: "Build Devel Images" + # - script: | + # docker stop $(docker ps -aq) + # docker rm -vf $(docker ps -aq) || true + # displayName: "Clean Docker" + + # - task: Bash@3 + # inputs: + # targetType: "inline" + # script: | + # docker run --disable-content-trust --privileged --name="pyspelling" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} bash /neural-compressor/$(CODE_SCAN_PATH)/pyspelling/pyspelling.sh + + # - task: PublishPipelineArtifact@1 + # inputs: + # targetPath: $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/lpot_pyspelling.log + # artifact: pyspelling + # publishLocation: "pipeline" + + # - stage: CopyRight + # displayName: CopyRight Code Scan + # dependsOn: [] + # jobs: + # - job: CopyRight + # displayName: CopyRight + # steps: + # - task: Bash@3 + # inputs: + # targetType: "inline" + # script: | + # set -ex + # supported_extensions=(py, sh, yaml) + + # git --no-pager diff --name-only $(git show-ref -s remotes/origin/$(System.PullRequest.TargetBranch)) $(CURRENT_PATH)/neural_compressor > $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/diff.log + # files=$(cat $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/diff.log | awk '!a[$0]++') + + # for file in ${files} + # do + # if [[ "${supported_extensions[@]}" =~ "${file##*.}" ]]; then + # echo "Checking license in 1 ${file}" + # if [ $(grep -E -c "Copyright \\(c\\) ([0-9]{4})(-[0-9]{4})? Intel Corporation" ${file}) = 0 ]; then + # echo ${file} >> $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log + # fi + # else + # echo "Skipping ${file}" + # fi + # done + + # - task: PublishPipelineArtifact@1 + # inputs: + # targetPath: $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log + # artifact: copyright + # publishLocation: "pipeline" + + # - task: Bash@3 + # inputs: + # targetType: "inline" + # script: | + # ls $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log + # if [ $? -eq 0 ] ; then + # echo "------------------Check for wrong file list !!!!!!!!!!!!!!!!!!!!!!!"; exit 1 + # fi + # exit 0 \ No newline at end of file diff --git a/.azure-pipelines/docker/Dockerfile.devel b/.azure-pipelines/docker/Dockerfile.devel new file mode 100644 index 00000000000..5e9db3f57b7 --- /dev/null +++ b/.azure-pipelines/docker/Dockerfile.devel @@ -0,0 +1,43 @@ +# +# Copyright (c) 2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +ARG UBUNTU_VER=20.04 +FROM ubuntu:${UBUNTU_VER} as devel + +# See http://bugs.python.org/issue19846 +ENV LANG C.UTF-8 + +RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \ + python3 \ + python3-pip \ + python3-dev \ + python3-distutils \ + autoconf \ + build-essential \ + git \ + libgl1-mesa-glx \ + libglib2.0-0 \ + numactl \ + time \ + wget + +RUN ln -sf $(which python3) /usr/bin/python + +RUN python -m pip --no-cache-dir install --upgrade pip +RUN python -m pip install --no-cache-dir setuptools + +RUN pip list + +WORKDIR / + diff --git a/.azure-pipelines/docker/DockerfileWithNC.devel b/.azure-pipelines/docker/DockerfileWithNC.devel new file mode 100644 index 00000000000..113fdead9b9 --- /dev/null +++ b/.azure-pipelines/docker/DockerfileWithNC.devel @@ -0,0 +1,53 @@ +# +# Copyright (c) 2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG UBUNTU_VER=20.04 +FROM ubuntu:${UBUNTU_VER} as devel + +# See http://bugs.python.org/issue19846 +ENV LANG C.UTF-8 + +ARG REPO=x +ARG TARGET_BRANCH=y +ARG SOURCE_BRANCH=z +ARG DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \ + python3 \ + python3-pip \ + python3-dev \ + python3-distutils \ + autoconf \ + build-essential \ + git \ + libgl1-mesa-glx \ + libglib2.0-0 + +RUN ln -sf $(which python3) /usr/bin/python + +RUN python -m pip --no-cache-dir install --upgrade pip + +RUN git clone --single-branch --branch=${SOURCE_BRANCH} ${REPO} neural-compressor && \ + cd neural-compressor && \ + python -m pip install --no-cache-dir setuptools && \ + python -m pip install --no-cache-dir pycocotools && \ + python -m pip install --no-cache-dir -r requirements.txt && \ + python setup.py install + +RUN pip list +RUN pip list | grep neural + +WORKDIR /neural-compressor +RUN git rev-parse HEAD \ No newline at end of file diff --git a/.azure-pipelines/scripts/codeScan/bandit/bandit.sh b/.azure-pipelines/scripts/codeScan/bandit/bandit.sh new file mode 100644 index 00000000000..e8f868a6f3f --- /dev/null +++ b/.azure-pipelines/scripts/codeScan/bandit/bandit.sh @@ -0,0 +1,16 @@ +# set -ex +bandit_log_dir="/neural-compressor/.azure-pipelines/scripts/codeScan/scanLog" +pip install bandit + +python -m bandit -r -lll -iii /neural-compressor/neural_compressor > $bandit_log_dir/lpot-bandit.log + +exit_code=$? +if [ ${exit_code} -eq 0 ] ; then + sed -i "s|CURRENT_STATUS\: true|CURRENT_STATUS\: false|g" /neural-compressor/azure-pipelines.yml + sed -i "s|ER|QR|g" /neural-compressor/azure-pipelines.yml + + # sed -i 's/CURRENT_STATUS:.*$/CURRENT_STATUS: false' /neural-compressor/azure-pipelines.yml + echo "Bandit exited with non-zero exit code."; exit 1 +fi +exit 0 + diff --git a/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh b/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh new file mode 100644 index 00000000000..6187b3ebeb0 --- /dev/null +++ b/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh @@ -0,0 +1,31 @@ + +supported_extensions=(py, sh, yaml) + +set -xe +git config --global --add safe.directory /neural_compressor +git --no-pager diff --name-only --no-index $(git show-ref -s remotes/origin/master) /neural_compressor/neural_compressor > /diff.log +# git --no-pager diff --name-only $(git show-ref -s remotes/origin$2) ./$1 > $3/copyright/diff.log + +files=$(cat /diff.log | awk '!a[$0]++') + +for file in ${files} + +do + if [[ "${supported_extensions[@]}" =~ "${file##*.}" ]]; then + echo "Checking license in 1 ${file}" + if [ $(grep -E -c "Copyright \\(c\\) ([0-9]{4})(-[0-9]{4})? Intel Corporation" ${file}) = 0 ]; then + echo ${file} >> /copyright_issue_summary.log + cat /copyright_issue_summary.log + fi + else + echo "Skipping ${file}" + fi +done + + +ls /copyright_issue_summary.log +exit_code=$? +if [ ${exit_code} -e 0 ] ; then + echo "------------------Check for wrong file list !!!!!!!!!!!!!!!!!!!!!!!"; exit 1 +fi +exit 0 \ No newline at end of file diff --git a/.azure-pipelines/scripts/codeScan/pylint/pylint.sh b/.azure-pipelines/scripts/codeScan/pylint/pylint.sh new file mode 100644 index 00000000000..cef8620dae5 --- /dev/null +++ b/.azure-pipelines/scripts/codeScan/pylint/pylint.sh @@ -0,0 +1,25 @@ +set -ex +pylint_log_dir="/neural-compressor/.azure-pipelines/scripts/codeScan/scanLog" + +pip install pylint==2.12.1 +pip install flask==2.1.3 +pip install google +pip install horovod +pip install ofa +pip install fvcore +pip install autograd +pip install pymoo +pip install tf_slim +pip install transformers +pip install -r /neural-compressor/requirements.txt +pip install onnxruntime_extensions + + + +python -m pylint -f json --disable=R,C,W,E1129 --enable=line-too-long --max-line-length=120 --extension-pkg-whitelist=numpy --ignored-classes=TensorProto,NodeProto --ignored-modules=tensorflow,torch,torch.quantization,torch.tensor,torchvision,mxnet,onnx,onnxruntime /neural-compressor/neural_compressor > $pylint_log_dir/lpot-pylint.json + +exit_code=$? +if [ ${exit_code} -ne 0 ] ; then + echo "PyLint exited with non-zero exit code."; exit 1 +fi +exit 0 \ No newline at end of file diff --git a/.azure-pipelines/scripts/codeScan/pyspelling/lpot_dict.txt b/.azure-pipelines/scripts/codeScan/pyspelling/lpot_dict.txt new file mode 100644 index 00000000000..25bf61a1cef --- /dev/null +++ b/.azure-pipelines/scripts/codeScan/pyspelling/lpot_dict.txt @@ -0,0 +1,2202 @@ +aa +aac +abc +Abc +AbcAdaptor +AbcTuneStrategy +abi +ABI +absl +abspath +abstractive +acc +Acc +accuracies +acdc +ACDC +Acknowledgement +activations +Adadelta +adam +AdamW +adaptor +Adaptor +AddEmbeddings +AddN +addr +ADDR +AddV +ade +ADE +adresses +AdvProp +ae +aea +af +AGS +ai +ailab +al +albert +alexnet +AlexNet +algo +algos +alibaba +Alibaba +AlignImageChannel +allenai +alloc +ALLREDUCE +alsologtostderr +Aman +amazonaws +amazonlinux +Amodei +AmpConf +AMX +analytics +Analytics +Anastasiia +AnchorGenerator +andravin +andreamad +anisotropic +anno +anton +ap +apache +api +APIs +APl +APm +approch +APs +arg +argmax +ArgMax +args +arxiv +arXiv +asd +astype +asym +async +atrous +att +AttentionReshape +attr +attredirects +AttributeProto +attrs +auc +aug +autgrad +autogenerate +autograd +AutoMixPrecision +autopep +Autoregressive +ava +AverageMeter +avgloss +AvgPool +avx +AVX +backend +backends +backticks +bart +barthez +bashrc +basicConfig +BasicTokenizer +batchgenerators +BatchMatMul +BatchMatMulV +batchnorm +BatchNorm +bayesian +BayesianOptimization +bazel +bbbb +bbox +bboxes +bccf +bce +bd +bdb +bdist +benchmarked +benchmarking +Benchmarking +Bengio +Benoît +berkeleyvision +bert +BERT's +BertAdam +BertConfig +BERTDataSet +BertForMaskedLM +BertForNextSentencePrediction +BertForPreTraining +BertForQuestionAnswering +BertForSequenceClassification +BertForTokenClassification +BertModel +berts +bertsquad +BertTokenizer +bfloat +BFP +BGR +Bianchi +BiasAdd +BibTeX +bicubic +bilibili +BiliBili +bilinear +BilinearImagenet +billsum +BiLSTM +binarize +binarized +BinaryAdd +biomedical +Biomedical +BLAS +blendcnn +BlendCnn +BlendCNN +bleu +BLEU +blocktime +blogpost +bn +bninception +BNInception +bobw +booktitle +bool +BoW +boxlist +BoxList +br +BrainTumour +BraTS +broadcasted +bs +bsnone +bugfix +buildin +builtin +Builtin +BV +bvlcalexnet +bzip +cadene +Cadene +caffe +Caffe +caffenet +Caffenet +cafferesnet +CaffeResnet +CaffeResNet +Caiming +calib +calibrationcsv +camembert +CamemBERT +canada +Carbonell +CascadeFullRes +cbica +cd +cdn +ce +cec +CenterCrop +centernet +centerNet +centos +CentOS +Centre +cern +CERN's +certfile +Cesa +cfa +cffi +cfg +CFLAGS +ChamNet +Chaumond +checkbox +checkboxes +Cheng +chmod +Cho's +chongruo +Chongruo +chris +Chuanqi +ci +cifar +CIFAR +circleci +Cistac +cityscapes +Cityscapes +cityscapesscripts +cityscapesScripts +cknowledge +ckpt +ckpts +ClassPredictionTower +clcarwin +Clergerie +cli +CLI +clipnorm +clm +CLM +cls +CLX +cly +cmake +CMake +CMAKE +CMakeLists +cmd +CMU +cn +cnn +cnt +CoCo +cocoapi +cocoApi +cocodataset +COCODataset +COCODemo +COCOEval +COCOmAP +COCOmAPv +COCONpy +cocoraw +COCORaw +COCORecord +codalab +codecogs +codenamed +CoLA +colorama +ColorJitter +colspan +compat +compilervars +concat +ConcatV +cond +conda +CONDA +condconv +CondConv +Condensenet +conf +config +Config +configs +CoNLL +Conneau +const +ConstantOfShape +ConstDataLoader +constexpr +contaning +conv +Conv +ConvBNReLU +ConvertingSSDMobilenetToONNX +convertion +ConvNets +convolutional +Convolutional +ConvPerStage +ConvReLU +copt +coreml +CoreML +cp +cpp +cpu +cpus +CPUs +CPX +cpython +creafz +creatis +creativecommons +criteo +Criteo +CriteoTerabyte +croping +CropResize +CropToBoundingBox +CrossEntropyLoss +crossvalidaton +crt +csv +ctrl +CTRLModel +CTRLTokenizer +ctuning +ctx +cuda +cudaPopCallConfiguration +cudatoolkit +CUDAToolKit +cudnn +CUHK +curr +Curran +custormer +cv +CvAClvFfyA +CXX +cxxopt +cypw +cython +da +dae +DagnyT +Dai +dailymail +Danqi +darknet +Darknet +datadir +datafile +dataloader +dataLoader +DataLoader +DataLoadermodule +dataloaders +DataParallel +datapoints +DataProcessor +dataset +Dataset +DatasetAnalyzer +datasets +datatype +datatypes +dathath +Dathathri +datset +dbmdz +DBMDZ +dbox +dbs +DCMAKE +dcn +ddp +DDP +DDR +de +deberta +DecodeImage +deepengine +deeplab +DeepLab +deeplabv +DeepLabV +DeepLearningExamples +Delangue +DENABLE +denseblock +denselayer +densenet +DenseNet +deps +DepthwiseConv +dequant +dequantize +DequantizeLinear +DequantStub +DeQuantStub +desc +dest +destructor +detections +detectron +Detectron +dev +devel +Devlin +devtool +DFabiansResUNet +DFS +DGAN +dialogpt +DialoGPT +dicts +dir +dirname +Discrim +distil +Distil +distilbert +DistilBert +DistilBERT +DistilBERT +DistilBertModel +DistilBertTokenizer +distilgpt +DistilGPT +distillated +Distillated +distillating +DistilmBERT +distilrobert +distilroberta +DistilRoBERTa +DistributedDataParallel +DistributedOptimizer +DistributedSampler +distro +dividiti +Djamé +DKFZ +dl +dlabel +dlboost +dlrm +DLRM's +dmlc +DMQA +dNative +dnf +dnn +dnnl +DNNL +Dockerfile +doclist +docstrings +doctrings +docutils +doteq +dowmsampling +downloader +downsampled +downsampling +doxygen +dpn +DPNs +dpr +DropOut +ds +dscore +dst +dtype +DualPathNet +dualpathnetworks +DualPathNetworks +DummyDataLoader +dunet +DUNet +Dupont +Durand +dvdt +dw +dynamiccaly +ead +EAQkaohzrJbd +earlystop +eb +ecotrust +edgetpu +EdgeTPU +edu +ee +eer +ef +efficientnet +efficientNet +EfficientNet +EfficientNets +eg +eightbit +einstein +el +elif +eltwise +emb +embeddings +embs +EMC +enablerepo +EncodeJped +enfr +eng +ensembling +ensp +entrypoint +enum +env +eq +erf +Erf +Éric +eriklindernoren +Errno +esri +et +eval +evaluator +evel +exemplarily +exising +existing +exmaple +expanduser +ExperimentPlanner +ExperimentPlanners +extractive +EzjbRL +fabian +FabiansUNet +facebook +FaceBook +facebookresearch +fairseq +fallbacks +fanout +faq +Farhadi +FashionMNIST +FasterRCNN +FastFormers +fastrcnn +fatihcakirs +favourably +fb +fbgemm +FBNet +fbnetc +fbresnet +FBResNet +fc +fcn +FCN +fd +FeatureExtractor +feedbacks +Feng +ffc +filename +filenames +FileNotFoundError +filepath +filesystem +finbert +finetune +Finetune +finetuned +finetuning +flac +FlatMapDataset +flaubert +flavour +flavours +Flavours +floatfunctional +FloatFunctional +FloatTensor +FLOPs +Florian +fmfn +fmt +fmtstr +fn +fname +fns +foregound +fp +FP +fpic +fPIC +fpn +FPN +FRN +FromConfig +frontend +fstack +ftfy +Fu +fullres +func +functionalities +functionet +functools +Funtowicz +fw +FWK +fx +GameAI +GANs +Garnett +gcc +gclient +gd +geffnet +gelu +Gelu +GeluOperator +GenEfficientNet +GenericPreprocessor +german +germeval +GermEval +gestaltit +getitem +getsize +GetStrides +GFLOPs +gh +gid +Gimpel +Girshick +github +GitHub +githubusercontent +gitmodules +GLIBCXX +GLOG +GLUE +gluebenchmark +gluepy +gluon +Gluon +gluoncv +GluonCV +gluonnlp +gn +GN +goldsborough +goog +google +googleapis +googleblog +googlenet +googlesource +Goyal +gpg +GPG +gpt +gpu +gpus +GPUs +graphdef +GraphDef +GraphModule +GraphProto +Grauman +grpc +gtFile +gtFine +Gui +Guillaume +Guoming +gz +gzY +Haibin +haibinlin +Haihao +hangzhang +hardcoding +HasAns +hawq +HAWQ +HdQ +heatmaps +Hein +helloworld +HelloWorld +henson +hiddenlayer +hippocampus +Hippocampus +HistogramObserver +hlu +horovod +Horovod +HOROVOD +horovodrun +hostfile +Hounsfield +howpublished +HqEgzS +href +html +http +https +Hu +hubert +huggingface +HuggingFace +HuggingFace's +HuggingFacesTS +hujie +hvd +HybirdBlock +HybridBlock +hyperparameter +hyperparameters +icc +ICCV +Icelake +icpc +icx +ide +idx +ie +IEEE +ILSVR +ilsvrc +ILSVRC +Ilya +im +imagecocodataset +ImageFolder +ImageList +imagenet +ImageNet +ImagenetRaw +ImageRecord +ImageRecordIter +imagesTr +imagesTs +img +imgrec +imgs +imgx +IML +impl +ImportError +IMS +inceptionresnetv +InceptionResNetV +inceptionv +InceptionV +incollection +IndexType +indexValue +indices +indico +inferencer +informations +infos +init +InnerProduct +innersource +inp +inplace +inproceedings +inputcsv +InputData +InputExample +InputFile +Inria +insa +instanceonly +instantiation +integerops +intel +intelai +IntelAI +interoperability +introudces +ints +inturn +InvertedResidual +io +ios +iOS +iou +IoU +ipc +ipex +IPEX +ipynb +ipython +ir +irv +ISA +Isensee +isinstance +issuecomment +IssueQuery +IssueQueryThreads +iter +IteratorGetNext +iters +Jäger +jemalloc +Jens +Jie +jim +Jingfei +Jiong +jit +jitter +Joshi +jpeg +JPEGImages +jpg +jpwarren +json +jsons +Julien +JunWang +jupyter +kaggle +kaggleAdDisplayChallenge +kaiming +KaimingHe +Karthik +kcho +keepbs +keepdim +keras +Keskar +keyfile +keypoint +Keypoint +kimiyoung +kitti +kmp +KMP +KnowledgeDistillationLoss +kriz +kwargs +Kyunghyun +LabelShift +labelsTr +Lample +Lan +lang +LanguageModeling +Lapata +Larochelle +LastLayerShape +latencies +LaTeX +Lavin +layernorm +LayerNorm +layoutlm +ld +len +LessEqual +lf +lfaidata +lfs +li +libdeep +libengine +libffi +libGL +libglib +libiomp +libmlperf +librispeech +LibriSpeech +librosa +libsndfile +libstdc +libz +licence +liKE +Limitting +lin +linkopt +linoxide +linux +linuxfoundation +ListDataset +LiTS +Liu +Liu's +llvmlite +lm +LMHeadModel +ln +loadgen +LoadGen +LOADGEN +LoadGen's +LoadgenAPI +LoadgenAPITestSettings +LoadgenVersion +LoadImage +LOC +localdisk +localhost +LOCderiv +LOCpart +logdir +logfile +login +logits +LOGLEVEL +LogSettings +logtostderr +longformer +lossy +Louf +LowPrecisionInferenceTool +lowproposals +lowres +Lp +lpot +LPOT +LPOT's +lr +lS +LSVRC +lt +LTS +lua +Luan +lutzroeder +lyon +Lysandre +lzma +macOS +MACOSX +MAdds +Madotto +MagnitudePrunePolicy +Maier +mainpage +Makefile +MakefileGnProj +MakeIterator +Mandar +Manmatha +manylinux +mAp +mAP +Mapillary +marianmt +MaskPostProcessor +maskrcnn +MaskRCNN +MaskRCNNFPNFeatureExtractor +maskrnn +massa +Massa +matcher +matmul +MatMul +MatMulWithBias +MatMulWithBiasAdd +MatMulWithBiasGelu +MatMulWithBiasTanh +matplotlib +matricses +maxdepth +maxindrange +maxk +MaxPool +maxSizeInComplete +mbart +mBERT +mcc +McCann +mcordts +md +MeanSquaredError +measurer +Medcial +medicaldecathlon +meetup +mem +membind +mems +messi +metabuild +metadata +metamind +MICCAI +microsoft +miguelgrinberg +Mingda +minibatch +minilm +minimalistic +minival +minloglevel +minmax +MinMaxObserver +mins +mIoU +mIOU +Mirella +misalignments +miscs +Mish +missmatches +MixedConv +mixnet +MixNet +mixup +mkdir +mkl +MKL +mlap +mlas +MLAS +mlcommons +mll +mlm +mlp +mlpc +mlperf +MLperf +MLPerf +mlt +mmdetection +mmlab +MMLAB +mnasnet +MNASNet +mnist +MNIST +mnli +MNLI +mobilebert +MobileBERT +mobilenet +MobileNet +mobilenetv +Mobilenetv +MobileNetv +MobileNetV +modalities +Modalities +modality +ModelConversion +modelfeatures +modelforward +modelinput +modellogits +modelmean +modelsize +modelstd +ModuleDict +ModuleNotFoundError +Molino +mpi +mrcnn +mrpc +MRPC +MSD +mse +MSE +msvc +mul +mult +multi +Multi +multiclass +multilabel +multinli +MultiNLI +multiscale +MULTISCALE +MultiStream +MultiStream's +MultiStreamFree +mutli +mv +mx +mxnet +MxNet +MXNet +MyDataset +Mykhailo +Myle +MyMetric +myModel +MYTASK +MYTASKNAME +Naman +namedtuple +nanohanno +Narasimhan +NAS +nasnet +NASNet +nasnetalarge +nasnetamobile +nb +nbest +nbsp +nc +NCCL +nchw +NCHW +nd +ndarray +NDArray +nderlu +nepoch +ner +NER +nervanasystems +nesterov +NetEase +netron +Netron +networkbuilders +NeurIPS +neval +NewMetric +newstest +nextplatform +ng +ngatang +NGPUS +ngram +NHWC +NIC +nifti +niftis +nii +Nijmegen +Nitish +nl +NLG +nli +nll +nlp +NLP +nlpyang +nltk +NLU +nm +nms +nn +nnodes +nnu +nnU +nnunet +nnUnet +nnUNet +nnUNetPlansv +nnUNetTrainer +nnUNetTrainers +nnUNetTrainerV +NNZ +noduplicates +NoisyStudent +Nonlinearity +NonNestedTuple +NoNormalization +NonZero +noobj +np +nproc +npy +npz +nq +nrix +ns +nsdf +nSsKchNAySU +nthreads +ntrain +num +numactl +numba +numCompleteThreads +numerics +numpy +numTest +numTraining +NVAITC +nvcc +nvidia +NVIDIA +NVIDIA's +nvme +Nx +nyu +ok +ol +Omer +OMP +onboarding +oneapi +oneAPI +onednn +oneDNN +onlinedocs +onnx +ONNX +ONNXQuantizer +onnxrt +ONNXRT +onnxruntime +OnnxRuntime +oob +OOM +OOQtYMH +openai +OpenAI +OpenAI's +OpenAIAdam +OpenAIGPTModel +OpenAIGPTTokenizer +opencv +OpenCV +openmp +openslr +opensource +openssl +openvino +OpenVINO +openvinotoolkit +OpenWebTextCorpus +OperatorConfig +OPs +opset +opsetid +optim +optimizations +Optimizations +optimizers +Optimizers +optypewise +opwise +OrderedDict +ORGderiv +ORGpart +os +osJJ +OTH +OTHderiv +OTHpart +Ott +oup +outdir +OutputData +outputfile +ov +overfeat +overfit +overfitted +PaddingSequence +PaddingSequence +pageId +palletsprojects +panoptic +Panoptic +paperswithcode +param +params +Parinov +ParseDecodeImagenet +ParseDecodeVoc +participations +Parzen +pastebin +patientIDs +pb +pbar +pdf +Peason +pegasus +pelee +peleenet +PeleeNet +Penghui +Pengxin +pepy +PerChannelMinMaxObserver +PERderiv +perf +perftests +PERpart +phrasebank +phy +physcpubind +PhYUmn +Piero +Pierric +PIL +pixAcc +Piyush +pjreddie +pkill +pkl +pky +plm +PLM +pls +pnasnet +PNASNet +png +POC +polynet +PolyNet +Pooler +pos +postprocesing +postprocess +postprocessed +postprocessing +PostProcessor +PostTransform +PowerTools +pplm +PPLM +PQ +pre +prebuild +prebuilt +Prec +precisions +pred +preds +preformance +Preload +preprint +preprocess +preprocessed +preprocesses +preprocessing +preprocessor +Preprocessor +PreprocessorFor +Preprocessors +prerelease +PreSumm +pretrain +pretrained +pretrainedmodels +pretraining +prev +prioritizies +probs +proc +productizing +profilings +ProgressBar +proto +Protobuf +protoc +protractortest +PRs +PrunePolicy +pth +ptq +PTQ +ptr +pudae +pw +PWC +pwd +PWD +px +py +pybind +pycocotools +pyguide +pylint +pymodule +PyObject +pypi +PyPI +PySUT +pytest +PythonAPI +PYTHONPATH +pytorch +PyTorch +pytorchic +PyTorchKnowledgeDistillationLoss +pyyaml +PyYAML +PZ +qat +QAT +qconfig +QConfig +QiaoranC +qint +qlinear +QLinear +qlinearops +QnA +qnli +QNLI +qps +QPS +qqp +QQP +qscheme +qsl +QSL +qtcreator +qtype +quant +quantile +quantizable +Quantizable +quantization +Quantization +quantize +quantized +QuantizedConv +QuantizedConvReLU +QuantizedInput +quantizer +quantizes +Quantizes +quantizing +QuantStub +QueryBackendCapability +QuerySampleComplete +QuerySampleLibrary +quickstart +Quickstart +QuickStart +Quoc +R'emi +Radboud +Radford +Radu +rAjHyXhTzz +rajpurkar +ramdisk +RandAug +RandAugment +randn +RandomCrop +RandomHorizontalFlip +RandomResizedCrop +RandomVerticalFlip +Rault +rc +rcnn +readme +README +ReadmeBuild +ReadmeFAQ +ReadmeHtmlDocs +ReadmeTests +readthedocs +realtime +Realtime +rebase +recommonmark +RecordingObserver +recordio +RecordIO +recurse +Redmon +ReduceMean +regex +RegNet +rehm +Rehm +reinstall +relase +relu +Relu +ReLU +repo +repo's +repo’s +repos +representating +requantize +resampled +resampling +rescale +Rescale +ResencUNet +resize +Resize +ResizeCropImagenet +resized +Resizes +ResizeWithRatio +resnest +ResNest +ResNeSt +resnet +Resnet +ResNet +resnetv +ResNetV +resnext +ResNext +ResNeXt +ressource +ressources +reStructuredText +ret +RetinaMask +retinanet +retinaNet +RetinaNet +reusability +Rewon +rf +rfcn +rgb +RGB +rmax +rmin +RMSE +rn +rng +RNN +rnnt +ro +roberta +RoBERTa +RobertaModel +RobertaTokenizer +ROC +RocStories +Romary +rosanneliu +rougeL +rougeLsum +rowanz +rowspan +RPN +RPNHead +RPNPostProcessor +Rsqrt +rst +rtd +RTX +runhooks +runtime +Runtime +RuntimeError +Rusia +rusiaaman +Ruslan +rw +rwightman +sacremoses +Sagot +Salakhutdinov +salesforce +Salesforce +Salimans +sanh +Sanh +sata +SavedModel +SavedModel +Scalable +scaler +scatterFillKernel +sched +scikit +scm +screenshots +ScriptModule +se +sed +Seddah +seg +segm +SegmentationMask +segmentations +seid +senet +SENet +sentencepiece +Sep +SEP +SeqDataCollator +serializable +ServerPool +sess +setuptools +sexualized +SGD +sgmoid +SHA +sharded +Sharma +Shen +Shirish +shouldn +showEvent +shufflenet +Shufflenet +ShuffleNet +shufflenetv +Shvets +sigmoid +signup +sigopt +Sigopt +SigOpt +SingleStream +skx +Skylake +skylion +SMBO +SMBOs +Smola +smoothes +sndfile +Socher +socio +SocketIO +softmax +somain +Soricut +sota +SOTA +sox +SoX +spacings +spacy +SpaCy +SparseCategoricalAccuracy +SparseCategoricalCrossentropy +sparsified +Spearman +spearmanr +specificities +splitted +spm +spnasnet +sqlalchemy +Sqrt +sqSiUy +Squad +SQuAD +SquadF +squadpy +SquadV +SquaredDifference +squeezebert +squeezenet +SqueezeNet +src +SrcTuple +sryqufw +ssd +SSDMobilenet +SSDSC +sshleifer +sst +stackoverflow +Standley +startswith +StartTest +stdout +stds +stefan +stemblock +stepsize +Stoyanov +str +strided +struct +sts +STS +stsb +styleguide +Suárez +subexpression +subfolder +subfolders +Subgraph +submodule +submodules +Submodules +subsample +subtoken +sudo +Sumanth +summarization +Summarization +SummaryWriter +superseeds +suported +sut +SUT +Sutskever +sv +svg +swagaf +sym +symlink +symlinked +symlinks +Symlinks +synset +sys +SystemUnderTest +tanh +TaskXX +TaskXXX +tb +TBD +tbe +tbody +td +techdecoded +tencent +tensor's +tensorboard +tensorBoard +TensorBoard +tensorcore +TensorDataset +tensorflow +TensorFlow +TensorflowQuery +tensorImageSize +TensorInfo +TensorProto +teraoperations +tesla +testability +TestSettings +tf +TF +TFBertForSequenceClassification +tflite +tfp +tfrecord +TFRecord +TFRecordDataset +tfrecords +TFRobertaModel +TFSlimNetsFactory +TFSlimNetsFactory's +tg +tgt +tgz +th +THCudaTensor +thead +thepath +thres +thrs +Tian +Tidx +timeline +timestamps +TinyBERT +tl +tlkh +tLoss +TLS +tmp +tmpfs +ToArray +ToBGR +toc +toctree +TODO +tokenization +tokenize +tokenized +tokenizer +Tokenizer +tokenizers +Tokenizers +tokenizing +tol +TOL +tolist +toml +ToNDArray +toolchains +ToPILImage +topk +TopK +topologies +ToRange +torchaudio +torchscript +TorchScript +torchtext +torchvision +TorchVision +toronto +totalizing +ToTensor +Toutanova +tp +tpe +TPE +tpu +TPU +tqdm +traceback +trainings +trainval +trainvaltest +transfo +TransformImage +TransfoXLModel +TransfoXLTokenizer +travis +trigram +tstandley +tsv +TuneStrategy +tunings +tuningusage +tuple +tuples +txt +TZ +uber +ubuntu +ubyte +UI +UID +uint +uk +un +uncomment +uncompress +unet +Unet +UNet +unidecode +uniq +unittest +unref +unsqueeze +unstack +upenn +uploader +upscaled +Upscaled +upstreamed +url +userspace +usp +usr +UTC +util +utils +ux +UX +valminusminival +valset +ValueError +Varshney +VCVTNE +VCVTNEPS +VDPBF +vec +Veronika +veronikayurchuk +versioned +Veselin +vgg +viewpage +Villemonte +ViT +voc +VOC +VOCdevkit +VOCmAP +VOCMApMetrics +VOCRecord +voxel +voxels +vram +VRAM +VTune +waleedka +Wallach +wangg +warmup +wav +wd +webcam +Webcam +webite +webpage +WebSockets +WebText +wedam +WeightSharedConvolutionalBoxPredictor +Wformat +wget +whitelist +whl +WideResNet +WideResNet +Wightman +wikipedia +wikitext +WikiText +WilsonCity +WIP +WLYDCRB +wmt +wnd +WnD +wnli +Wnxu +WordPiece +workdir +workflow +Workflow +workflows +workspace +wrt +wwm +www +xad +xception +Xception +xcode +xeon +Xeon +Xiang +Xiong +xl +XLA +xlm +XLMModel +XLMTokenizer +xlnet +XLNet +XLNetModel +XLNetTokenizer +XlUH +xml +xnli +XNLI +xsum +xV +xvf +xvzf +XXXX +xxy +xxz +xYNrZdEAnrHk +xywh +xyxy +xz +xzvf +yacs +yaml +yamls +Yi +Yiming +Yinhan +yizhu +yjxiong +YKd +Yoann +yolo +yolov +YOLOv +YOLOV +yosinski +Yosinski +YqgzY +Yuanjun +Yue +Yunpeng +Yurchuk +YY +zenodo +Zettlemoyer +zfnet +ZFNet +zh +zhang +Zhang +zhanghang +Zhenzhong +Zhi +Zhilin +Zhongyue +zhongyuezhang +Zhu +Zihang +zihangdai +znoexecstack +znow +Zptls +zrelro +zrl +zxvf +CustomObj +ModelSize +QDQ +QLinearOps +qdq +qdqops +CodeGenerator +GEMM +SparseLib +Xbyak +brgemm +cfgs +gtests +hpp +hypotype +kd +ker +kern +sparsednn +spmm +xxxx +GraphModules +wsl +descs +gtest +IOMP +MALLOC +PRETAINED +SPR +libjemalloc +preload +thp +GCP +gcp +gif +solutionslibrary +geomean +VNNI +Preloading +DUC +duc +leftImg +roc +sklearn +CLA +cla +whitehat +codeofconduct +CYP +SBSTD +xd +samsum +IntelCaffe +baremetal +HWs +IC +KH +NVidia +OC +bolded +sparsification +tensorrt +hardwares +BenchmarkConf +PruningConf +DistillationConf +grey +ModelZoo +mzbert +CaffeNet +FlauBERT +GoogleNet +SqueezeBERT +iz +lvwerra +mBart +oje +za +zk +QIntegerops +QLinearops +criterions +HuBERT +csarron +gpb +howey +huawei +noah +nreimers +pruneofa +textattack +scheduler's +BiDAF +bidaf +FERPlus +ferplus +MixedPrecision +DUnetCNN +calibrationset +ndhwc +ArcFace +arcface +arcfaceresnet +nfolds +RFB +WIDERFACE +shuoyang +ultraface +XKeyboard +lscpu +qpa +vnni +xcb +DevCloud +PyPi +aidevcloud +awk +clx +devcloud +lAtr +nda +ppn +qstat +qsub +qsvr +ruserok +scp +spr +stderr +uXXXXX +QuantConf +SuperBench +autocast +kai +mailto +superbench +yao +Lecun +NLPToolkit +Yan +exdb +lecun +publis +yann +abcadf +bcb +INTRA +WARMUPS +ende +intra +inteltensorflow \ No newline at end of file diff --git a/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh b/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh new file mode 100644 index 00000000000..73b431ed8e4 --- /dev/null +++ b/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh @@ -0,0 +1,23 @@ +set -ex +pyspelling_dir="/neural-compressor/.azure-pipelines/scripts/codeScan" +pyspelling_log_dir="/neural-compressor/.azure-pipelines/scripts/codeScan/scanLog" + +pip install pyspelling +pip install -r /neural-compressor/requirements.txt + +apt-get install aspell -y +apt-get install aspell-en -y + + + +sed -i "s|\${VAL_REPO}|$pyspelling_dir|g" $pyspelling_dir/pyspelling/pyspelling_conf.yaml +sed -i "s|\${LPOT_REPO}|/neural-compressor|g" $pyspelling_dir/pyspelling/pyspelling_conf.yaml + +pyspelling -c $pyspelling_dir/pyspelling/pyspelling_conf.yaml > $pyspelling_log_dir/lpot_pyspelling.log +exit_code=$? +if [ ${exit_code} -ne 0 ] ; then + echo "Pyspelling exited with non-zero exit code."; exit 1 +fi +exit 0 + + diff --git a/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling_conf.yaml b/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling_conf.yaml new file mode 100644 index 00000000000..c407f846ea2 --- /dev/null +++ b/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling_conf.yaml @@ -0,0 +1,11 @@ +matrix: +- name: Markdown + hunspell: + d: en_US.ISO8859-15 + dictionary: + wordlists: + - ${VAL_REPO}/pyspelling/lpot_dict.txt + output: ${VAL_REPO}/pyspelling/lpot_dict.dic + sources: + - ${LPOT_REPO}/neural_coder/**/*.md + - ${LPOT_REPO}/neural_coder/*.md \ No newline at end of file diff --git a/.azure-pipelines/scripts/codeScan/scanLog/lpot-bandit.log b/.azure-pipelines/scripts/codeScan/scanLog/lpot-bandit.log new file mode 100644 index 00000000000..9d07aa0df55 --- /dev/null +++ b/.azure-pipelines/scripts/codeScan/scanLog/lpot-bandit.log @@ -0,0 +1 @@ +111 \ No newline at end of file diff --git a/.azure-pipelines/scripts/collect_log.sh b/.azure-pipelines/scripts/collect_log.sh new file mode 100644 index 00000000000..2c36ca2afa7 --- /dev/null +++ b/.azure-pipelines/scripts/collect_log.sh @@ -0,0 +1,12 @@ + +export COVERAGE_RCFILE=/neural-compressor/.azure-pipelines/scripts/ut/coverage.file +cd /neural-compressor/log_dir +cp ut-coverage-strategy/.coverage.strategy ./ +cp ut-coverage-util/.coverage.util ./ +coverage combine --keep --rcfile=${COVERAGE_RCFILE} +cp .coverage /neural-compressor/.coverage +cd /neural-compressor +coverage report -m --rcfile=${COVERAGE_RCFILE} +coverage html -d log_dir/htmlcov --rcfile=${COVERAGE_RCFILE} +ls -l log_dir/htmlcov +cat log_dir/htmlcov/index.html diff --git a/.azure-pipelines/scripts/install_nc.sh b/.azure-pipelines/scripts/install_nc.sh new file mode 100644 index 00000000000..38fd439404b --- /dev/null +++ b/.azure-pipelines/scripts/install_nc.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +cd /neural-compressor +python -m pip install --no-cache-dir -r requirements.txt +python setup.py sdist bdist_wheel +pip install dist/neural_compressor*.whl +pip list \ No newline at end of file diff --git a/.azure-pipelines/scripts/install_nc_coverage.sh b/.azure-pipelines/scripts/install_nc_coverage.sh new file mode 100644 index 00000000000..63c9bdcaab9 --- /dev/null +++ b/.azure-pipelines/scripts/install_nc_coverage.sh @@ -0,0 +1,7 @@ +cd /neural-compressor +python -m pip install --no-cache-dir -r requirements.txt +python setup.py sdist bdist_wheel +pip install dist/neural_compressor*.whl +pip install pytest +pip install coverage +pip list \ No newline at end of file diff --git a/.azure-pipelines/scripts/install_nc_full.sh b/.azure-pipelines/scripts/install_nc_full.sh new file mode 100644 index 00000000000..b3f59626dfd --- /dev/null +++ b/.azure-pipelines/scripts/install_nc_full.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +cd /neural-compressor +python -m pip install --no-cache-dir -r requirements.txt +python setup.py --full sdist bdist_wheel +pip install dist/neural_compressor*.whl +pip list \ No newline at end of file diff --git a/.azure-pipelines/scripts/ut/coverage.file b/.azure-pipelines/scripts/ut/coverage.file new file mode 100644 index 00000000000..4e649650502 --- /dev/null +++ b/.azure-pipelines/scripts/ut/coverage.file @@ -0,0 +1,25 @@ +[run] +branch = True +relative_files=True +[paths] +source = + /usr/local/lib/python3.8/dist-packages/neural_compressor + /neural-compressor/neural_compressor +[report] +omit = + */**/fake*yaml + */**/fake.py + */neural_compressor/ux/* + */neural_compressor/model/nets_factory.py + */neural_compressor/experimental/benchmark.py + */neural_compressor/contrib/strategy/tpe.py + */nlp_toolkit/backends/* +exclude_lines = + pragma: no cover + raise NotImplementedError + raise TypeError + if self.device == "gpu": + if device == "gpu": + except ImportError: + onnx_version < ONNX18_VERSION + onnx_version >= ONNX18_VERSION diff --git a/.azure-pipelines/scripts/ut/env_setup.sh b/.azure-pipelines/scripts/ut/env_setup.sh new file mode 100644 index 00000000000..af008a8bd01 --- /dev/null +++ b/.azure-pipelines/scripts/ut/env_setup.sh @@ -0,0 +1,66 @@ +#!/bin/bash +set -x + +echo $tensorflow_version +echo $pytorch_version +echo $torchvision_version +echo $onnx_version +echo $onnxruntime_version +echo $mxnet_version + +if [[ "${tensorflow_version}" == *"-official" ]]; then + pip install tensorflow==${tensorflow_version%-official} +elif [[ "${tensorflow_version}" == "spr-base" ]]; then + pip install /tf_dataset/tf_binary/tensorflow*.whl + if [[ $? -ne 0 ]]; then + exit 1 + fi +elif [[ "${tensorflow_version}" != "" ]]; then + pip install intel-tensorflow==${tensorflow_version} +fi + +if [[ "${pytorch_version}" != "" ]]; then + pip install torch==${pytorch_version} -f https://download.pytorch.org/whl/torch_stable.html +fi + +if [[ "${torchvision_version}" != "" ]]; then + pip install torchvision==${torchvision_version} -f https://download.pytorch.org/whl/torch_stable.html +fi + +if [[ "${onnx_version}" != "" ]]; then + pip install onnx==${onnx_version} +fi + +if [[ "${onnxruntime_version}" != "" ]]; then + pip install onnxruntime==${onnxruntime_version} + pip install onnxruntime-extensions +fi + +if [ "${mxnet_version}" == '1.6.0' ]; then + pip install mxnet-mkl==${mxnet_version} +elif [ "${mxnet_version}" == '1.7.0' ]; then + pip install mxnet==${mxnet_version}.post2 +elif [ "${mxnet_version}" != '' ]; then + pip install mxnet==${mxnet_version} +fi + + +cd /neural-compressor/test +if [ -f "requirements.txt" ]; then + sed -i '/^neural-compressor/d' requirements.txt + sed -i '/^intel-tensorflow/d' requirements.txt + sed -i '/find-links https:\/\/download.pytorch.org\/whl\/torch_stable.html/d' requirements.txt + sed -i '/^torch/d;/^torchvision/d' requirements.txt + sed -i '/^mxnet-mkl/d' requirements.txt + sed -i '/^onnx/d;/^onnxruntime/d;/^onnxruntime-extensions/d' requirements.txt + n=0 + until [ "$n" -ge 3 ] + do + python -m pip install --no-cache-dir -r requirements.txt && break + n=$((n+1)) + sleep 5 + done + pip list +else + echo "Not found requirements.txt file." +fi diff --git a/.azure-pipelines/scripts/ut/run_basic_adaptor.sh b/.azure-pipelines/scripts/ut/run_basic_adaptor.sh new file mode 100644 index 00000000000..334be82959e --- /dev/null +++ b/.azure-pipelines/scripts/ut/run_basic_adaptor.sh @@ -0,0 +1,29 @@ +#!/bin/bash +set -x + +echo "specify fwk version..." +export tensorflow_version='2.9.1' +export pytorch_version='1.11.0+cpu' +export torchvision_version='0.12.0' +export onnx_version='1.9.0' +export onnxruntime_version='1.10.0' +export mxnet_version='1.7.0' + +echo "set up UT env..." +bash /neural-compressor/.azure-pipelines/scripts/ut/env_setup.sh + +cd /neural-compressor/test || exit 1 +find ./adaptor -name "test*.py" | sed 's,\.\/,python ,g' | sed 's/$/ --verbose/' > run.sh + +LOG_DIR=/neural-compressor/log_dir +mkdir -p ${LOG_DIR} +ut_log_name=${LOG_DIR}/ut_tf_${tensorflow_version}_pt_${pytorch_version}.log + +echo "cat run.sh..." +cat run.sh | tee ${ut_log_name} +echo "-------------" +bash run.sh 2>&1 | tee -a ${ut_log_name} + +if [ $(grep -c "FAILED" ${ut_log_name}) != 0 ] || [ $(grep -c "OK" ${ut_log_name}) == 0 ];then + exit 1 +fi \ No newline at end of file diff --git a/.azure-pipelines/scripts/ut/run_basic_adaptor_tfnewapi.sh b/.azure-pipelines/scripts/ut/run_basic_adaptor_tfnewapi.sh new file mode 100644 index 00000000000..50b11cdf351 --- /dev/null +++ b/.azure-pipelines/scripts/ut/run_basic_adaptor_tfnewapi.sh @@ -0,0 +1,26 @@ +#!/bin/bash +set -x +python -c "import neural_compressor as nc;print(nc.version.__version__)" +echo "run basic adaptor tfnewapi" + +echo "specify fwk version..." +export tensorflow_version='spr-base' + +echo "set up UT env..." +bash /neural-compressor/.azure-pipelines/scripts/ut/env_setup.sh + +cd /neural-compressor/test || exit 1 +find ./tfnewapi -name "test*.py" | sed 's,\.\/,python ,g' | sed 's/$/ --verbose/' > run.sh + +LOG_DIR=/neural-compressor/log_dir +mkdir -p ${LOG_DIR} +ut_log_name=${LOG_DIR}/ut_tf_newapi.log + +echo "cat run.sh..." +cat run.sh | tee ${ut_log_name} +echo "-------------" +bash run.sh 2>&1 | tee -a ${ut_log_name} + +if [ $(grep -c "FAILED" ${ut_log_name}) != 0 ] || [ $(grep -c "OK" ${ut_log_name}) == 0 ];then + exit 1 +fi \ No newline at end of file diff --git a/.azure-pipelines/scripts/ut/run_basic_others.sh b/.azure-pipelines/scripts/ut/run_basic_others.sh new file mode 100644 index 00000000000..5cd2e39d1e5 --- /dev/null +++ b/.azure-pipelines/scripts/ut/run_basic_others.sh @@ -0,0 +1,35 @@ +#!/bin/bash +set -x +python -c "import neural_compressor as nc;print(nc.version.__version__)" +echo "run basic" + +echo "specify fwk version..." +export tensorflow_version='2.9.1' +export pytorch_version='1.11.0+cpu' +export torchvision_version='0.12.0' +export onnx_version='1.9.0' +export onnxruntime_version='1.10.0' +export mxnet_version='1.7.0' + +echo "set up UT env..." +bash /neural-compressor/.azure-pipelines/scripts/ut/env_setup.sh + +cd /neural-compressor/test || exit 1 +find . -name "test*.py" | sed 's,\.\/,python ,g' | sed 's/$/ --verbose/' > run.sh +sed -i '/ adaptor\//d' run.sh +sed -i '/ tfnewapi\//d' run.sh +sed -i '/ ux\//d' run.sh +sed -i '/ neural_coder\//d' run.sh + +LOG_DIR=/neural-compressor/log_dir +mkdir -p ${LOG_DIR} +ut_log_name=${LOG_DIR}/ut_tf_${tensorflow_version}_pt_${pytorch_version}.log + +echo "cat run.sh..." +cat run.sh | tee ${ut_log_name} +echo "-------------" +bash run.sh 2>&1 | tee -a ${ut_log_name} + +if [ $(grep -c "FAILED" ${ut_log_name}) != 0 ] || [ $(grep -c "OK" ${ut_log_name}) == 0 ];then + exit 1 +fi \ No newline at end of file diff --git a/.azure-pipelines/scripts/ut/run_basic_strategy.sh b/.azure-pipelines/scripts/ut/run_basic_strategy.sh new file mode 100644 index 00000000000..f5b66560a7f --- /dev/null +++ b/.azure-pipelines/scripts/ut/run_basic_strategy.sh @@ -0,0 +1,35 @@ +#!/bin/bash +set -x + +echo "specify fwk version..." +export tensorflow_version='2.9.1' +export pytorch_version='1.11.0+cpu' +export torchvision_version='0.12.0' +export onnx_version='1.9.0' +export onnxruntime_version='1.10.0' +export mxnet_version='1.7.0' + +echo "set up UT env..." +bash /neural-compressor/.azure-pipelines/scripts/ut/env_setup.sh +lpot_path=$(python -c 'import neural_compressor; import os; print(os.path.dirname(neural_compressor.__file__))') +cd /neural-compressor/test || exit 1 +find ./strategy -name "test*.py" | sed 's,\.\/,coverage run --source='"${lpot_path}"' --append ,g' | sed 's/$/ --verbose/'> run.sh + +LOG_DIR=/neural-compressor/log_dir +mkdir -p ${LOG_DIR} +ut_log_name=${LOG_DIR}/ut_tf_${tensorflow_version}_pt_${pytorch_version}.log + +echo "cat run.sh..." +cat run.sh | tee ${ut_log_name} +echo "-------------" +bash run.sh 2>&1 | tee -a ${ut_log_name} +echo "working in directory" +pwd +echo "list all component" +ls -a +cp .coverage ${LOG_DIR}/.coverage.strategy +echo "list all in ${LOG_DIR}" +ls -a ${LOG_DIR} +if [ $(grep -c "FAILED" ${ut_log_name}) != 0 ] || [ $(grep -c "OK" ${ut_log_name}) == 0 ];then + exit 1 +fi \ No newline at end of file diff --git a/.azure-pipelines/scripts/ut/run_basic_utils.sh b/.azure-pipelines/scripts/ut/run_basic_utils.sh new file mode 100644 index 00000000000..7cd4c371b88 --- /dev/null +++ b/.azure-pipelines/scripts/ut/run_basic_utils.sh @@ -0,0 +1,41 @@ +#!/bin/bash +set -x +python -c "import neural_compressor as nc;print(nc.version.__version__)" +echo "run basic" + +echo "specify fwk version..." +export tensorflow_version='2.9.1' +export pytorch_version='1.11.0+cpu' +export torchvision_version='0.12.0' +export onnx_version='1.9.0' +export onnxruntime_version='1.10.0' +export mxnet_version='1.7.0' + +echo "set up UT env..." +bash /neural-compressor/.azure-pipelines/scripts/ut/env_setup.sh +lpot_path=$(python -c 'import neural_compressor; import os; print(os.path.dirname(neural_compressor.__file__))') +cd /neural-compressor/test || exit 1 +find ./utils -name "test*.py" | sed 's,\.\/,coverage run --source='"${lpot_path}"' --append ,g' | sed 's/$/ --verbose/'> run.sh +sed -i '/ adaptor\//d' run.sh +sed -i '/ tfnewapi\//d' run.sh +sed -i '/ ux\//d' run.sh +sed -i '/ neural_coder\//d' run.sh + +LOG_DIR=/neural-compressor/log_dir +mkdir -p ${LOG_DIR} +ut_log_name=${LOG_DIR}/ut_tf_${tensorflow_version}_pt_${pytorch_version}.log + +echo "cat run.sh..." +cat run.sh | tee ${ut_log_name} +echo "-------------" +bash run.sh 2>&1 | tee -a ${ut_log_name} +echo "working in directory" +pwd +echo "list all component" +ls -a +cp .coverage ${LOG_DIR}/.coverage.util +echo "list all in ${LOG_DIR}" +ls -a ${LOG_DIR} +if [ $(grep -c "FAILED" ${ut_log_name}) != 0 ] || [ $(grep -c "OK" ${ut_log_name}) == 0 ];then + exit 1 +fi \ No newline at end of file diff --git a/.azure-pipelines/scripts/ut/run_ncoder.sh b/.azure-pipelines/scripts/ut/run_ncoder.sh new file mode 100644 index 00000000000..aef05d13e3f --- /dev/null +++ b/.azure-pipelines/scripts/ut/run_ncoder.sh @@ -0,0 +1,23 @@ +#!/bin/bash +set -x +python -c "import neural_compressor as nc;print(nc.version.__version__)" +echo "run coder" + +echo "no FWKs need to be installed..." +echo "no requirements need to be installed..." + +cd /neural-compressor/test || exit 1 +find ./neural_coder -name "test*.py" | sed 's,\.\/,python ,g' | sed 's/$/ --verbose/' > run.sh + +LOG_DIR=/neural-compressor/log_dir +mkdir -p ${LOG_DIR} +ut_log_name=${LOG_DIR}/ut_neural_coder.log + +echo "cat run.sh..." +cat run.sh | tee ${ut_log_name} +echo "-------------" +bash run.sh 2>&1 | tee -a ${ut_log_name} + +if [ $(grep -c "FAILED" ${ut_log_name}) != 0 ] || [ $(grep -c "OK" ${ut_log_name}) == 0 ];then + exit 1 +fi \ No newline at end of file diff --git a/.azure-pipelines/scripts/ut/run_ux.sh b/.azure-pipelines/scripts/ut/run_ux.sh new file mode 100644 index 00000000000..ddfff991652 --- /dev/null +++ b/.azure-pipelines/scripts/ut/run_ux.sh @@ -0,0 +1,30 @@ +#!/bin/bash +set -x +python -c "import neural_compressor as nc;print(nc.version.__version__)" +echo "run ux" + +echo "specify fwk version..." +export tensorflow_version='2.9.1' +export pytorch_version='1.11.0+cpu' +export torchvision_version='0.12.0' +export onnx_version='1.9.0' +export onnxruntime_version='1.10.0' + +echo "set up UT env..." +bash /neural-compressor/.azure-pipelines/scripts/ut/env_setup.sh + +cd /neural-compressor/test || exit 1 +find ./ux -name "test*.py" | sed 's,\.\/,python ,g' | sed 's/$/ --verbose/' > run.sh + +LOG_DIR=/neural-compressor/log_dir +mkdir -p ${LOG_DIR} +ut_log_name=${LOG_DIR}/ut_tf_${tensorflow_version}_pt_${pytorch_version}.log + +echo "cat run.sh..." +cat run.sh | tee ${ut_log_name} +echo "-------------" +bash run.sh 2>&1 | tee -a ${ut_log_name} + +if [ $(grep -c "FAILED" ${ut_log_name}) != 0 ] || [ $(grep -c "OK" ${ut_log_name}) == 0 ];then + exit 1 +fi \ No newline at end of file diff --git a/.azure-pipelines/ut-basic.yml b/.azure-pipelines/ut-basic.yml new file mode 100644 index 00000000000..4e7ec62d2bc --- /dev/null +++ b/.azure-pipelines/ut-basic.yml @@ -0,0 +1,132 @@ +trigger: none + +pr: + branches: + include: + - master + paths: + include: + - neural_compressor + exclude: + - neural_compressor/ux + +pool: suyue-test + +variables: + IMAGE_NAME: 'neural-compressor' + IMAGE_TAG: 'py38' + +stages: +- stage: + displayName: Unit Test FWKs adaptor + dependsOn: [] + jobs: + - job: + steps: + - script: | + echo ${BUILD_SOURCESDIRECTORY} + sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + displayName: 'Clean workspace' + + - checkout: self + displayName: "Checkout out Repo" + + - script: | + if [[ ! $(docker images | grep -i '${IMAGE_NAME}:${IMAGE_TAG}' ) ]]; then + docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + fi + docker images | grep -i ${IMAGE_NAME} + if [[ $? -ne 0 ]]; then + echo "NO Such Image ${IMAGE_NAME}" + exit 1 + fi + displayName: "Build develop docker image" + + - script: | + docker stop $(docker ps -aq) + docker rm -vf $(docker ps -aq) || true + env | sort + displayName: 'Clean docker' + + - script: | + docker run --disable-content-trust --privileged --name="pr" --hostname="pr-host" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} \ + bash -c "cd /neural-compressor/.azure-pipelines/scripts \ + && bash install_nc.sh \ + && bash ut/run_basic_adaptor.sh" + displayName: 'Install Neural Compressor and run UT' + +- stage: + displayName: Unit Test tf newAPI + dependsOn: [] + jobs: + - job: + steps: + - script: | + echo ${BUILD_SOURCESDIRECTORY} + sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + displayName: 'Clean workspace' + + - checkout: self + displayName: "Checkout out Repo" + + - script: | + if [[ ! $(docker images | grep -i '${IMAGE_NAME}:${IMAGE_TAG}' ) ]]; then + docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + fi + docker images | grep -i ${IMAGE_NAME} + if [[ $? -ne 0 ]]; then + echo "NO Such Image ${IMAGE_NAME}" + exit 1 + fi + displayName: "Build develop docker image" + + - script: | + docker stop $(docker ps -aq) + docker rm -vf $(docker ps -aq) || true + env | sort + displayName: 'Clean docker' + + - script: | + docker run --disable-content-trust --privileged --name="pr" --hostname="pr-host" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor -v /tf_dataset:/tf_dataset ${IMAGE_NAME}:${IMAGE_TAG} \ + bash -c "cd /neural-compressor/.azure-pipelines/scripts \ + && bash install_nc.sh \ + && bash ut/run_basic_adaptor_tfnewapi.sh" + displayName: 'Install Neural Compressor and run UT' + +- stage: + displayName: Unit Test other basic case + dependsOn: [] + jobs: + - job: + steps: + - script: | + echo ${BUILD_SOURCESDIRECTORY} + sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + displayName: 'Clean workspace' + + - checkout: self + displayName: "Checkout out Repo" + + - script: | + if [[ ! $(docker images | grep -i '${IMAGE_NAME}:${IMAGE_TAG}' ) ]]; then + docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + fi + docker images | grep -i ${IMAGE_NAME} + if [[ $? -ne 0 ]]; then + echo "NO Such Image ${IMAGE_NAME}" + exit 1 + fi + displayName: "Build develop docker image" + + - script: | + docker stop $(docker ps -aq) + docker rm -vf $(docker ps -aq) || true + env | sort + displayName: 'Clean docker' + + - script: | + docker run --disable-content-trust --privileged --name="pr" --hostname="pr-host" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} \ + bash -c "cd /neural-compressor/.azure-pipelines/scripts \ + && bash install_nc.sh \ + && bash ut/run_basic_others.sh" + displayName: 'Install Neural Compressor and run UT' diff --git a/.azure-pipelines/ut-coverage.yml b/.azure-pipelines/ut-coverage.yml new file mode 100644 index 00000000000..9c968673edb --- /dev/null +++ b/.azure-pipelines/ut-coverage.yml @@ -0,0 +1,163 @@ +trigger: none + +pr: + - wenxin/ut-coverage + +pool: suyue-test + +variables: + IMAGE_NAME: 'neural-compressor' + IMAGE_TAG: 'py38' + UPLOAD_PATH: $(Build.SourcesDirectory)/log_dir + DOWNLOAD_PATH: $(Build.SourcesDirectory)/log_dir + OS: 'linux' + PLATFORM: 'clx8280' + ARTIFACT_NAME: 'UT_coverage_report' + +stages: +- stage: Strategy + displayName: "Unit Test Strategy" + dependsOn: [] + jobs: + - job: + steps: + - script: | + echo ${BUILD_SOURCESDIRECTORY} + sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + displayName: 'Clean workspace' + + - checkout: self + displayName: "Checkout out Repo" + + - script: | + if [[ ! $(docker images | grep -i '${IMAGE_NAME}:${IMAGE_TAG}' ) ]]; then + docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + fi + docker images | grep -i ${IMAGE_NAME} + if [[ $? -ne 0 ]]; then + echo "NO Such Image ${IMAGE_NAME}" + exit 1 + fi + displayName: "Build develop docker image" + + - script: | + docker stop $(docker ps -aq) + docker rm -vf $(docker ps -aq) || true + env | sort + displayName: 'Clean docker' + + - script: | + docker run --disable-content-trust --privileged --name="ut-stra" --hostname="pr-host" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} \ + bash -c "cd /neural-compressor/.azure-pipelines/scripts \ + && bash install_nc_coverage.sh \ + && bash ut/run_basic_strategy.sh" + # docker cp 'ut-stra':/neural-compressor/log_dir/.coverage.strategy $(UPLOAD_PATH)/ + ls -l $(UPLOAD_PATH) + ls -l $(DOWNLOAD_PATH) + displayName: 'Install Neural Compressor and run UT' + - task: PublishPipelineArtifact@1 + inputs: + targetPath: $(UPLOAD_PATH) + artifact: 'ut-coverage-strategy' + publishLocation: 'pipeline' + +- stage: Utils + displayName: "Unit Test Utils" + dependsOn: [] + jobs: + - job: + steps: + - script: | + echo ${BUILD_SOURCESDIRECTORY} + sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + displayName: 'Clean workspace' + + - checkout: self + displayName: "Checkout out Repo" + + - script: | + if [[ ! $(docker images | grep -i '${IMAGE_NAME}:${IMAGE_TAG}' ) ]]; then + docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + fi + docker images | grep -i ${IMAGE_NAME} + if [[ $? -ne 0 ]]; then + echo "NO Such Image ${IMAGE_NAME}" + exit 1 + fi + displayName: "Build develop docker image" + + - script: | + docker stop $(docker ps -aq) + docker rm -vf $(docker ps -aq) || true + env | sort + displayName: 'Clean docker' + + - script: | + docker run --disable-content-trust --privileged --name="ut-util" --hostname="pr-host" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} \ + bash -c "cd /neural-compressor/.azure-pipelines/scripts \ + && bash install_nc_coverage.sh \ + && bash ut/run_basic_utils.sh" + # docker cp 'ut-util':/neural-compressor/log_dir/.coverage.util $(UPLOAD_PATH)/ + ls -l $(UPLOAD_PATH) + ls -l $(DOWNLOAD_PATH) + displayName: 'Install Neural Compressor and run UT' + - task: PublishPipelineArtifact@1 + inputs: + targetPath: $(UPLOAD_PATH) + artifact: 'ut-coverage-util' + publishLocation: 'pipeline' + +- stage: Coverage + displayName: "Coverage Combine" + dependsOn: [Utils, Strategy] + jobs: + - job: CollectDatafiles + displayName: collect coverage files + steps: + - script: | + echo ${BUILD_SOURCESDIRECTORY} + sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + echo y | docker system prune + displayName: 'Clean workspace' + - checkout: self + clean: true + displayName: 'Checkout out Repo' + - script: | + if [[ ! $(docker images | grep -i ${IMAGE_NAME}:${IMAGE_TAG}) ]]; then + docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + fi + docker images | grep -i ${IMAGE_NAME} + if [[ $? -ne 0 ]]; then + echo "NO Such Image ${IMAGE_NAME}" + exit 1 + fi + displayName: "Build Devel Images" + - script: | + docker stop $(docker ps -aq) + docker rm -vf $(docker ps -aq) || true + displayName: 'Clean Docker' + - task: DownloadPipelineArtifact@2 + inputs: + artifact: + path: $(DOWNLOAD_PATH) + - script: | + echo "------ Collecting logs ------" + echo "--- create container ---" + docker run -d -it --name="collectLogs" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash + #echo "------ make coverage datas directory" + #docker exec collectLogs /bin/bash +x -c "mkdir -p /neural-compressor/.azure-pipelines/scripts/ut/coverage_datas" + #echo "--- copy files into container ---" + #docker cp $(DOWNLOAD_PATH)/ 'collectLogs':/neural-compressor/log_dir + echo "--- docker ps ---" + docker ps + echo "--- collect logs ---" + docker exec collectLogs /bin/bash +x -c "cd /neural-compressor/.azure-pipelines/scripts \ + && bash install_nc_coverage.sh \ + && bash collect_log.sh" + #docker cp 'collectLogs':/neural-compressor/log_dir/htmlcov $(UPLOAD_PATH)/htmlcov + displayName: 'collect logs' + - task: PublishPipelineArtifact@1 + inputs: + targetPath: $(UPLOAD_PATH) + artifact: $(ARTIFACT_NAME) + publishLocation: 'pipeline' diff --git a/.azure-pipelines/ut-ncoder.yml b/.azure-pipelines/ut-ncoder.yml new file mode 100644 index 00000000000..e2a60d39a76 --- /dev/null +++ b/.azure-pipelines/ut-ncoder.yml @@ -0,0 +1,53 @@ +trigger: none + +pr: + branches: + include: + - master + paths: + include: + - neural_coder + +pool: suyue-test + +variables: + IMAGE_NAME: 'neural-compressor' + IMAGE_TAG: 'py38' + +stages: +- stage: + displayName: Unit Test for Neural Coder + jobs: + - job: + steps: + - script: | + echo ${BUILD_SOURCESDIRECTORY} + sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + displayName: 'Clean workspace' + + - checkout: self + displayName: "Checkout out Repo" + + - script: | + if [[ ! $(docker images | grep -i '${IMAGE_NAME}:${IMAGE_TAG}' ) ]]; then + docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + fi + docker images | grep -i ${IMAGE_NAME} + if [[ $? -ne 0 ]]; then + echo "NO Such Image ${IMAGE_NAME}" + exit 1 + fi + displayName: "Build develop docker image" + + - script: | + docker stop $(docker ps -aq) + docker rm -vf $(docker ps -aq) || true + env | sort + displayName: 'Clean docker' + + - script: | + docker run --disable-content-trust --privileged --name="pr" --hostname="pr-host" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} \ + bash -c "cd /neural-compressor/.azure-pipelines/scripts \ + && bash install_nc_full.sh \ + && bash ut/run_ncoder.sh" + displayName: 'Install Neural Compressor and run UT' \ No newline at end of file diff --git a/.azure-pipelines/ut-ux.yml b/.azure-pipelines/ut-ux.yml new file mode 100644 index 00000000000..41922165d84 --- /dev/null +++ b/.azure-pipelines/ut-ux.yml @@ -0,0 +1,53 @@ +trigger: none + +pr: + branches: + include: + - master + paths: + include: + - neural_compressor/ux + +pool: suyue-test + +variables: + IMAGE_NAME: 'neural-compressor' + IMAGE_TAG: 'py38' + +stages: +- stage: + displayName: Unit Test for UX + jobs: + - job: + steps: + - script: | + echo ${BUILD_SOURCESDIRECTORY} + sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + displayName: 'Clean workspace' + + - checkout: self + displayName: "Checkout out Repo" + + - script: | + if [[ ! $(docker images | grep -i '${IMAGE_NAME}:${IMAGE_TAG}' ) ]]; then + docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + fi + docker images | grep -i ${IMAGE_NAME} + if [[ $? -ne 0 ]]; then + echo "NO Such Image ${IMAGE_NAME}" + exit 1 + fi + displayName: "Build develop docker image" + + - script: | + docker stop $(docker ps -aq) + docker rm -vf $(docker ps -aq) || true + env | sort + displayName: 'Clean docker' + + - script: | + docker run --disable-content-trust --privileged --name="pr" --hostname="pr-host" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} \ + bash -c "cd /neural-compressor/.azure-pipelines/scripts \ + && bash install_nc_full.sh \ + && bash ut/run_ux.sh" + displayName: 'Install Neural Compressor and run UT' \ No newline at end of file diff --git a/.gitignore b/.gitignore index 6f4be8433c0..b222b2390cc 100644 --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,6 @@ *.csv *.pb *.ckpt -*.log *.swp *.onnx *.so diff --git a/azure-pipelines.yml b/azure-pipelines.yml new file mode 100644 index 00000000000..77719aa486b --- /dev/null +++ b/azure-pipelines.yml @@ -0,0 +1,67 @@ +# Starter pipeline +# Start with a minimal pipeline that you can customize to build and deploy your code. +# Add steps that build, run tests, deploy, and more: +# https://aka.ms/yaml + +pr: + - letong_performance + +pool: wenjiao-test + +stages: + - stage: pylint + jobs: + - job: pylint_test + steps: + - task: Bash@3 + inputs: + targetType: "inline" + script: | + pwd + pip install pylint==2.12.1 + pip install intel-tensorflow + python -m pylint -f json --disable=R,C,W,E1129 --enable=line-too-long --max-line-length=120 --extension-pkg-whitelist=numpy --ignored-classes=TensorProto,NodeProto --ignored-modules=tensorflow,torch,torch.quantization,torch.tensor,torchvision,mxnet,onnx,onnxruntime ./neural_compressor > /lpot-pylint.json + # # exit_code=$? + # # if [ ${exit_code} -ne 0 ] ; then + # # echo "PyLint exited with non-zero exit code."; exit 1 + # # fi + # # exit 0 + + - stage: bandit + jobs: + - job: bandit_test + steps: + - task: Bash@3 + inputs: + targetType: "inline" +# script: | +# pip install bandit +# # python -m bandit -r -lll -iii neural_compressor > ${WORKSPACE}/lpot-bandit.log +# # exit_code=$? +# # if [ ${exit_code} -ne 0 ] ; then +# # echo "Bandit exited with non-zero exit code."; exit 1 +# # fi +# # exit 0 + + # - stage: pyspelling + # jobs: + # - job: pyspelling_test +# steps: +# - task: Bash@3 +# inputs: +# targetType: "inline" +# script: | +# pip install pyspelling +# # Update paths to validation and lpot repositories +# # VAL_REPO=${WORKSPACE} + +# # sed -i "s|\${VAL_REPO}|$VAL_REPO|g" ${VAL_REPO}/pyspelling_conf.yaml +# # sed -i "s|\${LPOT_REPO}|$REPO_DIR|g" ${VAL_REPO}/pyspelling_conf.yaml +# # echo "Modified config:" +# # cat ${VAL_REPO}/pyspelling_conf.yaml +# # pyspelling -c ${VAL_REPO}/pyspelling_conf.yaml > ${WORKSPACE}/pyspelling_output.log +# # exit_code=$? +# # if [ ${exit_code} -ne 0 ] ; then +# # echo "Pyspelling exited with non-zero exit code."; exit 1 +# # fi +# # exit 0 diff --git a/docker/README.md b/docker/README.md index dd8b285c1ab..dc3923e1ec4 100644 --- a/docker/README.md +++ b/docker/README.md @@ -6,7 +6,7 @@ https://pypi.org/project/neural-compressor/#history ```console $ PYTHON=python3.8 -$ INC_VER=1.12 +$ INC_VER=1.13 $ IMAGE_NAME=neural-compressor $ IMAGE_TAG=${INC_VER} $ docker build --build-arg PYTHON=${PYTHON} --build-arg INC_VER=${INC_VER} -f Dockerfile -t ${IMAGE_NAME}:${IMAGE_TAG} . @@ -20,7 +20,7 @@ then please modify the tag so that the tagging requirement is met. For example r ```console $ PYTHON=python3.8 -$ INC_BRANCH=v1.12 +$ INC_BRANCH=v1.13 $ IMAGE_NAME=neural-compressor $ IMAGE_TAG=${INC_BRANCH}-devel $ docker build --build-arg PYTHON=${PYTHON} --build-arg INC_BRANCH=${INC_BRANCH} -f Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . diff --git a/neural_compressor/objective.py b/neural_compressor/objective.py index b6e656ff0a9..a7561cbd02b 100644 --- a/neural_compressor/objective.py +++ b/neural_compressor/objective.py @@ -1,7 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- # -# Copyright (c) 2021 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/neural_compressor/ux/__init__.py b/neural_compressor/ux/__init__.py index 4dad25e4928..9100b4bb131 100644 --- a/neural_compressor/ux/__init__.py +++ b/neural_compressor/ux/__init__.py @@ -12,4 +12,5 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + """The app package contains all components required for UX server."""