fix issues : no crash for parsing a large log file and IPS#00555843

Libraries/oneCCL/tutorials/oneCCL_Getting_Started.ipynb

@@ -45,7 +45,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "%env ONEAPI_INSTALL=/opt/intel/inteloneapi/"
+    "%env ONEAPI_INSTALL=/opt/intel/oneapi/"
    ]
   },
   {
@@ -78,7 +78,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "!mkdir lab"
+    "!rm -rf lab;mkdir -p lab"
    ]
   },
   {
@@ -307,9 +307,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {
-    "scrolled": false
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "!rm -rf cpu_gomp; chmod 755 q; chmod 755 build.sh; chmod 755 run.sh;if [ -x \"$(command -v qsub)\" ]; then ./q build.sh; ./q run.sh; else ./build.sh; ./run.sh; fi"
@@ -984,9 +982,9 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 2",
+   "display_name": "tutorial",
    "language": "python",
-   "name": "python2"
+   "name": "tutorial"
   },
   "language_info": {
    "codemirror_mode": {
@@ -998,7 +996,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.6.9"
+   "version": "3.7.7"
   },
   "toc": {
    "base_numbering": 1,
@@ -1020,5 +1018,5 @@
   }
  },
  "nbformat": 4,
- "nbformat_minor": 2
+ "nbformat_minor": 4
 }

Libraries/oneDNN/tutorials/CMakeLists.txt

@@ -1,7 +1,7 @@
 cmake_minimum_required(VERSION 2.8.11)
 if("${CMAKE_CXX_COMPILER}" STREQUAL "")
        set(CMAKE_C_COMPILER "clang")
-       set(CMAKE_CXX_COMPILER "dpcpp")
+       set(CMAKE_CXX_COMPILER "clang++")
 endif()
 project (oneDNN)
 if("$ENV{EXAMPLE_ROOT}" STREQUAL "")

Libraries/oneDNN/tutorials/codes_for_ipynb/cnn_inference_f32.patch

@@ -1,6 +1,14 @@
---- cnn_inference_f32.cpp	2020-02-12 10:12:10.467690007 -0800
-+++ cnn_inference_f32_gpu.cpp	2020-02-12 10:12:28.395690295 -0800
-@@ -19,12 +19,62 @@
+--- cnn_inference_f32.cpp	2020-12-02 16:40:15.494038678 -0800
++++ cnn_inference_f32_gpu.cpp	2020-12-02 18:06:15.773550045 -0800
+@@ -9,6 +9,7 @@
+ #include <vector>
+
+ #include "dnnl.hpp"
++#include "dnnl_sycl.hpp"
+
+ using namespace dnnl;
+
+@@ -19,12 +20,63 @@
              std::multiplies<memory::dim>());
  }
 
@@ -18,7 +26,8 @@
 +            && eng.get_kind() == dnnl::engine::kind::gpu);
 +    if (is_cpu_sycl || is_gpu_sycl) {
 +
-+        auto buffer = mem.get_sycl_buffer<uint8_t>();
++        //auto buffer = mem.get_sycl_buffer<uint8_t>();
++        auto buffer = dnnl::sycl_interop::get_buffer<uint8_t>(mem);
 +        auto dst = buffer.get_access<cl::sycl::access::mode::write>();
 +        uint8_t *dst_ptr = dst.get_pointer();
 +
@@ -64,7 +73,7 @@
      stream s(eng);
 
      std::vector<primitive> net;
-@@ -53,13 +103,17 @@
+@@ -53,13 +105,17 @@
  //[Allocate buffers]
 
 
@@ -86,7 +95,7 @@
 
 
 
-@@ -175,13 +229,18 @@
+@@ -175,13 +231,18 @@
      std::vector<float> conv2_weights(product(conv2_weights_tz));
      std::vector<float> conv2_bias(product(conv2_bias_tz));
 
@@ -108,7 +117,7 @@
      // create memory descriptors for convolution data w/ no specified format
      auto conv2_src_md = memory::desc({ conv2_src_tz }, dt::f32, tag::any);
      auto conv2_bias_md = memory::desc({ conv2_bias_tz }, dt::f32, tag::any);
-@@ -291,13 +350,18 @@
+@@ -291,13 +352,18 @@
      std::vector<float> conv3_weights(product(conv3_weights_tz));
      std::vector<float> conv3_bias(product(conv3_bias_tz));
 
@@ -130,7 +139,7 @@
      // create memory descriptors for convolution data w/ no specified format
      auto conv3_src_md = memory::desc({ conv3_src_tz }, dt::f32, tag::any);
      auto conv3_bias_md = memory::desc({ conv3_bias_tz }, dt::f32, tag::any);
-@@ -364,13 +428,17 @@
+@@ -364,13 +430,17 @@
      std::vector<float> conv4_weights(product(conv4_weights_tz));
      std::vector<float> conv4_bias(product(conv4_bias_tz));
 
@@ -151,7 +160,7 @@
      // create memory descriptors for convolution data w/ no specified format
      auto conv4_src_md = memory::desc({ conv4_src_tz }, dt::f32, tag::any);
      auto conv4_bias_md = memory::desc({ conv4_bias_tz }, dt::f32, tag::any);
-@@ -436,13 +504,18 @@
+@@ -436,13 +506,18 @@
      std::vector<float> conv5_weights(product(conv5_weights_tz));
      std::vector<float> conv5_bias(product(conv5_bias_tz));
 
@@ -173,7 +182,7 @@
      // create memory descriptors for convolution data w/ no specified format
      auto conv5_src_md = memory::desc({ conv5_src_tz }, dt::f32, tag::any);
      auto conv5_weights_md
-@@ -532,13 +605,18 @@
+@@ -532,13 +607,18 @@
      std::vector<float> fc6_weights(product(fc6_weights_tz));
      std::vector<float> fc6_bias(product(fc6_bias_tz));
 
@@ -195,7 +204,7 @@
      // create memory descriptors for convolution data w/ no specified format
      auto fc6_src_md = memory::desc({ fc6_src_tz }, dt::f32, tag::any);
      auto fc6_bias_md = memory::desc({ fc6_bias_tz }, dt::f32, tag::any);
-@@ -583,13 +661,18 @@
+@@ -583,13 +663,18 @@
      std::vector<float> fc7_weights(product(fc7_weights_tz));
      std::vector<float> fc7_bias(product(fc7_bias_tz));
 
@@ -218,7 +227,7 @@
      // create memory descriptors for convolution data w/ no specified format
      auto fc7_bias_md = memory::desc({ fc7_bias_tz }, dt::f32, tag::any);
      auto fc7_weights_md = memory::desc({ fc7_weights_tz }, dt::f32, tag::any);
-@@ -624,14 +707,20 @@
+@@ -624,14 +709,20 @@
      std::vector<float> fc8_weights(product(fc8_weights_tz));
      std::vector<float> fc8_bias(product(fc8_bias_tz));
 
@@ -243,3 +252,10 @@
      // create memory descriptors for convolution data w/ no specified format
      auto fc8_bias_md = memory::desc({ fc8_bias_tz }, dt::f32, tag::any);
      auto fc8_weights_md = memory::desc({ fc8_weights_tz }, dt::f32, tag::any);
+@@ -697,4 +788,4 @@
+         std::cerr << "message: " << e.message << std::endl;
+     }
+     return 0;
+-}
+\ No newline at end of file
++}

Libraries/oneDNN/tutorials/profiling/profile_utils.py

@@ -46,6 +46,25 @@ def run_workload(outfile='mkldnn_log.csv'):
             if 'dnnl' in l and 'exec' in l:
                 f.write(l + '\n')
 
+class FileUtils:
+
+    def __init_(self):
+        return
+    def replace_string_in_file(self, filename, oldstring, newstring):
+        fin = open(filename, "rt")
+        #output file to write the result to
+        fout = open('tmp.txt', "wt")
+        #for each line in the input file
+        for line in fin:
+            #read replace the string and write to output file
+            fout.write(line.replace(oldstring, newstring))
+        #close input and output files
+        fin.close()
+        fout.close()
+        os.remove(filename)
+        os.rename('tmp.txt',filename)
+
+
 class oneDNNLog:
 
     def __init_(self):
@@ -72,14 +91,14 @@ def load_log(self, log):
     def load_log_dnnl(self, log):
         import pandas as pd
         # dnnl_verbose,exec,cpu,convolution,jit:avx2,forward_inference,src_f32::blocked:abcd:f0 wei_f32::blocked:Acdb8a:f0 bia_f32::blocked:a:f0 dst_f32::blocked:aBcd8b:f0,,alg:convolution_direct,mb1_ic3oc96_ih227oh55kh11sh4dh0ph0_iw227ow55kw11sw4dw0pw0,1.21704
-        data = pd.read_csv(log, names=[ 'dnnl_verbose','exec','arch','type', 'jit', 'pass', 'fmt', 'opt', 'alg', 'shape', 'time'])
+        data = pd.read_csv(log, names=[ 'dnnl_verbose','exec','arch','type', 'jit', 'pass', 'fmt', 'opt', 'alg', 'shape', 'time'], engine='python')
         return data
 
     def load_log_mkldnn(self, log):
         import pandas as pd
         #mkldnn_verbose,exec,convolution,jit:avx512_common,forward_training,fsrc:nChw16c fwei:OIhw16i16o fbia:undef fdst:nChw16c,alg:convolution_direct,mb100_ic128oc32_ih7oh7kh3sh1dh0ph1_iw7ow7kw3sw1dw0pw1,0.201904
         print("load_log_mkldnn")
-        data = pd.read_csv(log, names=[ 'mkldnn_verbose','exec','type', 'jit', 'pass', 'fmt', 'alg', 'shape', 'time'])
+        data = pd.read_csv(log, names=[ 'mkldnn_verbose','exec','type', 'jit', 'pass', 'fmt', 'alg', 'shape', 'time'], engine='python')
         return data
 
 

Libraries/oneDNN/tutorials/q

@@ -4,12 +4,28 @@
 #
 # Version: 0.5
 #========================================
-#property='gpu'
-#property='clx'
-#property='skx'
-if [ -z "$property" ]; then
-    property='gpu'
-fi
+properties=("xe_hp" "dg1" "skx" "clx" "gpu") 
+avail_1=$(pbsnodes | grep properties | sort |  cut -d',' -f6 |  uniq)
+avail_2=$(pbsnodes | grep properties | sort |  cut -d',' -f2 |  uniq)
+for value in "${properties[@]}"
+do
+     for avail in $avail_1
+     do
+	 if [ "$avail" == "$value" ]; then
+             property="$value"
+	     break 2
+         fi
+     done
+     for avail in $avail_2
+     do
+	 if [ "$avail" == "$value" ]; then
+             property="$value"
+	     break 2
+         fi
+     done
+done
+
+echo $property
 
 if [ -z "$1" ]; then
 	echo "Missing script argument, Usage: ./q run.sh"

Libraries/oneDNN/tutorials/tutorial_analyze_isa_with_dispatcher_control.ipynb

@@ -98,7 +98,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "!mkdir -p lab"
+    "!rm -rf lab;mkdir -p lab"
    ]
   },
   {
@@ -228,6 +228,27 @@
     "echo \"########## Done with the run\"\n"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "\n",
+    "#### OPTIONAL : replace $ONEAPI_INSTALL with set value in both build.sh and run.sh\n",
+    "> NOTE : this step is mandatory if you run the notebook on DevCloud"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from profiling.profile_utils import FileUtils\n",
+    "file_utils = FileUtils()\n",
+    "file_utils.replace_string_in_file('build.sh','$ONEAPI_INSTALL', os.environ['ONEAPI_INSTALL'] )\n",
+    "file_utils.replace_string_in_file('run.sh','$ONEAPI_INSTALL', os.environ['ONEAPI_INSTALL'] )\n"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -464,6 +485,28 @@
     "echo \"########## Done with the run\"\n"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "#### OPTIONAL : replace ONEAPI_INSTALL, DNNL_MAX_CPU_ISA_VAL1, DNNL_APP_VAL1 and DNNL_LOG_VAL1 with set values in run.sh\n",
+    "> NOTE : this step is mandatory if you run the notebook on DevCloud"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from profiling.profile_utils import FileUtils\n",
+    "file_utils = FileUtils()\n",
+    "file_utils.replace_string_in_file('run.sh','$ONEAPI_INSTALL', os.environ['ONEAPI_INSTALL'] )\n",
+    "file_utils.replace_string_in_file('run.sh','$DNNL_MAX_CPU_ISA_VAL1', os.environ['DNNL_MAX_CPU_ISA_VAL1'] )\n",
+    "file_utils.replace_string_in_file('run.sh','$DNNL_APP_VAL1', os.environ['DNNL_APP_VAL1'] )\n",
+    "file_utils.replace_string_in_file('run.sh','$DNNL_LOG_VAL1', os.environ['DNNL_LOG_VAL1'] )"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -552,6 +595,28 @@
     "echo \"########## Done with the run\"\n"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "#### OPTIONAL : replace ONEAPI_INSTALL, DNNL_MAX_CPU_ISA_VAL2, DNNL_APP_VAL2 and DNNL_LOG_VAL2 with set values in run.sh\n",
+    "> NOTE : this step is mandatory if you run the notebook on DevCloud"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from profiling.profile_utils import FileUtils\n",
+    "file_utils = FileUtils()\n",
+    "file_utils.replace_string_in_file('run.sh','$ONEAPI_INSTALL', os.environ['ONEAPI_INSTALL'] )\n",
+    "file_utils.replace_string_in_file('run.sh','$DNNL_MAX_CPU_ISA_VAL2', os.environ['DNNL_MAX_CPU_ISA_VAL2'] )\n",
+    "file_utils.replace_string_in_file('run.sh','$DNNL_APP_VAL2', os.environ['DNNL_APP_VAL2'] )\n",
+    "file_utils.replace_string_in_file('run.sh','$DNNL_LOG_VAL2', os.environ['DNNL_LOG_VAL2'] )"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -843,9 +908,9 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 2",
+   "display_name": "tutorial",
    "language": "python",
-   "name": "python2"
+   "name": "tutorial"
   },
   "language_info": {
    "codemirror_mode": {
@@ -857,7 +922,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.6.9"
+   "version": "3.7.7"
   },
   "toc": {
    "base_numbering": 1,