Add ground truth comparison, update sample scores (#2186)

sdk/python/foundation-models/system/inference/fill-mask/book-corpus-dataset/sample_score.json

@@ -1 +1 @@
-{"inputs": ["but [MASK] soon [MASK] the guy came back with a smile on his face , emerson relaxed , knowing that everything w[MASK] okay ."]}
+{"inputs": {"input_string": ["`` was there no [MASK] between you two? \\'\\'"]}}

sdk/python/foundation-models/system/inference/fill-mask/fill-mask-online-endpoint.ipynb

@@ -130,7 +130,7 @@
     "\n",
     "# take the value of the \"text\" column, replace a random word with the mask token and save the result in the \"masked_text\" column\n",
     "import random, os\n",
-    "train_df[\"masked_text\"] = train_df[\"text\"].apply(lambda x: x.replace(random.choice(x.split()), mask_token))\n",
+    "train_df[\"masked_text\"] = train_df[\"text\"].apply(lambda x: x.replace(random.choice(x.split()), mask_token, 1))\n",
     "# save the train_df dataframe to a jsonl file in the ./book-corpus-dataset folder with the masked_ prefix\n",
     "train_df.to_json(os.path.join(\".\", \"book-corpus-dataset\",\"masked_train.jsonl\"), orient=\"records\", lines=True)\n",
     "train_df.head()"
@@ -212,9 +212,7 @@
     "# pick 1 random row\n",
     "sample_df=df.sample(1)\n",
     "# create a json object with the key as \"inputs\" and value as a list of values from the masked_text column of the sample_df dataframe\n",
-    "sample_json = {\"inputs\": sample_df[\"masked_text\"].tolist()}\n",
-    "# save the json object to a file named sample_score.json in the ./book-corpus-dataset folder\n",
-    "test_json = {\"inputs\": sample_df[\"masked_text\"].tolist()}\n",
+    "test_json = {\"inputs\": {\"input_string\": sample_df[\"masked_text\"].tolist()}}\n",
     "# save the json object to a file named sample_score.json in the ./book-corpus-dataset folder\n",
     "with open(os.path.join(\".\", \"book-corpus-dataset\",\"sample_score.json\"), \"w\") as f:\n",
     "    json.dump(test_json, f)\n",
@@ -239,6 +237,17 @@
     "response_df.head()"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# compare the predicted squences with the ground truth sequence\n",
+    "compare_df = pd.DataFrame({\"ground_truth_sequence\": sample_df[\"text\"].tolist()*5, \"predicted_sequence\": response_df[\"sequence\"].tolist(), \"score\": response_df[\"score\"].tolist()})\n",
+    "compare_df.head()"
+   ]
+  },
   {
    "attachments": {},
    "cell_type": "markdown",

sdk/python/foundation-models/system/inference/question-answering/question-answering-online-endpoint.ipynb

@@ -197,9 +197,7 @@
     "# pick 1 random row\n",
     "sample_df=df.sample(1)\n",
     "# create a json object with the key as \"inputs\" and value as a list of question-context pairs from columns of the sample_df dataframe\n",
-    "sample_json = {\"inputs\": [{\"question\": sample_df[\"question\"].to_list()[0], \"context\": sample_df[\"context\"].to_list()[0]}]}\n",
-    "# save the json object to a file named sample_score.json in the ./squad-dataset folder\n",
-    "test_json = {\"inputs\": [{\"question\": sample_df[\"question\"].to_list()[0], \"context\": sample_df[\"context\"].to_list()[0]}]}\n",
+    "test_json = {\"inputs\": {\"question\": sample_df[\"question\"].to_list(), \"context\": sample_df[\"context\"].to_list()}}\n",
     "# save the json object to a file named sample_score.json in the ./squad-dataset folder\n",
     "with open(os.path.join(\".\", \"squad-dataset\",\"sample_score.json\"), \"w\") as f:\n",
     "    json.dump(test_json, f)\n",
@@ -224,6 +222,18 @@
     "response_df.head()"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# compare the predicted answer with the actual answer\n",
+    "response_df = pd.DataFrame({\"predicted_answer\": [response_df[\"answer\"]]})\n",
+    "response_df[\"ground_truth_answer\"] = sample_df[\"answers\"].to_list()[0][\"text\"]\n",
+    "response_df.head()"
+   ]
+  },
   {
    "attachments": {},
    "cell_type": "markdown",

sdk/python/foundation-models/system/inference/question-answering/squad-dataset/sample_score.json

@@ -1 +1 @@
-{"inputs": [{"question": "What high profile controversial project was Von Neumann a prinipal of?", "context": "He was a pioneer of the application of operator theory to quantum mechanics, in the development of functional analysis, a principal member of the Manhattan Project and the Institute for Advanced Study in Princeton (as one of the few originally appointed), and a key figure in the development of game theory and the concepts of cellular automata, the universal constructor and the digital computer. He published 150 papers in his life; 60 in pure mathematics, 20 in physics, and 60 in applied mathematics. His last work, an unfinished manuscript written while in the hospital, was later published in book form as The Computer and the Brain."}]}
+{"inputs": {"question": ["What high profile controversial project was Von Neumann a prinipal of?"], "context": ["He was a pioneer of the application of operator theory to quantum mechanics, in the development of functional analysis, a principal member of the Manhattan Project and the Institute for Advanced Study in Princeton (as one of the few originally appointed), and a key figure in the development of game theory and the concepts of cellular automata, the universal constructor and the digital computer. He published 150 papers in his life; 60 in pure mathematics, 20 in physics, and 60 in applied mathematics. His last work, an unfinished manuscript written while in the hospital, was later published in book form as The Computer and the Brain."]}}

sdk/python/foundation-models/system/inference/summarization/news-summary-dataset/sample_score.json

@@ -1 +1 @@
-{"inputs": ["NEW YORK (CNN) -- Nearly a year after being beaten into a coma, Bryan Steinhauer said Wednesday he does not hate the Serbian basketball player witnesses said brutally assaulted him. Bryan Steinhauer, who was beaten into a coma, is making significant progress in his rehabilitation. \\\"I am not full of hate; hatred kills progress,\\\" he said. Appearing alongside his parents and doctors at New York\\'s Mount Sinai Hospital, Steinhauer struggled to speak but his message was clear. \\\"Please don\\'t feel sorry for me,\\\" he said. \\\"Tragedy leads to wisdom, and this experience has taught me so much about life.\\\"  Watch Steinhauer talk about his recovery \u00bb . The 22-year-old from Brooklyn was about to graduate and had a job lined up at accounting giant KPMG when he got into an argument last May that nearly cost him his life. According to police, witnesses said Steinhauer and college basketball player Miladin Kovacevic had exchanged harsh words at an upstate New York bar near Binghamton University after Steinhauer danced with the girlfriend of one of Kovacevic\\'s friends. The witnesses said the fight went outside the bar, where several men attacked Steinhauer, with Kovacevic beating him about the head, according to police. Kovacevic is 6-foot-9 and 260 pounds while Steinhauer was 130 pounds. Kovacevic was arrested but jumped bail and fled to Serbia with the help of Serbian consular officials in New York. The case strained relations between the United States and Serbia.Hillary Clinton intervened, first as U.S. senator from New York and later as secretary of state, as did Sen. Charles Schumer, D-New York, to make sure Kovacevic was prosecuted. Serbia has no extradition treaty with the United States, but Serbian officials arrested Kovacevic last October and are working on prosecuting him with the assistance of the district attorney of Broome County, where the beating took place. In addition, the Serbian government recently paid the Steinhauer family $900,000 in recognition of the misconduct of Serbian government officials and the financial burdens placed upon the Steinhauer family as a result of the beating. Steinhauer awoke from his coma last August, three months after the beating that left him with skull fractures, a severe brain injury and no memory of the attack. He weighed less than 100 pounds, could not speak or walk, and was fed through a tube, doctors said Wednesday. \\\"He had hemorrhages and contusions affecting almost every lobe of his brain,\\\" said Dr. Brian Greenwald. Working with specialists and boosted by the support of his family, Steinhauer surprised even his doctors in his quick recovery, they said. He now has outpatient rehabilitation four times a week, goes to a gym, and receives acupuncture treatments. While he can eat on his own now and is making significant progress, Steinhauer continues to undergo intensive therapy. Steinhauer says he doesn\\'t think about Kovacevic because he\\'s not concerned about other people. \\\"I\\'ve had a second birth and raising at Mount Sinai,\\\" he said. \\\"Live long and prosper.\\\""]}
+{"inputs": {"input_string": ["NEW YORK (CNN) -- Nearly a year after being beaten into a coma, Bryan Steinhauer said Wednesday he does not hate the Serbian basketball player witnesses said brutally assaulted him. Bryan Steinhauer, who was beaten into a coma, is making significant progress in his rehabilitation. \\\"I am not full of hate; hatred kills progress,\\\" he said. Appearing alongside his parents and doctors at New York\\'s Mount Sinai Hospital, Steinhauer struggled to speak but his message was clear. \\\"Please don\\'t feel sorry for me,\\\" he said. \\\"Tragedy leads to wisdom, and this experience has taught me so much about life.\\\"  Watch Steinhauer talk about his recovery \u00bb . The 22-year-old from Brooklyn was about to graduate and had a job lined up at accounting giant KPMG when he got into an argument last May that nearly cost him his life. According to police, witnesses said Steinhauer and college basketball player Miladin Kovacevic had exchanged harsh words at an upstate New York bar near Binghamton University after Steinhauer danced with the girlfriend of one of Kovacevic\\'s friends. The witnesses said the fight went outside the bar, where several men attacked Steinhauer, with Kovacevic beating him about the head, according to police. Kovacevic is 6-foot-9 and 260 pounds while Steinhauer was 130 pounds. Kovacevic was arrested but jumped bail and fled to Serbia with the help of Serbian consular officials in New York. The case strained relations between the United States and Serbia.Hillary Clinton intervened, first as U.S. senator from New York and later as secretary of state, as did Sen. Charles Schumer, D-New York, to make sure Kovacevic was prosecuted. Serbia has no extradition treaty with the United States, but Serbian officials arrested Kovacevic last October and are working on prosecuting him with the assistance of the district attorney of Broome County, where the beating took place. In addition, the Serbian government recently paid the Steinhauer family $900,000 in recognition of the misconduct of Serbian government officials and the financial burdens placed upon the Steinhauer family as a result of the beating. Steinhauer awoke from his coma last August, three months after the beating that left him with skull fractures, a severe brain injury and no memory of the attack. He weighed less than 100 pounds, could not speak or walk, and was fed through a tube, doctors said Wednesday. \\\"He had hemorrhages and contusions affecting almost every lobe of his brain,\\\" said Dr. Brian Greenwald. Working with specialists and boosted by the support of his family, Steinhauer surprised even his doctors in his quick recovery, they said. He now has outpatient rehabilitation four times a week, goes to a gym, and receives acupuncture treatments. While he can eat on his own now and is making significant progress, Steinhauer continues to undergo intensive therapy. Steinhauer says he doesn\\'t think about Kovacevic because he\\'s not concerned about other people. \\\"I\\'ve had a second birth and raising at Mount Sinai,\\\" he said. \\\"Live long and prosper.\\\""]}}

sdk/python/foundation-models/system/inference/summarization/summarization-online-endpoint.ipynb

@@ -199,7 +199,7 @@
     "# create a json object with the key as \"inputs\" and value as a list of values from the article column of the sample_df dataframe\n",
     "sample_json = {\"inputs\": sample_df[\"article\"].tolist()}\n",
     "# save the json object to a file named sample_score.json in the ./news-summary-dataset folder\n",
-    "test_json = {\"inputs\": sample_df[\"article\"].tolist()}\n",
+    "test_json = {\"inputs\": {\"input_string\": sample_df[\"article\"].tolist()}}\n",
     "# save the json object to a file named sample_score.json in the ./news-summary-dataset folder\n",
     "with open(os.path.join(\".\", \"news-summary-dataset\",\"sample_score.json\"), \"w\") as f:\n",
     "    json.dump(test_json, f)\n",
@@ -224,6 +224,18 @@
     "response_df.head()"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# compare the true summary with the predicted summary\n",
+    "response_df.rename(columns={\"summary_text\": \"predicted_summary\"}, inplace=True)\n",
+    "response_df[\"ground_truth_summary\"] = [sample_df[\"highlights\"].tolist()[0]]\n",
+    "response_df.head()"
+   ]
+  },
   {
    "attachments": {},
    "cell_type": "markdown",

sdk/python/foundation-models/system/inference/text-classification/entailment-contradiction-online.ipynb

@@ -225,7 +225,7 @@
     "sample_df = sample_df.reset_index(drop=True)\n",
     "\n",
     "# save the json object to a file named sample_score.json in the \n",
-    "test_json = {\"inputs\": sample_df[\"text\"].tolist()}\n",
+    "test_json = {\"inputs\": {\"input_string\": sample_df[\"text\"].tolist()}}\n",
     "# save the json object to a file named sample_score.json in the ./glue-mnli-dataset folder\n",
     "with open(os.path.join(\".\", dataset_dir, score_file), \"w\") as f:\n",
     "    json.dump(test_json, f)\n",
@@ -276,25 +276,10 @@
    "cell_type": "code",
    "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "...."
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "workspace_ml_client.online_endpoints.begin_delete(name=online_endpoint_name).wait()"
    ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
   }
  ],
  "metadata": {

sdk/python/foundation-models/system/inference/text-classification/glue-mnli-dataset/sample_score.json

@@ -1 +1 @@
-{"inputs": ["[CLS] These two separate increases in the fees lawyers pay is a step closer to achieving those two goals, McMorrow said in written statement.  [SEP] Two separate fees that lawyers pay are decreasing. [SEP]", "[CLS] In this particular case, let's just say when the original advice was given the wheel was spinning, but the hamster had gone. [SEP] There has been advice given.  [SEP]", "[CLS] The gardens are popular with children who enjoy watching the marionette shows, riding donkeys, and sailing boats on the circular ponds. [SEP] Children really like the marionette shows in the gardens. [SEP]", "[CLS] this is the kid who who really you you know barely made it through high school [SEP] He passed high school with straight A's.  [SEP]", "[CLS] good don't ever drink Scotch it's terrible i quit drinking Scotch when i found out about that but anyway but uh as far as as far as you know Central and South America we our policy pretty much uh it depends on who we're what government we're buying down there at the particular time [SEP] I no longer drink Scotch. [SEP]"]}
+{"inputs": {"input_string": ["[CLS] These two separate increases in the fees lawyers pay is a step closer to achieving those two goals, McMorrow said in written statement.  [SEP] Two separate fees that lawyers pay are decreasing. [SEP]", "[CLS] In this particular case, let's just say when the original advice was given the wheel was spinning, but the hamster had gone. [SEP] There has been advice given.  [SEP]", "[CLS] The gardens are popular with children who enjoy watching the marionette shows, riding donkeys, and sailing boats on the circular ponds. [SEP] Children really like the marionette shows in the gardens. [SEP]", "[CLS] this is the kid who who really you you know barely made it through high school [SEP] He passed high school with straight A's.  [SEP]", "[CLS] good don't ever drink Scotch it's terrible i quit drinking Scotch when i found out about that but anyway but uh as far as as far as you know Central and South America we our policy pretty much uh it depends on who we're what government we're buying down there at the particular time [SEP] I no longer drink Scotch. [SEP]"]}}

sdk/python/foundation-models/system/inference/text-generation/book-corpus-dataset/sample_score.json

@@ -1 +1 @@
-{"inputs": ["rocco noticed the almost defeated look on her lovely face and did n\\'t like it ."]}
+{"inputs": {"input_string": ["rocco noticed the almost defeated look on her lovely face and did n\\'t like it ."]}}

sdk/python/foundation-models/system/inference/text-generation/text-generation-online-endpoint.ipynb

@@ -195,7 +195,7 @@
     "# create a json object with the key as \"inputs\" and value as a list of values from the article column of the sample_df dataframe\n",
     "sample_json = {\"inputs\": sample_df[\"text\"].tolist()}\n",
     "# save the json object to a file named sample_score.json in the ./book-corpus-dataset folder\n",
-    "test_json = {\"inputs\": sample_df[\"text\"].tolist()}\n",
+    "test_json = {\"inputs\": {\"input_string\": sample_df[\"text\"].tolist()}}\n",
     "# save the json object to a file named sample_score.json in the ./book-corpus-dataset folder\n",
     "with open(os.path.join(\".\", \"book-corpus-dataset\",\"sample_score.json\"), \"w\") as f:\n",
     "    json.dump(test_json, f)\n",

sdk/python/foundation-models/system/inference/token-classification/Jean-Baptiste-wikiner_fr/sample_score.json

@@ -1 +1 @@
-{"inputs": ["Pour la premi\u00e8re \u00e9dition de la Coupe du monde , la comp\u00e9tition se d\u00e9roule en Uruguay \u00e0 Montevideo en 1930 ."]}
+{"inputs": {"input_string": ["Il est difficile de se rendre compte de la taille de la Maison Blanche , car une grande partie est souterraine ou cach\u00e9e par le paysage ."]}}

sdk/python/foundation-models/system/inference/token-classification/token-classification-online-endpoint.ipynb

@@ -195,7 +195,7 @@
     "# create a json object with the key as \"inputs\" and value as a list of values from the en column of the sample_df dataframe\n",
     "sample_json = {\"inputs\": sample_df[\"text\"].tolist()}\n",
     "# save the json object to a file named sample_score.json in the ./Jean-Baptiste-wikiner_fr folder\n",
-    "test_json = {\"inputs\": sample_df[\"text\"].tolist()}\n",
+    "test_json = {\"inputs\": {\"input_string\": sample_df[\"text\"].tolist()}}\n",
     "# save the json object to a file named sample_score.json in the ./Jean-Baptiste-wikiner_fr folder\n",
     "with open(os.path.join(\".\", \"Jean-Baptiste-wikiner_fr\",\"sample_score.json\"), \"w\") as f:\n",
     "    json.dump(test_json, f)\n",
@@ -220,6 +220,22 @@
     "response_df.head()"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# compare the predicted labels with the actual labels\n",
+    "num_tokens = len(sample_df[\"text\"].tolist()[0].split())\n",
+    "predicted_labels = ['O']*num_tokens\n",
+    "for col in response_df.columns:\n",
+    "    prediction = response_df[col].tolist()[0]\n",
+    "    predicted_labels[prediction[\"index\"]-1] = prediction[\"entity\"]\n",
+    "compare_df = pd.DataFrame({\"ground_truth_labels\": sample_df[\"ner_tags_str\"].tolist(), \"predicted_labels\": [predicted_labels]})\n",
+    "compare_df.head()"
+   ]
+  },
   {
    "attachments": {},
    "cell_type": "markdown",