Skip to content

Commit

Permalink
Add aka.ms aliases for files downloaded from Azure blobs (#4523)
Browse files Browse the repository at this point in the history 
* update tlcresources links

* Update more links

* update more links

* Undo changes in Samples project
  • Loading branch information
@yaeldMS @codemzs
yaeldMS authored and codemzs committed Dec 26, 2019
1 parent 6ae3a3f commit 78c406c
Show file tree
Hide file tree
Showing 7 changed files with 21 additions and 36 deletions.
5 changes: 1 addition & 4 deletions ...namic/Trainers/MulticlassClassification/ImageClassification/ImageClassificationDefault.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -242,10 +242,7 @@ public static string DownloadImageSet(string imagesDownloadFolder)

//SINGLE SMALL FLOWERS IMAGESET (200 files)
string fileName = "flower_photos_small_set.zip";
string url = $"https://mlnetfilestorage.file.core.windows.net/" +
$"imagesets/flower_images/flower_photos_small_set.zip?st=2019-08-" +
$"07T21%3A27%3A44Z&se=2030-08-08T21%3A27%3A00Z&sp=rl&sv=2018-03-" +
$"28&sr=f&sig=SZ0UBX47pXD0F1rmrOM%2BfcwbPVob8hlgFtIlN89micM%3D";
string url = $"https://aka.ms/mlnet-resources/datasets/flower_photos_small_set.zip";

Download(url, imagesDownloadFolder, fileName);
UnZip(Path.Combine(imagesDownloadFolder, fileName), imagesDownloadFolder);
Expand Down
2 changes: 1 addition & 1 deletion ...ssClassification/ImageClassification/LearningRateSchedulingCifarResnetTransferLearning.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -271,7 +271,7 @@ public static string DownloadImageSet(string imagesDownloadFolder)
// get a set of images to teach the network about the new classes
// CIFAR dataset ( 50000 train images and 10000 test images )
string fileName = "cifar10.zip";
string url = $"https://tlcresources.blob.core.windows.net/" +
string url = $"https://aka.ms/mlnet-resources/" +
"datasets/cifar10.zip";

Download(url, imagesDownloadFolder, fileName);
Expand Down
5 changes: 1 addition & 4 deletions .../MulticlassClassification/ImageClassification/ResnetV2101TransferLearningEarlyStopping.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -230,10 +230,7 @@ public static string DownloadImageSet(string imagesDownloadFolder)

//SINGLE SMALL FLOWERS IMAGESET (200 files)
string fileName = "flower_photos_small_set.zip";
string url = $"https://mlnetfilestorage.file.core.windows.net/" +
$"imagesets/flower_images/flower_photos_small_set.zip?st=2019-08-" +
$"07T21%3A27%3A44Z&se=2030-08-08T21%3A27%3A00Z&sp=rl&sv=2018-03-" +
$"28&sr=f&sig=SZ0UBX47pXD0F1rmrOM%2BfcwbPVob8hlgFtIlN89micM%3D";
string url = $"https://aka.ms/mlnet-resources/datasets/flower_photos_small_set.zip";

Download(url, imagesDownloadFolder, fileName);
UnZip(Path.Combine(imagesDownloadFolder, fileName), imagesDownloadFolder);
Expand Down
5 changes: 1 addition & 4 deletions ...MulticlassClassification/ImageClassification/ResnetV2101TransferLearningTrainTestSplit.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -251,10 +251,7 @@ public static string DownloadImageSet(string imagesDownloadFolder)

//SINGLE SMALL FLOWERS IMAGESET (200 files)
string fileName = "flower_photos_small_set.zip";
string url = $"https://mlnetfilestorage.file.core.windows.net/" +
$"imagesets/flower_images/flower_photos_small_set.zip?st=2019-08-" +
$"07T21%3A27%3A44Z&se=2030-08-08T21%3A27%3A00Z&sp=rl&sv=2018-03-" +
$"28&sr=f&sig=SZ0UBX47pXD0F1rmrOM%2BfcwbPVob8hlgFtIlN89micM%3D";
string url = $"https://aka.ms/mlnet-resources/datasets/flower_photos_small_set.zip";

Download(url, imagesDownloadFolder, fileName);
UnZip(Path.Combine(imagesDownloadFolder, fileName), imagesDownloadFolder);
Expand Down
9 changes: 3 additions & 6 deletions test/Microsoft.ML.AutoML.Tests/DatasetUtil.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ internal static class DatasetUtil

public static IDataView GetUciAdultDataView()
{
if(_uciAdultDataView == null)
if (_uciAdultDataView == null)
{
var context = new MLContext();
var uciAdultDataFile = DownloadUciAdultDataset();
Expand Down Expand Up @@ -69,7 +69,7 @@ private static string DownloadIfNotExists(string baseGitPath, string dataFile)
}
}
}
catch(Exception)
catch (Exception)
{
}
}
Expand Down Expand Up @@ -146,10 +146,7 @@ public static IEnumerable<ImageData> LoadImagesFromDirectory(string folder)
public static string DownloadImageSet(string imagesDownloadFolder)
{
string fileName = "flower_photos_tiny_set_for_unit_tests.zip";
string url = $"https://mlnetfilestorage.file.core.windows.net/imagesets" +
$"/flower_images/flower_photos_tiny_set_for_unit_tests.zip?st=2019" +
$"-08-29T00%3A07%3A21Z&se=2030-08-30T00%3A07%3A00Z&sp=rl&sv=2018" +
$"-03-28&sr=f&sig=N8HbLziTcT61kstprNLmn%2BDC0JoMrNwo6yRWb3hLLag%3D";
string url = $"https://aka.ms/mlnet-resources/datasets/flower_photos_tiny_set_for_unit_test.zip";

Download(url, imagesDownloadFolder, fileName);
UnZip(Path.Combine(imagesDownloadFolder, fileName), imagesDownloadFolder);
Expand Down
5 changes: 1 addition & 4 deletions test/Microsoft.ML.Benchmarks/ImageClassificationBench.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -139,10 +139,7 @@ public static string DownloadImageSet(string imagesDownloadFolder)

//SINGLE SMALL FLOWERS IMAGESET (200 files)
string fileName = "flower_photos_small_set.zip";
string url = $"https://mlnetfilestorage.file.core.windows.net/" +
$"imagesets/flower_images/flower_photos_small_set.zip?st=2019-08-" +
$"07T21%3A27%3A44Z&se=2030-08-08T21%3A27%3A00Z&sp=rl&sv=2018-03-" +
$"28&sr=f&sig=SZ0UBX47pXD0F1rmrOM%2BfcwbPVob8hlgFtIlN89micM%3D";
string url = $"https://aka.ms/mlnet-resources/datasets/flower_photos_small_set.zip/";

Download(url, imagesDownloadFolder, fileName);
UnZip(Path.Combine(imagesDownloadFolder, fileName), imagesDownloadFolder);
Expand Down
26 changes: 13 additions & 13 deletions test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/TensorflowTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1316,12 +1316,12 @@ internal bool ShouldReuse(string workspacePath, string trainSetBottleneckCachedV
}
return isReuse;
}

internal (string, string, string, bool) getInitialParameters(ImageClassificationTrainer.Architecture arch, string finalImagesFolderName)
{
string trainSetBottleneckCachedValuesFileName = "TrainsetCached_" + finalImagesFolderName + "_" + (int) arch;
string validationSetBottleneckCachedValuesFileName = "validationsetCached_" + finalImagesFolderName + "_" + (int) arch;
string workspacePath = Path.Combine(TensorFlowScenariosTestsFixture.parentWorkspacePath, finalImagesFolderName + "_" + (int) arch);
string trainSetBottleneckCachedValuesFileName = "TrainsetCached_" + finalImagesFolderName + "_" + (int)arch;
string validationSetBottleneckCachedValuesFileName = "validationsetCached_" + finalImagesFolderName + "_" + (int)arch;
string workspacePath = Path.Combine(TensorFlowScenariosTestsFixture.parentWorkspacePath, finalImagesFolderName + "_" + (int)arch);
bool isReuse = ShouldReuse(workspacePath, trainSetBottleneckCachedValuesFileName, validationSetBottleneckCachedValuesFileName);
return (trainSetBottleneckCachedValuesFileName, validationSetBottleneckCachedValuesFileName, workspacePath, isReuse);
}
Expand Down Expand Up @@ -1368,9 +1368,9 @@ public void TensorFlowImageClassification(ImageClassificationTrainer.Architectur
.Transform(testDataset);

// Check if the bottleneck cached values already exist
var (trainSetBottleneckCachedValuesFileName, validationSetBottleneckCachedValuesFileName,
workspacePath, isReuse) = getInitialParameters(arch, finalImagesFolderName);
var (trainSetBottleneckCachedValuesFileName, validationSetBottleneckCachedValuesFileName,
workspacePath, isReuse) = getInitialParameters(arch, finalImagesFolderName);

var options = new ImageClassificationTrainer.Options()
{
FeatureColumnName = "Image",
Expand Down Expand Up @@ -1424,7 +1424,7 @@ public void TensorFlowImageClassification(ImageClassificationTrainer.Architectur

string[] directories = Directory.GetDirectories(fullImagesetFolderPath);
string[] labels = new string[directories.Length];
for(int j = 0; j < labels.Length; j++)
for (int j = 0; j < labels.Length; j++)
{
var dir = new DirectoryInfo(directories[j]);
labels[j] = dir.Name;
Expand Down Expand Up @@ -1813,21 +1813,21 @@ public static string DownloadImageSet(string imagesDownloadFolder)
{
string fileName = "flower_photos_tiny_set_for_unit_tests.zip";
string filenameAlias = "FPTSUT"; // FPTSUT = flower photos tiny set for unit tests
string url = "https://aka.ms/mlnet-resources/flower_photos_tiny_set_for_unit_test.zip";
string url = "https://aka.ms/mlnet-resources/datasets/flower_photos_tiny_set_for_unit_test.zip";

Download(url, imagesDownloadFolder, fileName);
UnZip(Path.Combine(imagesDownloadFolder, fileName), imagesDownloadFolder);
// Sometimes tests fail because the path is too long. So rename the dataset folder to a shorter directory.
if(!Directory.Exists(Path.Combine(imagesDownloadFolder, filenameAlias)))
Directory.Move(Path.Combine(imagesDownloadFolder, Path.GetFileNameWithoutExtension(fileName)), Path.Combine(imagesDownloadFolder,"FPTSUT"));
if (!Directory.Exists(Path.Combine(imagesDownloadFolder, filenameAlias)))
Directory.Move(Path.Combine(imagesDownloadFolder, Path.GetFileNameWithoutExtension(fileName)), Path.Combine(imagesDownloadFolder, "FPTSUT"));
return filenameAlias;
}

public static string DownloadBadImageSet(string imagesDownloadFolder)
{
string fileName = "CatsVsDogs_tiny_for_unit_tests.zip";
string url = $"https://tlcresources.blob.core.windows.net/datasets/" +
$"CatsVsDogs_tiny_for_unit_tests.zip";
string url = $"https://aka.ms/mlnet-resources/datasets/" +
$"CatsVsDogs_tiny_for_unit_tests.zip";

Download(url, imagesDownloadFolder, fileName);
UnZip(Path.Combine(imagesDownloadFolder, fileName), imagesDownloadFolder);
Expand Down

0 comments on commit 78c406c

Please sign in to comment.