Major improvements to External Storage Support

.github/workflows/External-Storage-Tests.yml

@@ -0,0 +1,100 @@
+name: External Storage Tests
+
+on: 
+  pull_request:
+    branches:
+      - 'master'
+      - 'release-*'
+  push:
+    branches: 
+      - 'master'
+      - 'beta'
+      - 'release-*'
+
+jobs:
+  build:
+    env:
+      GIT_REF: ${{ github.ref }}
+      GIT_SHA: ${{ github.sha }}
+      Configuration: Release
+      SolutionFile: dotnet\DotNetStandardClasses.sln
+
+    runs-on: windows-latest
+    environment: external-storage-tests
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+
+    - name: Install .NET Core 3.1
+      uses: actions/setup-dotnet@v1
+      with:
+        dotnet-version: '3.1.x'
+
+    - name: Install .NET 6
+      uses: actions/setup-dotnet@v1.7.2
+      with:
+        dotnet-version: '6.0.100-preview.3.21202.5'
+
+    - uses: actions/setup-dotnet@v1
+      with:
+        source-url: https://nuget.pkg.github.com/genexuslabs/index.json
+      env:
+        NUGET_AUTH_TOKEN: ${{secrets.GITHUB_TOKEN}}
+
+    - name: Calculate environment variables
+      run: |     
+        $IsPrerelease = !($Env:GIT_REF -match 'release-[0-9]+(?:\.[0-9]+)?$')
+        echo "IsPrerelease=$IsPrerelease" >> $env:GITHUB_ENV
+
+        $COMMIT_NUMBER = @($(git rev-list --count origin/master..), $(git rev-list --count HEAD))[$IsPrerelease]
+
+        echo "COMMIT_NUMBER=$COMMIT_NUMBER" >> $env:GITHUB_ENV
+
+    - name: Calculate package version
+      env:
+        PackageVersionString: ./.github/generatePackageVersion.ps1
+      run: |
+        $NuGetPackageVersion = & "$Env:PackageVersionString"
+
+        Write-Output "Packge version to be used: $NuGetPackageVersion"
+
+        echo "NuGetPackageVersion=$NuGetPackageVersion" >> $env:GITHUB_ENV
+
+    - name: Restore packages
+      run: dotnet restore $Env:SolutionFile
+
+    - name: Build
+      run: dotnet build $Env:SolutionFile --no-restore --configuration $Env:Configuration
+
+    - name: Test External Storage
+      run: |
+        $Env:AWSS3_TEST_ENABLED="true"
+        $Env:STORAGE_AWSS3_ACCESS_KEY="${{ secrets.AWSS3_ACCESS_KEY }}"
+        $Env:STORAGE_AWSS3_SECRET_KEY="${{ secrets.AWSS3_SECRET_KEY }}"
+        $Env:STORAGE_AWSS3_BUCKET_NAME="genexus-s3-test"
+        $Env:STORAGE_AWSS3_FOLDER_NAME="gxclasses"
+        $Env:STORAGE_AWSS3_REGION="us-east-1"
+        $Env:IBMCOS_TEST_ENABLED="true"
+        $Env:STORAGE_IBMCOS_ACCESS_KEY="${{ secrets.IBMCOS_ACCESS_KEY }}"
+        $Env:STORAGE_IBMCOS_SECRET_KEY="${{ secrets.IBMCOS_SECRET_KEY }}"
+        $Env:STORAGE_IBMCOS_BUCKET_NAME="gxclasses-unit-tests"
+        $Env:STORAGE_IBMCOS_FOLDER_NAME="tests"
+        $Env:STORAGE_IBMCOS_REGION="us-south"
+        $Env:AZUREBS_TEST_ENABLED="true"
+        $Env:STORAGE_AZUREBS_ACCESS_KEY="${{ secrets.AZUREBS_ACCESS_KEY }}"
+        $Env:STORAGE_AZUREBS_ACCOUNT_NAME="${{ secrets.AZUREBS_ACCOUNT_NAME }}"
+        $Env:STORAGE_AZUREBS_FOLDER_NAME="tests"
+        $Env:STORAGE_AZUREBS_PUBLIC_CONTAINER_NAME="contluispublic"
+        $Env:STORAGE_AZUREBS_PRIVATE_CONTAINER_NAME="contluisprivate"
+        $Env:GOOGLECS_TEST_ENABLED="true"
+        $Env:STORAGE_GOOGLECS_KEY='${{ secrets.GOOGLECS_KEY }}'
+        $Env:STORAGE_GOOGLECS_PROJECT_ID="gxjavacloudstorageunittests"
+        $Env:STORAGE_GOOGLECS_BUCKET_NAME="javaclasses-unittests"
+        $Env:STORAGE_GOOGLECS_FOLDER_NAME="gxclasses"
+        $Env:STORAGE_GOOGLECS_APPLICATION_NAME="gxjavacloudstorageunittests"
+
+        dotnet test $Env:SolutionFile --no-restore --no-build --configuration $Env:Configuration
+

dotnet/DotNetStandardClasses.sln

@@ -65,8 +65,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "GXAmazonS3", "src\dotnetfra
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "GXAzureStorage", "src\dotnetframework\Providers\Storage\GXAzureStorage\GXAzureStorage.csproj", "{F6372249-AF37-4455-B572-5BDF8DE2ACC8}"
 EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "GXBluemix", "src\dotnetframework\Providers\Storage\GXBluemix\GXBluemix.csproj", "{FCA5536D-542E-480E-92B0-316E0290553B}"
-EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "GXGoogleCloud", "src\dotnetframework\Providers\Storage\GXGoogleCloud\GXGoogleCloud.csproj", "{E9072D95-D116-4D4B-B981-46146BCDE052}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "GXOpenStack", "src\dotnetframework\Providers\Storage\GXOpenStack\GXOpenStack.csproj", "{64E958D8-CE7B-482D-B339-3B52E26CA4AC}"
@@ -241,10 +239,6 @@ Global
 		{F6372249-AF37-4455-B572-5BDF8DE2ACC8}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{F6372249-AF37-4455-B572-5BDF8DE2ACC8}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{F6372249-AF37-4455-B572-5BDF8DE2ACC8}.Release|Any CPU.Build.0 = Release|Any CPU
-		{FCA5536D-542E-480E-92B0-316E0290553B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{FCA5536D-542E-480E-92B0-316E0290553B}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{FCA5536D-542E-480E-92B0-316E0290553B}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{FCA5536D-542E-480E-92B0-316E0290553B}.Release|Any CPU.Build.0 = Release|Any CPU
 		{E9072D95-D116-4D4B-B981-46146BCDE052}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{E9072D95-D116-4D4B-B981-46146BCDE052}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{E9072D95-D116-4D4B-B981-46146BCDE052}.Release|Any CPU.ActiveCfg = Release|Any CPU
@@ -378,7 +372,6 @@ Global
 		{F82842DA-F15E-49C5-993E-4C269818FF1F} = {F900A4AD-7249-41B4-B918-CB9E8C73747C}
 		{3701565F-1A95-4592-B90D-291CCDE5505D} = {F82842DA-F15E-49C5-993E-4C269818FF1F}
 		{F6372249-AF37-4455-B572-5BDF8DE2ACC8} = {F82842DA-F15E-49C5-993E-4C269818FF1F}
-		{FCA5536D-542E-480E-92B0-316E0290553B} = {F82842DA-F15E-49C5-993E-4C269818FF1F}
 		{E9072D95-D116-4D4B-B981-46146BCDE052} = {F82842DA-F15E-49C5-993E-4C269818FF1F}
 		{64E958D8-CE7B-482D-B339-3B52E26CA4AC} = {F82842DA-F15E-49C5-993E-4C269818FF1F}
 		{A14C2C2C-ACE3-4712-A527-E4E5F02729FA} = {F900A4AD-7249-41B4-B918-CB9E8C73747C}

dotnet/src/dotnetcore/GxClasses/GxClasses.csproj

@@ -78,6 +78,7 @@
     <Compile Include="..\..\dotnetframework\GxClasses\Services\Search\GXSearch.cs" Link="Services\Search\GXSearch.cs" />
     <Compile Include="..\..\dotnetframework\GxClasses\Security\GxSecurityProvider.cs" Link="Security\GxSecurityProvider.cs" />
     <Compile Include="..\..\dotnetframework\GxClasses\Services\Storage\GXServices.cs" Link="Services\Storage\GXServices.cs" />
+		 <Compile Include="..\..\dotnetframework\GxClasses\Services\Storage\ExternalProviderBase.cs" Link="Services\Storage\ExternalProviderBase.cs" />
     <Compile Include="..\..\dotnetframework\GxClasses\Domain\GxSession.cs" Link="Domain\GxSession.cs" />
     <Compile Include="..\..\dotnetframework\GxClasses\Model\GXSilentTrn.cs" Link="Model\GXSilentTrn.cs" />
     <Compile Include="..\..\dotnetframework\GxClasses\Services\Caching\GxSmartCacheProvider.cs" Link="Services\Caching\GxSmartCacheProvider.cs" />

dotnet/src/dotnetframework/GxClasses/Configuration/ExternalStorage.cs

@@ -95,7 +95,7 @@ public bool Connect(string profileName, GXProperties properties, ref GxStoragePr
 
 		private void preprocess(String name, GXProperties properties)
 		{
-			string className = null;
+			string className;
 
 			switch (name)
 			{
@@ -104,37 +104,27 @@ private void preprocess(String name, GXProperties properties)
 					className = "GeneXus.Storage.GXAmazonS3.ExternalProviderS3";
 					SetDefaultProperty(properties, "STORAGE_PROVIDER_REGION", "us-east-1");
 					SetDefaultProperty(properties, "STORAGE_ENDPOINT", "s3.amazonaws.com");
-					SetEncryptProperty(properties, "STORAGE_PROVIDER_ACCESSKEYID");
-					SetEncryptProperty(properties, "STORAGE_PROVIDER_SECRETACCESSKEY");
-					SetEncryptProperty(properties, "BUCKET_NAME");
+					SetEncryptedProperty(properties, "STORAGE_PROVIDER_ACCESSKEYID");
+					SetEncryptedProperty(properties, "STORAGE_PROVIDER_SECRETACCESSKEY");
+					SetEncryptedProperty(properties, "BUCKET_NAME");
 					break;
 
 				case "AZURESTORAGE":
 					className = "GeneXus.Storage.GXAzureStorage.AzureStorageExternalProvider";
-					SetEncryptProperty(properties, "PUBLIC_CONTAINER_NAME");
-					SetEncryptProperty(properties, "PRIVATE_CONTAINER_NAME");
-					SetEncryptProperty(properties, "ACCOUNT_NAME");
-					SetEncryptProperty(properties, "ACCESS_KEY");
+					SetEncryptedProperty(properties, "PUBLIC_CONTAINER_NAME");
+					SetEncryptedProperty(properties, "PRIVATE_CONTAINER_NAME");
+					SetEncryptedProperty(properties, "ACCOUNT_NAME");
+					SetEncryptedProperty(properties, "ACCESS_KEY");
 					break;
-
-				case "BLUEMIXSTORAGE":
-					className = "GeneXus.Storage.GXBluemix.ExternalProviderBluemix";
-					SetDefaultProperty(properties, "SERVER_URL", "https://identity.open.softlayer.com");
-					SetDefaultProperty(properties, "STORAGE_PROVIDER_REGION", "dallas");
-					SetEncryptProperty(properties, "PUBLIC_BUCKET_NAME");
-					SetEncryptProperty(properties, "PRIVATE_BUCKET_NAME");
-					SetEncryptProperty(properties, "STORAGE_PROVIDER_USER");
-					SetEncryptProperty(properties, "STORAGE_PROVIDER_PASSWORD");
-					break;
-
+
 				//case "BOX":
 				//	className = "{class}";
 				//	break;
 
 				case "GOOGLE":
 					className = "GeneXus.Storage.GXGoogleCloud.ExternalProviderGoogle";
-					SetEncryptProperty(properties, "KEY");
-					SetEncryptProperty(properties, "BUCKET_NAME");
+					SetEncryptedProperty(properties, "KEY");
+					SetEncryptedProperty(properties, "BUCKET_NAME");
 					break;
 
 				//case "IBMCOS":
@@ -146,9 +136,9 @@ private void preprocess(String name, GXProperties properties)
 
 				case "OPENSTACKSTORAGE":
 					className = "GeneXus.Storage.GXOpenStack.ExternalProviderOpenStack";
-					SetEncryptProperty(properties, "BUCKET_NAME");
-					SetEncryptProperty(properties, "STORAGE_PROVIDER_USER");
-					SetEncryptProperty(properties, "STORAGE_PROVIDER_PASSWORD");
+					SetEncryptedProperty(properties, "BUCKET_NAME");
+					SetEncryptedProperty(properties, "STORAGE_PROVIDER_USER");
+					SetEncryptedProperty(properties, "STORAGE_PROVIDER_PASSWORD");
 					break;
 
 				default:
@@ -171,7 +161,7 @@ private void SetDefaultProperty(GXProperties properties, String prop, String val
 				properties.Set(prop, value);
 		}
 
-		private void SetEncryptProperty(GXProperties properties, String prop)
+		private void SetEncryptedProperty(GXProperties properties, String prop)
 		{
 			String value = properties.Get(prop);
 			if (string.IsNullOrEmpty(value))

dotnet/src/dotnetframework/GxClasses/Core/GXApplication.cs

@@ -679,13 +679,10 @@ public static bool GetHttpRequestPostedFile(IGxContext gxContext, string varName
 			if (httpContext != null)
 			{
 				HttpPostedFile pf = httpContext.Request.GetFile(varName);
-				if (pf != null)
+				if (pf != null && pf.ContentLength > 0)
 				{
-#pragma warning disable SCS0018 // Path traversal: injection possible in {1} argument passed to '{0}'
-					FileInfo fi = new FileInfo(pf.FileName);
-#pragma warning restore SCS0018 // Path traversal: injection possible in {1} argument passed to '{0}'
 					string tempDir = Preferences.getTMP_MEDIA_PATH();
-					string ext = fi.Extension;
+					string ext = Path.GetExtension(pf.FileName);
 					if (ext != null)
 						ext = ext.TrimStart('.');
 					string filePath = FileUtil.getTempFileName(tempDir);
@@ -695,7 +692,7 @@ public static bool GetHttpRequestPostedFile(IGxContext gxContext, string varName
 					string fileGuid = GxUploadHelper.GetUploadFileGuid();
 					fileToken = GxUploadHelper.GetUploadFileId(fileGuid);
 
-					GxUploadHelper.CacheUploadFile(fileGuid, filePath, Path.GetFileName(pf.FileName), ext, file, gxContext);
+					GxUploadHelper.CacheUploadFile(fileGuid, Path.GetFileName(pf.FileName), ext, file, gxContext);
 
 				return true;
 				}

dotnet/src/dotnetframework/GxClasses/Core/GXUtils.cs

@@ -1116,12 +1116,13 @@ public bool Upload(string filefullpath, string storageobjectfullname, GxFile upl
 			try
 			{				
 				ValidProvider();
+				GxFileType acl = GxFileType.PublicRead;
 				if (String.IsNullOrEmpty(storageobjectfullname))
 				{
 					storageobjectfullname = Path.GetFileName(filefullpath);
 				}				
-				string url = provider.Upload(filefullpath, storageobjectfullname, GxFileType.Public);
-				uploadedFile.FileInfo = new GxExternalFileInfo(storageobjectfullname, url, provider);
+				string url = provider.Upload(filefullpath, storageobjectfullname, acl);
+				uploadedFile.FileInfo = new GxExternalFileInfo(storageobjectfullname, url, provider, acl);
 				return true;
 			}
 			catch (Exception ex)
@@ -1139,8 +1140,9 @@ public bool UploadPrivate(string filefullpath, string storageobjectfullname, GxF
 				{
 					storageobjectfullname=Path.GetFileName(filefullpath);
 				}
-				string url = provider.Upload(filefullpath, storageobjectfullname, GxFileType.Private);
-				uploadedFile.FileInfo = new GxExternalFileInfo(storageobjectfullname, url, provider);
+				GxFileType acl = GxFileType.Private;
+				string url = provider.Upload(filefullpath, storageobjectfullname, acl);
+				uploadedFile.FileInfo = new GxExternalFileInfo(storageobjectfullname, url, provider, acl);
 				return true;
 			}
 			catch (Exception ex)
@@ -1160,7 +1162,7 @@ public bool Download(string storageobjectfullname, GxFile localFile, GXBaseColle
                     destFileName = localFile.GetAbsoluteName();
                 else
                     destFileName = Path.Combine(GxContext.StaticPhysicalPath(), localFile.Source);
-                provider.Download(storageobjectfullname, destFileName, GxFileType.Public);
+                provider.Download(storageobjectfullname, destFileName, GxFileType.PublicRead);
                 return true;
             }
 			catch (Exception ex)
@@ -1197,15 +1199,16 @@ public bool Get(string storageobjectfullname, GxFile externalFile, GXBaseCollect
 			try
 			{
 				ValidProvider();
-				string url = provider.Get(storageobjectfullname, GxFileType.Public, 0);
+				GxFileType acl = GxFileType.PublicRead;
+				string url = provider.Get(storageobjectfullname, acl, 0);
 				if (String.IsNullOrEmpty(url))
 				{
 					GXUtil.ErrorToMessages("Get Error", "File doesn't exists", messages);
 					return false;
 				}
 				else
 				{
-					externalFile.FileInfo = new GxExternalFileInfo(storageobjectfullname, url, provider);
+					externalFile.FileInfo = new GxExternalFileInfo(storageobjectfullname, url, provider, acl);
 					return true;
 				}
 			}
@@ -1222,15 +1225,16 @@ public bool GetPrivate(string storageobjectfullname, GxFile externalFile, int ex
             try
             {
 				ValidProvider();
-				string url = provider.Get(storageobjectfullname, GxFileType.Private, expirationMinutes);
+				GxFileType acl = GxFileType.Private;
+				string url = provider.Get(storageobjectfullname, acl, expirationMinutes);
 				if (String.IsNullOrEmpty(url))
 				{
 					GXUtil.ErrorToMessages("Get Error", "File doesn't exists", messages);
 					return false;
 				}
 				else
 				{
-					externalFile.FileInfo = new GxExternalFileInfo(storageobjectfullname, url, provider, GxFileType.Private);
+					externalFile.FileInfo = new GxExternalFileInfo(storageobjectfullname, url, provider, acl);
 					return true;
 				}
 			}

dotnet/src/dotnetframework/GxClasses/Core/GXUtilsCommon.cs

@@ -37,6 +37,8 @@
 using System.Security.Cryptography;
 using System.Collections.Concurrent;
 using System.Drawing.Drawing2D;
+using GeneXus.Storage;
+using GeneXus.Services;
 
 namespace GeneXus.Utils
 {
@@ -3364,7 +3366,7 @@ public static string UriToPath(string uriString)
 				return uriString;
 			}
 		}
-		public static string getTempFileName(string baseDir, string name="", string extension="tmp", GxFileType fileType = GxFileType.Public)
+		public static string getTempFileName(string baseDir, string name="", string extension="tmp", GxFileType fileType = GxFileType.Private)
 		{
 			name = FixFileName(FileUtil.FileNamePrettify(name), string.Empty);
 			return tempFileName(baseDir, name, extension);
@@ -3409,7 +3411,7 @@ public static string GetFileType(string FileName)
 
 			if (GxUploadHelper.IsUpload(FileName))
 			{
-				return new GxFile(string.Empty, FileName, GxFileType.Private).GetExtension();
+				return new GxFile(string.Empty, FileName, GxFileType.PrivateAttribute).GetExtension();
 			}
 
 			string extension = string.Empty;
@@ -3454,7 +3456,7 @@ public static string GetFileName(string FileName)
 
 			if (GxUploadHelper.IsUpload(FileName))
 			{
-				FileName = new GxFile(string.Empty, FileName, GxFileType.Private).GetName();
+				FileName = new GxFile(string.Empty, FileName, GxFileType.PrivateAttribute).GetName();
 			}
 			try
 			{
@@ -5257,15 +5259,22 @@ public static string ResolveUri(string uriString, IGxContext context = null)
 		public static string ResolveUri(string uriString, bool absUrl, IGxContext context = null)
 		{
 			if (String.IsNullOrEmpty(uriString))
-				return "";
+				return string.Empty;
+
+			string providerObjectName;
+			if (PathUtil.IsAbsoluteUrl(uriString) && StorageFactory.TryGetProviderObjectName(ServiceFactory.GetExternalProvider(), uriString, out providerObjectName))
+			{
+				return new GxFile(string.Empty, providerObjectName, GxFileType.DefaultAttribute).GetURI();
+			}
+
 			if (schemeRegex.IsMatch(uriString))
 			{
 				string fileName = schemeRegex.Replace(uriString, "");
 				//Same way as getBlobFile, creates a GxFile in order to take into account external storage when enabled.
 				string basePath = Path.Combine(Path.Combine(Preferences.getBLOB_PATH(), MultimediaDirectory));
 				try
 				{
-					GxFile file = new GxFile(string.Empty, PathUtil.SafeCombine(basePath, fileName), GxFileType.PublicAttribute);
+					GxFile file = new GxFile(string.Empty, PathUtil.SafeCombine(basePath, fileName), GxFileType.PrivateAttribute);
 					return PathToUrl(file.GetURI(), absUrl, context);
 				}
 				catch (ArgumentException ex)
@@ -5318,7 +5327,7 @@ public static string GetUriFromFile(string name, string type, string path)
 			{
 				if (GxUploadHelper.IsUpload(path))
 				{
-					return new GxFile(string.Empty, path, GxFileType.Private).GetName();
+					return new GxFile(string.Empty, path, GxFileType.PrivateAttribute).GetName();
 				}
 				string fromPathType = Path.GetExtension(path);
 				if (!String.IsNullOrEmpty(fromPathType) && fromPathType != "tmp")

dotnet/src/dotnetframework/GxClasses/Data/GXDataCommon.cs

@@ -903,7 +903,7 @@ protected static byte[] GetBinary(string fileNameParm, bool dbBlob)
 							{
 								if (ServiceFactory.GetExternalProvider() != null) 
 								{
-									GxFile file = new GxFile(string.Empty, fileNameParm, GxFileType.Private);
+									GxFile file = new GxFile(string.Empty, fileNameParm, GxFileType.PrivateAttribute);
 									if (file.Exists())
 									{
 										binary = file.ToByteArray();