From e4bbf11cf3cac669d08a9e22df4d8aff7d0972f4 Mon Sep 17 00:00:00 2001 From: Gonzalo Date: Wed, 30 Sep 2020 17:06:06 -0300 Subject: [PATCH 01/42] AWS S3 Add Support for using Custom Service URls such as: Minio, Oracle, etc. --- .../Storage/GXAmazonS3/ExternalProviderS3.cs | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs b/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs index 148ad28bb..23d3aa8ad 100644 --- a/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs +++ b/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs @@ -21,6 +21,8 @@ public class ExternalProviderS3 : ExternalProvider const string SECRET_ACCESS_KEY = "STORAGE_PROVIDER_SECRETACCESSKEY"; const string REGION = "STORAGE_PROVIDER_REGION"; const string ENDPOINT = "STORAGE_ENDPOINT"; + const string STORAGE_CUSTOM_ENDPOINT = "STORAGE_CUSTOM_ENDPOINT"; + const string STORAGE_CUSTOM_ENDPOINT_VALUE = "custom"; const string BUCKET = "BUCKET_NAME"; const string FOLDER = "FOLDER_NAME"; @@ -29,9 +31,13 @@ public class ExternalProviderS3 : ExternalProvider string Folder { get; set; } string Endpoint { get; set; } + bool ForcePathStyle = false; + public string StorageUri { - get { return $"https://{Bucket}.{Endpoint}/"; } + get { + return (ForcePathStyle)? $"{Endpoint}/": $"https://{Bucket}.{Endpoint}/"; + } } public string GetBaseURL() @@ -55,14 +61,21 @@ public ExternalProviderS3(GXService providerService) } var region = Amazon.RegionEndpoint.GetBySystemName(providerService.Properties.Get(REGION)); + Endpoint = providerService.Properties.Get(ENDPOINT); + if (Endpoint == STORAGE_CUSTOM_ENDPOINT_VALUE) + { + Endpoint = providerService.Properties.Get(STORAGE_CUSTOM_ENDPOINT); + ForcePathStyle = true; + } AmazonS3Config config = new AmazonS3Config() { + RegionEndpoint = region, ServiceURL = Endpoint, - RegionEndpoint = region + ForcePathStyle = ForcePathStyle }; - + #if NETCORE if (credentials != null) { From 7b337d82b7fac94df59f2a5f144f9f58d2e94734 Mon Sep 17 00:00:00 2001 From: Gonzalo Date: Thu, 18 Jun 2020 15:14:32 -0300 Subject: [PATCH 02/42] Remove IBM Bluemix as has been deprecated by IBM --- .../GXBluemix/ExternalProviderBluemix.cs | 1434 ----------------- .../Storage/GXBluemix/GXBluemix.csproj | 28 - .../Storage/GXBluemix/GlobalSuppressions.cs | 18 - .../GXBluemix/Properties/AssemblyInfo.cs | 27 - .../Providers/Storage/GXBluemix/app.config | 15 - 5 files changed, 1522 deletions(-) delete mode 100644 dotnet/src/dotnetframework/Providers/Storage/GXBluemix/ExternalProviderBluemix.cs delete mode 100644 dotnet/src/dotnetframework/Providers/Storage/GXBluemix/GXBluemix.csproj delete mode 100644 dotnet/src/dotnetframework/Providers/Storage/GXBluemix/GlobalSuppressions.cs delete mode 100644 dotnet/src/dotnetframework/Providers/Storage/GXBluemix/Properties/AssemblyInfo.cs delete mode 100644 dotnet/src/dotnetframework/Providers/Storage/GXBluemix/app.config diff --git a/dotnet/src/dotnetframework/Providers/Storage/GXBluemix/ExternalProviderBluemix.cs b/dotnet/src/dotnetframework/Providers/Storage/GXBluemix/ExternalProviderBluemix.cs deleted file mode 100644 index 058182ba4..000000000 --- a/dotnet/src/dotnetframework/Providers/Storage/GXBluemix/ExternalProviderBluemix.cs +++ /dev/null @@ -1,1434 +0,0 @@ -using GeneXus.Configuration; -using GeneXus.Services; -using JSIStudios.SimpleRESTServices.Client; -using log4net; -using net.openstack.Core; -using net.openstack.Core.Caching; -using net.openstack.Core.Domain; -using net.openstack.Core.Domain.Mapping; -using net.openstack.Core.Exceptions; -using net.openstack.Core.Exceptions.Response; -using net.openstack.Core.Providers; -using net.openstack.Core.Validators; -using net.openstack.Providers.Rackspace; -using net.openstack.Providers.Rackspace.Exceptions; -using net.openstack.Providers.Rackspace.Objects; -using net.openstack.Providers.Rackspace.Validators; -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; -using OpenStack.Authentication; -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Net; -using System.Security.Cryptography; -using System.Text; -using Thread = System.Threading.Thread; -using GeneXus.Utils; -using GeneXus.Encryption; - -namespace GeneXus.Storage.GXBluemix -{ - public class ExternalProviderBluemix : ExternalProvider - { - const string USER = "STORAGE_PROVIDER_USER"; - const string PASSWORD = "STORAGE_PROVIDER_PASSWORD"; - const string SERVER_URL = "SERVER_URL"; - const string PROJECT_ID = "PROJECT_ID"; - const string REGION = "STORAGE_PROVIDER_REGION"; - const string PRIVATE_BUCKET = "PRIVATE_BUCKET_NAME"; - const string PUBLIC_BUCKET = "PUBLIC_BUCKET_NAME"; - const string FOLDER = "FOLDER_NAME"; - - BluemixFilesProvider Client { get; set; } - string PublicBucket { get; set; } - string PrivateBucket { get; set; } - string Folder { get; set; } - string AuthToken { get; set; } - public string StorageUri { get; set; } - string PrivateTempKeyUrl { get; set; } - - public ExternalProviderBluemix() - : this(ServiceFactory.GetGXServices().Get(GXServices.STORAGE_SERVICE)) - { - } - - public ExternalProviderBluemix(GXService providerService) - { - var identityEndpoint = new Uri(providerService.Properties.Get(SERVER_URL)); - CloudIdentityWithProject identity = new CloudIdentityWithProject - { - Username = CryptoImpl.Decrypt(providerService.Properties.Get(USER)), - Password = CryptoImpl.Decrypt(providerService.Properties.Get(PASSWORD)), - ProjectName = providerService.Properties.Get(PROJECT_ID), - }; - - BlueMixIdentityProvider identityProvider = new BlueMixIdentityProvider(identityEndpoint, identity); - - GetStorageEndpoint(identityProvider, identity); - - Client = new BluemixFilesProvider(null, providerService.Properties.Get(REGION), identityProvider, null); - PublicBucket = CryptoImpl.Decrypt(providerService.Properties.Get(PUBLIC_BUCKET)); - PrivateBucket = CryptoImpl.Decrypt(providerService.Properties.Get(PRIVATE_BUCKET)); - Folder = providerService.Properties.Get(FOLDER); - AuthToken = identityProvider.AuthToken; - - CreateBuckets(); - CreateFolder(Folder); - } - - public ExternalProviderBluemix(string proj, string url, string user, string pass, string bucket, string region) - { - var identityEndpoint = new Uri(url); - CloudIdentityWithProject identity = new CloudIdentityWithProject - { - Username = user, - Password = pass, - ProjectName = proj, - }; - - BlueMixIdentityProvider identityProvider = new BlueMixIdentityProvider(identityEndpoint, identity); - - GetStorageEndpoint(identityProvider, identity); - - Client = new BluemixFilesProvider(null, region, identityProvider, null); - PublicBucket = bucket; - AuthToken = identityProvider.AuthToken; - - CreateBuckets(); - } - - private void GetStorageEndpoint(BlueMixIdentityProvider identityProvider, CloudIdentityWithProject identity) - { - UserAccessV3 user = identityProvider.GetUserAccess(identity); - var catalog = user.Catalog; - Endpoint objectStorageEndpoint = null; - foreach (Catalog service in catalog) - if (service.Type == "object-store") - if (service.Endpoints.Where(e => e.Region_id == "dallas").Any()) - objectStorageEndpoint = service.Endpoints.Where(e => e.Region_id == "dallas").First(); - - StorageUri = objectStorageEndpoint.Url; - - if (String.IsNullOrEmpty(StorageUri)) - throw new Exception("Can't find the object storage endpoint, please check the credentials in the storage configuration."); - } - - private Dictionary CreateObjectMetadata(string tableName, string fieldName, string key) - { - Dictionary metadata = new Dictionary(); - metadata.Add("Table", tableName); - metadata.Add("Field", fieldName); - metadata.Add("KeyValue", key); - return metadata; - } - - private void CreateBuckets() - { - Dictionary headers = new Dictionary(); - headers.Add("X-Auth-Token", AuthToken); - PrivateTempKeyUrl = Guid.NewGuid().ToString(); - headers.Add("X-Container-Meta-Temp-URL-Key", PrivateTempKeyUrl); - Client.CreateContainer(PrivateBucket, headers); - - headers = new Dictionary(); - headers.Add("X-Auth-Token", AuthToken); - headers.Add("X-Container-Read", ".r:*"); - Client.CreateContainer(PublicBucket, headers); - } - - private void CreateFolder(string folder, string table = null, string field = null) - { - string name = StorageUtils.NormalizeDirectoryName(folder); - if (table != null) - name += table + StorageUtils.DELIMITER; - if (field != null) - name += field + StorageUtils.DELIMITER; - using (var stream = new MemoryStream()) - { - Client.CreateObject(PublicBucket, stream, name, "application/directory"); - } - } - - public void Download(string externalFileName, string localFile, GxFileType fileType) - { - string bucket = GetBucket(fileType); - string localDirectory = Path.GetDirectoryName(localFile); - string localFileName = Path.GetFileName(localFile); - Client.GetObjectSaveToFile(bucket, localDirectory, externalFileName, localFileName); - } - - public string Upload(string localFile, string externalFileName, GxFileType fileType) - { - string bucket = GetBucket(fileType); - Client.CreateObjectFromFile(bucket, localFile, externalFileName); - return StorageUri + StorageUtils.DELIMITER + bucket + StorageUtils.DELIMITER + StorageUtils.EncodeUrl(externalFileName); - } - - private string GetBucket(GxFileType fileType) - { - return (fileType.HasFlag(GxFileType.Private)) ? PrivateBucket : PublicBucket; - } - - public string Get(string externalFileName, GxFileType fileType, int urlMinutes) - { - string bucket = GetBucket(fileType); - if (Exists(externalFileName, fileType)) - { - if (fileType.HasFlag(GxFileType.Private)) - return Client.CreateTemporaryPublicUri(HttpMethod.GET, bucket, externalFileName, PrivateTempKeyUrl, DateTimeOffset.Now.AddMinutes(urlMinutes)).ToString(); - return StorageUri + StorageUtils.DELIMITER + bucket + StorageUtils.DELIMITER + StorageUtils.EncodeUrl(externalFileName); - } - return string.Empty; - } - - public void Delete(string objectName, GxFileType fileType) - { - Client.DeleteObject(GetBucket(fileType), objectName); - } - - public bool Exists(string objectName, GxFileType fileType) - { - try - { - Client.GetObjectMetaData(GetBucket(fileType), objectName); - return true; - } - catch (ItemNotFoundException) - { - return false; - } - } - - public string Rename(string objectName, string newName, GxFileType fileType) - { - string bucket = GetBucket(fileType); - Copy(objectName, fileType, newName, fileType); - Delete(objectName, fileType); - return StorageUri + StorageUtils.DELIMITER + bucket + StorageUtils.DELIMITER + StorageUtils.EncodeUrl(newName); - } - - public string Copy(string objectName, GxFileType sourceFileType, string newName, GxFileType destFileType) - { - string sourceBucket = GetBucket(sourceFileType); - string destBucket = GetBucket(destFileType); - Client.CopyObject(sourceBucket, objectName, destBucket, newName); - return Get(objectName, destFileType, 0); - } - - public string Upload(string fileName, Stream stream, GxFileType fileType) - { - string bucket = GetBucket(fileType); - if (Path.GetExtension(fileName).Equals(".tmp")) - Client.CreateObject(bucket, stream, fileName, "image / jpeg"); - else - Client.CreateObject(bucket, stream, fileName); - - return Get(fileName, fileType, 0); - } - - public string Copy(string url, string newName, string tableName, string fieldName, GxFileType fileType) - { - string bucket = GetBucket(fileType); - string resourceKey = Folder + StorageUtils.DELIMITER + tableName + StorageUtils.DELIMITER + fieldName + StorageUtils.DELIMITER + newName; - CreateFolder(Folder, tableName, fieldName); - url = StorageUtils.DecodeUrl(url.Replace(StorageUri + StorageUtils.DELIMITER + bucket + StorageUtils.DELIMITER, "")); - - Copy(url, fileType, resourceKey, fileType); - Client.UpdateObjectMetadata(bucket, resourceKey, CreateObjectMetadata(tableName, fieldName, resourceKey)); - - return Get(resourceKey, fileType, 0); - } - - public Stream GetStream(string objectName, GxFileType fileType) - { - MemoryStream stream = new MemoryStream(); - Client.GetObject(GetBucket(fileType), objectName, stream); - return stream; - } - - public string GetDirectory(string directoryName) - { - directoryName = StorageUtils.NormalizeDirectoryName(directoryName); - if (ExistsDirectory(directoryName)) - return PublicBucket + StorageUtils.DELIMITER + directoryName; - else - return ""; - } - - public long GetLength(string objectName, GxFileType fileType) - { - foreach (ContainerObject obj in Client.ListObjects(GetBucket(fileType))) - if (obj.Name.Equals(objectName)) - return obj.Bytes; - return 0; - } - - public DateTime GetLastModified(string objectName, GxFileType fileType) - { - foreach (ContainerObject obj in Client.ListObjects(GetBucket(fileType))) - if (obj.Name.Equals(objectName)) - return obj.LastModified.UtcDateTime; - return new DateTime(); - } - - public void CreateDirectory(string directoryName) - { - CreateFolder(directoryName); - } - - public void DeleteDirectory(string directoryName) - { - List objs = new List(); - foreach (ContainerObject obj in Client.ListObjects(PublicBucket, prefix: StorageUtils.NormalizeDirectoryName(directoryName))) - { - objs.Add(obj.Name); - } - objs.Add(directoryName); - Client.DeleteObjects(PublicBucket, objs); - } - - public bool ExistsDirectory(string directoryName) - { - List directories = GetDirectories(); - return directories.Contains(directoryName) || directories.Contains(StorageUtils.NormalizeDirectoryName(directoryName)); - } - - public void RenameDirectory(string directoryName, string newDirectoryName) - { - directoryName = StorageUtils.NormalizeDirectoryName(directoryName); - newDirectoryName = StorageUtils.NormalizeDirectoryName(newDirectoryName); - - foreach (ContainerObject obj in Client.ListObjects(PublicBucket, prefix: directoryName)) - { - Client.CopyObject(PublicBucket, obj.Name, PublicBucket, obj.Name.Replace(directoryName, newDirectoryName)); - } - DeleteDirectory(directoryName); - } - - public List GetSubDirectories(string directoryName) - { - return GetDirectories(StorageUtils.NormalizeDirectoryName(directoryName)); - } - - private List GetDirectories(string directoryName = null) - { - List subdir = new List(); - foreach (ContainerObject obj in Client.ListObjects(PublicBucket, prefix: directoryName)) - { - if (directoryName == null) - { - string dir = ""; - string[] parts = obj.Name.Split(Convert.ToChar(StorageUtils.DELIMITER)); - for (int i = 0; i < parts.Length - 1; i++) - { - dir += parts[i] + StorageUtils.DELIMITER; - if (!subdir.Contains(dir)) - subdir.Add(dir); - } - } - else - { - string name = obj.Name.Replace(directoryName, ""); - int i = name.IndexOf(StorageUtils.DELIMITER); - if (i != -1) - { - name = name.Substring(0, i); - string dir = StorageUtils.NormalizeDirectoryName(directoryName + name); - if (!subdir.Contains(dir)) - subdir.Add(dir); - } - } - - } - if (directoryName != null) - { - subdir.Remove(directoryName); - } - return subdir; - } - - public List GetFiles(string directoryName, string filter = "") - { - directoryName = StorageUtils.NormalizeDirectoryName(directoryName); - List files = new List(); - foreach (ContainerObject obj in Client.ListObjects(PublicBucket, prefix: directoryName)) - { - if (IsFile(obj, directoryName) && (String.IsNullOrEmpty(filter) || obj.Name.Contains(filter))) - files.Add(obj.Name); - } - return files; - } - - private bool IsFile(ContainerObject obj, string directory = null) - { - char delimiter = Convert.ToChar(StorageUtils.DELIMITER); - if (directory == null) - return obj.Name.Split(delimiter).Length == 1; - else - return obj.Name.Replace(StorageUtils.NormalizeDirectoryName(directory), "").Split(delimiter).Length == 1 && !String.IsNullOrEmpty(obj.Name.Replace(StorageUtils.NormalizeDirectoryName(directory), "").Split(delimiter)[0]); - } - - public bool GetMessageFromException(Exception ex, SdtMessages_Message msg) - { - try - { - ResponseException rex = (ResponseException)ex; - msg.gxTpr_Id = rex.Response.Status; - return true; - } - catch (Exception) - { - return false; - } - } - - public string Save(Stream fileStream, string fileName, string tableName, string fieldName, GxFileType fileType) - { - CreateFolder(Folder, tableName, fieldName); - string resourceKey = Folder + StorageUtils.DELIMITER + tableName + StorageUtils.DELIMITER + fieldName + StorageUtils.DELIMITER + fileName; - return Upload(resourceKey, fileStream, fileType); - } - - public bool GetObjectNameFromURL(string url, out string objectName) - { - string baseUrl = StorageUri + StorageUtils.DELIMITER + PublicBucket + StorageUtils.DELIMITER; - if (url.StartsWith(baseUrl)) - { - objectName = url.Replace(baseUrl, string.Empty); - return true; - } - objectName = null; - return false; - } - public string GetBaseURL() - { - return StorageUri + StorageUtils.DELIMITER + PublicBucket + StorageUtils.DELIMITER + Folder + StorageUtils.DELIMITER; - } - - #region Extension of openstack.net (issue: https://github.com/openstacknetsdk/openstack.net/issues/503) to enable openstack Identity API v3: http://developer.openstack.org/api-ref-identity-v3.html - - public class IdentityTokenV3 : IdentityToken - { - public new string Id { get; set; } - } - - public class BlueMixIdentityProvider : CloudIdentityProvider - { - public string AuthToken { get; set; } - new ICache TokenCache { get; } - - public BlueMixIdentityProvider(Uri urlBase, CloudIdentity defaultIdentity) - : this(urlBase, defaultIdentity, null, null) - { - AuthToken = null; - } - - public BlueMixIdentityProvider(Uri urlBase, CloudIdentity defaultIdentity, IRestService restService, ICache tokenCache) - : base(defaultIdentity, restService, tokenCache, urlBase) - { - AuthToken = null; - if (urlBase == null) - throw new ArgumentNullException(nameof(urlBase)); - } - - public new IdentityToken GetToken(CloudIdentity identity, bool forceCacheRefresh = false) - { - CheckIdentity(identity); - - GetUserAccess(identity, forceCacheRefresh); - - IdentityTokenV3 token = new IdentityTokenV3(); - token.Id = AuthToken; - return token; - } - public new UserAccessV3 GetUserAccess(CloudIdentity identity, bool forceCacheRefresh = false) - { - identity = identity ?? DefaultIdentity; - - CloudIdentityWithProject identityWithProject = identity as CloudIdentityWithProject; - - if (string.IsNullOrEmpty(identityWithProject.Password)) - throw new NotSupportedException(string.Format("The {0} identity must specify a password.", typeof(CloudIdentityWithProject))); - if (!string.IsNullOrEmpty(identityWithProject.APIKey)) - throw new NotSupportedException(string.Format("The {0} identity does not support API key authentication.", typeof(CloudIdentityWithProject))); - - var auth = Authorization.CreateCredentials(identityWithProject); - var projectId = identityWithProject.ProjectId != null ? JToken.FromObject(identityWithProject.ProjectId) : string.Empty; - - var response = ExecuteRESTRequest(identity, new Uri(UrlBase, "/v3/auth/tokens"), HttpMethod.POST, auth, isTokenRequest: true); - if (response == null || response.Data == null) - return null; - - AuthToken = response.Headers[0].Value; - JToken userAccessObject = response.Data["token"]; - if (userAccessObject == null) - return null; - - UserAccessV3 access = userAccessObject.ToObject(); - if (access == null) - return null; - - string key = string.Format("{0}:{1}/{2}", UrlBase, identityWithProject.ProjectId, identityWithProject.Username); - - return access; - } - - protected override string LookupServiceTypeKey(IServiceType serviceType) - { - return serviceType.Type; - } - - } - - public class Authorization - { - //Generates the JSON with the credentials to authenticate to the Identity API v3 - public static JObject CreateCredentials(CloudIdentityWithProject identityWithProject) - { - return new JObject( - new JProperty("auth", new JObject( - new JProperty("identity", new JObject( - new JProperty("methods", new JArray("password")), - new JProperty("password", new JObject( - new JProperty("user", new JObject( - new JProperty("id", JValue.CreateString(identityWithProject.Username)), - new JProperty("password", JValue.CreateString(identityWithProject.Password)))))))), - new JProperty("scope", new JObject( - new JProperty("project", new JObject( - new JProperty("id", JValue.CreateString(identityWithProject.ProjectName))))))))); - } - } - - internal class EncodeDecodeProviderAux : IEncodeDecodeProvider - { - - private static readonly EncodeDecodeProviderAux _default = new EncodeDecodeProviderAux(); - - public static EncodeDecodeProviderAux Default - { - get - { - return _default; - } - } - - public string UrlEncode(string stringToEncode) - { - if (stringToEncode == null) - return null; - - return UriUtility.UriEncode(stringToEncode, UriPart.AnyUrl); - } - - public string UrlDecode(string stringToDecode) - { - if (stringToDecode == null) - return null; - - return UriUtility.UriDecode(stringToDecode); - } - } - - public class BluemixFilesProvider : CloudFilesProvider - { - private readonly IObjectStorageValidator _cloudFilesValidator; - private readonly IEncodeDecodeProvider _encodeDecodeProvider; - private readonly IObjectStorageMetadataProcessor _cloudFilesMetadataProcessor; - private readonly IStatusParser _statusParser; - private readonly IObjectMapper _bulkDeletionResultMapper; - public BluemixFilesProvider(CloudIdentity defaultIdentity, string defaultRegion, IIdentityProvider identityProvider, IRestService restService) - : base(defaultIdentity, defaultRegion, identityProvider, restService) - { - _cloudFilesValidator = CloudFilesValidator.Default; - _encodeDecodeProvider = EncodeDecodeProviderAux.Default; - _cloudFilesMetadataProcessor = CloudFilesMetadataProcessor.Default; - _statusParser = HttpStatusCodeParser.Default; - _bulkDeletionResultMapper = new BulkDeletionResultMapper(_statusParser); - } - - public new string GetServiceEndpointCloudFiles(CloudIdentity identity, string region = null, bool useInternalUrl = false) - { - string serviceType = "object-store"; - - if (serviceType == null) - throw new ArgumentNullException("serviceType"); - if (string.IsNullOrEmpty(serviceType)) - throw new ArgumentException("serviceType cannot be empty"); - CheckIdentity(identity); - - identity = GetDefaultIdentity(identity); - - var userAccess = ((BlueMixIdentityProvider)IdentityProvider).GetUserAccess(identity); - - if (userAccess == null || userAccess.Catalog == null) - throw new UserAuthenticationException("Unable to authenticate user and retrieve authorized service endpoints."); - - IEnumerable services = userAccess.Catalog.Where(sc => string.Equals(sc.Type, serviceType, StringComparison.OrdinalIgnoreCase)); - - IEnumerable> endpoints = - services.SelectMany(service => service.Endpoints.Select(endpoint => Tuple.Create(service, endpoint))); - - string effectiveRegion = region; - if (string.IsNullOrEmpty(effectiveRegion)) - { - if (!string.IsNullOrEmpty(DefaultRegion)) - effectiveRegion = DefaultRegion; - else if (!string.IsNullOrEmpty(userAccess.User.DefaultRegion)) - effectiveRegion = userAccess.User.DefaultRegion; - } - - IEnumerable> regionEndpoints = - endpoints.Where(i => string.Equals(i.Item2.Region ?? string.Empty, effectiveRegion ?? string.Empty, StringComparison.OrdinalIgnoreCase)); - - if (regionEndpoints.Any()) - endpoints = regionEndpoints; - else - endpoints = endpoints.Where(i => string.IsNullOrEmpty(i.Item2.Region)); - - if (effectiveRegion == null && !endpoints.Any()) - throw new NoDefaultRegionSetException("No region was provided, the service does not provide a region-independent endpoint, and there is no default region set for the user's account."); - - Tuple serviceEndpoint = endpoints.Where(e => e.Item2.Url.Contains("softlayer")).FirstOrDefault(); - if (serviceEndpoint == null) - throw new UserAuthorizationException("The user does not have access to the requested service or region."); - - return serviceEndpoint.Item2.Url; - } - public new ObjectStore CreateContainer(string container, Dictionary headers = null, string region = null, bool useInternalUrl = false, CloudIdentity identity = null) - { - if (container == null) - throw new ArgumentNullException(nameof(container)); - if (string.IsNullOrEmpty(container)) - throw new ArgumentException("container cannot be empty"); - CheckIdentity(identity); - - _cloudFilesValidator.ValidateContainerName(container); - var urlPath = new Uri(string.Format("{0}/{1}", GetServiceEndpointCloudFiles(identity, region, useInternalUrl), _encodeDecodeProvider.UrlEncode(container))); - - var response = ExecuteRESTRequest(identity, urlPath, HttpMethod.PUT, headers: headers); - - switch (response.StatusCode) - { - case HttpStatusCode.Created: - return ObjectStore.ContainerCreated; - - case HttpStatusCode.Accepted: - return ObjectStore.ContainerExists; - - default: - throw new ResponseException(string.Format("Unexpected status {0} returned by Create Container.", response.StatusCode), response); - } - } - - public new void CreateObject(string container, Stream stream, string objectName, string contentType = null, int chunkSize = 4096, Dictionary headers = null, string region = null, Action progressUpdated = null, bool useInternalUrl = false, CloudIdentity identity = null) - { - if (container == null) - throw new ArgumentNullException(nameof(container)); - if (stream == null) - throw new ArgumentNullException(nameof(stream)); - if (objectName == null) - throw new ArgumentNullException(nameof(objectName)); - if (string.IsNullOrEmpty(container)) - throw new ArgumentException("container cannot be empty"); - if (string.IsNullOrEmpty(objectName)) - throw new ArgumentException("objectName cannot be empty"); - if (chunkSize < 0) - throw new ArgumentOutOfRangeException(nameof(chunkSize)); - CheckIdentity(identity); - - _cloudFilesValidator.ValidateContainerName(container); - _cloudFilesValidator.ValidateObjectName(objectName); - - if (stream.Length > LargeFileBatchThreshold) - { - throw new ArgumentException("objectName is too big"); - } - var urlPath = new Uri(string.Format("{0}/{1}/{2}", GetServiceEndpointCloudFiles(identity, region, useInternalUrl), _encodeDecodeProvider.UrlEncode(container), _encodeDecodeProvider.UrlEncode(objectName))); - - RequestSettings settings = BuildDefaultRequestSettings(); - settings.ChunkRequest = true; - settings.ContentType = contentType; - - StreamRESTRequest(identity, urlPath, HttpMethod.PUT, stream, chunkSize, headers: headers, progressUpdated: progressUpdated, requestSettings: settings); - } - - public new void CreateObjectFromFile(string container, string filePath, string objectName = null, string contentType = null, int chunkSize = 4096, Dictionary headers = null, string region = null, Action progressUpdated = null, bool useInternalUrl = false, CloudIdentity identity = null) - { - if (container == null) - throw new ArgumentNullException(nameof(container)); - if (filePath == null) - throw new ArgumentNullException(nameof(filePath)); - if (string.IsNullOrEmpty(container)) - throw new ArgumentException("container cannot be empty"); - if (string.IsNullOrEmpty(filePath)) - throw new ArgumentException("filePath cannot be empty"); - if (chunkSize < 0) - throw new ArgumentOutOfRangeException(nameof(chunkSize)); - CheckIdentity(identity); - - if (string.IsNullOrEmpty(objectName)) - objectName = Path.GetFileName(filePath); - - using (var stream = File.OpenRead(filePath)) - { - CreateObject(container, stream, objectName, contentType, chunkSize, headers, region, progressUpdated, useInternalUrl, identity); - } - } - public new void UpdateObjectMetadata(string container, string objectName, Dictionary metadata, string region = null, bool useInternalUrl = false, CloudIdentity identity = null) - { - if (container == null) - throw new ArgumentNullException(nameof(container)); - if (objectName == null) - throw new ArgumentNullException(nameof(objectName)); - if (metadata == null) - throw new ArgumentNullException(nameof(metadata)); - if (string.IsNullOrEmpty(container)) - throw new ArgumentException("container cannot be empty"); - if (string.IsNullOrEmpty(objectName)) - throw new ArgumentException("objectName cannot be empty"); - CheckIdentity(identity); - - _cloudFilesValidator.ValidateContainerName(container); - _cloudFilesValidator.ValidateObjectName(objectName); - - var headers = new Dictionary(StringComparer.OrdinalIgnoreCase); - foreach (KeyValuePair m in metadata) - { - if (string.IsNullOrEmpty(m.Key)) - throw new ArgumentException("metadata cannot contain any empty keys"); - - headers.Add(ObjectMetaDataPrefix + m.Key, EncodeUnicodeValue(m.Value)); - } - - var urlPath = new Uri(string.Format("{0}/{1}/{2}", GetServiceEndpointCloudFiles(identity, region, useInternalUrl), _encodeDecodeProvider.UrlEncode(container), _encodeDecodeProvider.UrlEncode(objectName))); - - RequestSettings settings = BuildDefaultRequestSettings(); - // make sure the content type is not changed by the metadata operation - settings.ContentType = null; - - ExecuteRESTRequest(identity, urlPath, HttpMethod.POST, headers: headers, settings: settings); - } - - public new Dictionary GetObjectMetaData(string container, string objectName, string region = null, bool useInternalUrl = false, CloudIdentity identity = null) - { - if (container == null) - throw new ArgumentNullException(nameof(container)); - if (objectName == null) - throw new ArgumentNullException(nameof(objectName)); - if (string.IsNullOrEmpty(container)) - throw new ArgumentException("container cannot be empty"); - if (string.IsNullOrEmpty(objectName)) - throw new ArgumentException("objectName cannot be empty"); - CheckIdentity(identity); - - _cloudFilesValidator.ValidateContainerName(container); - _cloudFilesValidator.ValidateObjectName(objectName); - var urlPath = new Uri(string.Format("{0}/{1}/{2}", GetServiceEndpointCloudFiles(identity, region, useInternalUrl), _encodeDecodeProvider.UrlEncode(container), _encodeDecodeProvider.UrlEncode(objectName))); - - var response = ExecuteRESTRequest(identity, urlPath, HttpMethod.HEAD); - - var processedHeaders = _cloudFilesMetadataProcessor.ProcessMetadata(response.Headers); - - return processedHeaders[ProcessedHeadersMetadataKey]; - } - - public new void GetObject(string container, string objectName, Stream outputStream, int chunkSize = 4096, Dictionary headers = null, string region = null, bool verifyEtag = false, Action progressUpdated = null, bool useInternalUrl = false, CloudIdentity identity = null) - { - if (container == null) - throw new ArgumentNullException(nameof(container)); - if (objectName == null) - throw new ArgumentNullException(nameof(objectName)); - if (outputStream == null) - throw new ArgumentNullException(nameof(outputStream)); - if (string.IsNullOrEmpty(container)) - throw new ArgumentException("container cannot be empty"); - if (string.IsNullOrEmpty(objectName)) - throw new ArgumentException("objectName cannot be empty"); - if (chunkSize < 0) - throw new ArgumentOutOfRangeException(nameof(chunkSize)); - CheckIdentity(identity); - - _cloudFilesValidator.ValidateContainerName(container); - _cloudFilesValidator.ValidateObjectName(objectName); - - var urlPath = new Uri(string.Format("{0}/{1}/{2}", GetServiceEndpointCloudFiles(identity, region, useInternalUrl), _encodeDecodeProvider.UrlEncode(container), _encodeDecodeProvider.UrlEncode(objectName))); - - long? initialPosition; - try - { - initialPosition = outputStream.Position; - } - catch (NotSupportedException) - { - if (verifyEtag) - throw; - - initialPosition = null; - } - - // This flag indicates whether the outputStream needs to be set prior to copying data. - // See: https://github.com/openstacknetsdk/openstack.net/issues/297 - bool requiresPositionReset = false; - - var response = ExecuteRESTRequest(identity, urlPath, HttpMethod.GET, (resp, isError) => - { - if (resp == null) - return new Response(0, null, null); - - string body; - - if (!isError) - { - using (var respStream = resp.GetResponseStream()) - { - // The second condition will throw a proper NotSupportedException if the position - // cannot be checked. - if (requiresPositionReset && outputStream.Position != initialPosition) - outputStream.Position = initialPosition.Value; - - requiresPositionReset = true; - CopyStream(respStream, outputStream, chunkSize, progressUpdated); - } - - body = "[Binary]"; - } - else - { - using (StreamReader reader = new StreamReader(resp.GetResponseStream())) - { - body = reader.ReadToEnd(); - } - } - - var respHeaders = resp.Headers.AllKeys.Select(key => new HttpHeader(key, resp.GetResponseHeader(key))).ToList(); - - return new Response(resp.StatusCode, respHeaders, body); - }, headers: headers); - - } - - public new void GetObjectSaveToFile(string container, string saveDirectory, string objectName, string fileName = null, int chunkSize = 65536, Dictionary headers = null, string region = null, bool verifyEtag = false, Action progressUpdated = null, bool useInternalUrl = false, CloudIdentity identity = null) - { - if (container == null) - throw new ArgumentNullException(nameof(container)); - if (saveDirectory == null) - throw new ArgumentNullException(nameof(saveDirectory)); - if (objectName == null) - throw new ArgumentNullException(nameof(objectName)); - if (string.IsNullOrEmpty(container)) - throw new ArgumentException("container cannot be empty"); - if (string.IsNullOrEmpty(saveDirectory)) - throw new ArgumentException("saveDirectory cannot be empty"); - if (string.IsNullOrEmpty(objectName)) - throw new ArgumentException("objectName cannot be empty"); - if (chunkSize < 0) - throw new ArgumentOutOfRangeException(nameof(chunkSize)); - CheckIdentity(identity); - - if (string.IsNullOrEmpty(fileName)) - fileName = objectName; - - var filePath = Path.Combine(saveDirectory, string.IsNullOrEmpty(fileName) ? objectName : fileName); - - try - { - using (var fileStream = File.Open(filePath, FileMode.Create, FileAccess.ReadWrite)) - { - GetObject(container, objectName, fileStream, chunkSize, headers, region, verifyEtag, progressUpdated, useInternalUrl, identity); - } - } - catch (InvalidETagException) - { - File.Delete(filePath); - throw; - } - } - - private static string EncodeUnicodeValue(string value) - { - if (value == null) - return null; - - return Encoding.GetEncoding("ISO-8859-1").GetString(Encoding.UTF8.GetBytes(value)); - } - internal void CheckResponse(Response response) - { - if (response == null) - throw new ArgumentNullException(nameof(response)); - - ResponseCodeValidator.Validate(response); - } - protected new Response ExecuteRESTRequest(CloudIdentity identity, Uri absoluteUri, HttpMethod method, object body = null, Dictionary queryStringParameter = null, Dictionary headers = null, bool isRetry = false, bool isTokenRequest = false, RequestSettings settings = null) - { - if (absoluteUri == null) - throw new ArgumentNullException(nameof(absoluteUri)); - CheckIdentity(identity); - - return ExecuteRESTRequest>(identity, absoluteUri, method, body, queryStringParameter, headers, isRetry, isTokenRequest, settings, RestService.Execute); - } - protected new Response ExecuteRESTRequest(CloudIdentity identity, Uri absoluteUri, HttpMethod method, object body = null, Dictionary queryStringParameter = null, Dictionary headers = null, bool isRetry = false, bool isTokenRequest = false, RequestSettings settings = null) - { - if (absoluteUri == null) - throw new ArgumentNullException(nameof(absoluteUri)); - CheckIdentity(identity); - - return ExecuteRESTRequest(identity, absoluteUri, method, body, queryStringParameter, headers, isRetry, isTokenRequest, settings, RestService.Execute); - } - - protected new Response ExecuteRESTRequest(CloudIdentity identity, Uri absoluteUri, HttpMethod method, Func buildResponseCallback, object body = null, Dictionary queryStringParameter = null, Dictionary headers = null, bool isRetry = false, bool isTokenRequest = false, RequestSettings settings = null) - { - if (absoluteUri == null) - throw new ArgumentNullException(nameof(absoluteUri)); - CheckIdentity(identity); - - return ExecuteRESTRequest(identity, absoluteUri, method, body, queryStringParameter, headers, isRetry, isTokenRequest, settings, - (uri, requestMethod, requestBody, requestHeaders, requestQueryParams, requestSettings) => RestService.Execute(uri, requestMethod, buildResponseCallback, requestBody, requestHeaders, requestQueryParams, requestSettings)); - } - - private T ExecuteRESTRequest(CloudIdentity identity, Uri absoluteUri, HttpMethod method, object body, Dictionary queryStringParameter, Dictionary headers, bool isRetry, bool isTokenRequest, RequestSettings requestSettings, - Func, Dictionary, RequestSettings, T> callback) where T : Response - { - if (absoluteUri == null) - throw new ArgumentNullException(nameof(absoluteUri)); - CheckIdentity(identity); - - identity = GetDefaultIdentity(identity); - - if (requestSettings == null) - requestSettings = BuildDefaultRequestSettings(); - - if (headers == null) - headers = new Dictionary(); - - if (!isTokenRequest) - headers["X-Auth-Token"] = ((IdentityTokenV3)((BlueMixIdentityProvider)IdentityProvider).GetToken(identity, isRetry)).Id; - - string bodyStr = null; - if (body != null) - { - if (body is JObject) - bodyStr = body.ToString(); - else if (body is string) - bodyStr = body as string; - else - bodyStr = JsonConvert.SerializeObject(body, new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore }); - } - - if (string.IsNullOrEmpty(requestSettings.UserAgent)) - requestSettings.UserAgent = DefaultUserAgent; - - var response = callback(absoluteUri, method, bodyStr, headers, queryStringParameter, requestSettings); - - // on errors try again 1 time. - if (response.StatusCode == HttpStatusCode.Unauthorized && !isRetry && !isTokenRequest) - { - return ExecuteRESTRequest(identity, absoluteUri, method, body, queryStringParameter, headers, true, isTokenRequest, requestSettings, callback); - } - - CheckResponse(response); - - return response; - } - - protected new Response StreamRESTRequest(CloudIdentity identity, Uri absoluteUri, HttpMethod method, Stream stream, int chunkSize, long maxReadLength = 0, Dictionary queryStringParameter = null, Dictionary headers = null, bool isRetry = false, RequestSettings requestSettings = null, Action progressUpdated = null) - { - if (absoluteUri == null) - throw new ArgumentNullException(nameof(absoluteUri)); - if (stream == null) - throw new ArgumentNullException(nameof(stream)); - if (chunkSize <= 0) - throw new ArgumentOutOfRangeException(nameof(chunkSize)); - if (maxReadLength < 0) - throw new ArgumentOutOfRangeException(nameof(maxReadLength)); - CheckIdentity(identity); - - identity = GetDefaultIdentity(identity); - - if (requestSettings == null) - requestSettings = BuildDefaultRequestSettings(); - - requestSettings.Timeout = TimeSpan.FromMilliseconds(14400000); // Need to pass this in. - - if (headers == null) - headers = new Dictionary(); - - headers["X-Auth-Token"] = ((IdentityTokenV3)((BlueMixIdentityProvider)IdentityProvider).GetToken(identity, isRetry)).Id; ; - - if (string.IsNullOrEmpty(requestSettings.UserAgent)) - requestSettings.UserAgent = DefaultUserAgent; - - long? initialPosition; - try - { - initialPosition = stream.Position; - } - catch (NotSupportedException) - { - initialPosition = null; - } - - Response response; - try - { - response = RestService.Stream(absoluteUri, method, stream, chunkSize, maxReadLength, headers, queryStringParameter, requestSettings, progressUpdated); - } - catch (ProtocolViolationException) - { - ServicePoint servicePoint = ServicePointManager.FindServicePoint(absoluteUri); - if (servicePoint.ProtocolVersion < HttpVersion.Version11) - { - // this is a workaround for issue #333 - // https://github.com/openstacknetsdk/openstack.net/issues/333 - // http://stackoverflow.com/a/22976809/138304 - int maxIdleTime = servicePoint.MaxIdleTime; - servicePoint.MaxIdleTime = 0; - Thread.Sleep(1); - servicePoint.MaxIdleTime = maxIdleTime; - } - - response = RestService.Stream(absoluteUri, method, stream, chunkSize, maxReadLength, headers, queryStringParameter, requestSettings, progressUpdated); - } - - // on errors try again 1 time. - if (response.StatusCode == HttpStatusCode.Unauthorized && !isRetry && initialPosition != null) - { - bool canRetry; - - try - { - if (stream.Position != initialPosition.Value) - stream.Position = initialPosition.Value; - - canRetry = true; - } - catch (NotSupportedException) - { - // unable to retry the operation - canRetry = false; - } - - if (canRetry) - { - return StreamRESTRequest(identity, absoluteUri, method, stream, chunkSize, maxReadLength, queryStringParameter, headers, true, requestSettings, progressUpdated); - } - } - - CheckResponse(response); - - return response; - } - - public new void DeleteObject(string container, string objectName, Dictionary headers = null, bool deleteSegments = true, string region = null, bool useInternalUrl = false, CloudIdentity identity = null) - { - if (container == null) - throw new ArgumentNullException(nameof(container)); - if (objectName == null) - throw new ArgumentNullException(nameof(objectName)); - if (string.IsNullOrEmpty(container)) - throw new ArgumentException("container cannot be empty"); - if (string.IsNullOrEmpty(objectName)) - throw new ArgumentException("objectName cannot be empty"); - CheckIdentity(identity); - - _cloudFilesValidator.ValidateContainerName(container); - _cloudFilesValidator.ValidateObjectName(objectName); - - Dictionary objectHeader = null; - if (deleteSegments) - objectHeader = GetObjectHeaders(container, objectName, region, useInternalUrl, identity); - - if (deleteSegments && objectHeader != null && objectHeader.Any(h => h.Key.Equals(ObjectManifest, StringComparison.OrdinalIgnoreCase))) - { - var objects = ListObjects(container, region: region, useInternalUrl: useInternalUrl, - identity: identity); - - if (objects != null && objects.Any()) - { - var segments = objects.Where(f => f.Name.StartsWith(string.Format("{0}.seg", objectName))); - var delObjects = new List { objectName }; - if (segments.Any()) - delObjects.AddRange(segments.Select(s => s.Name)); - - DeleteObjects(container, delObjects, headers, region, useInternalUrl, identity); - } - else - throw new Exception(string.Format("Object \"{0}\" in container \"{1}\" does not exist.", objectName, container)); - } - else - { - var urlPath = new Uri(string.Format("{0}/{1}/{2}", GetServiceEndpointCloudFiles(identity, region, useInternalUrl), _encodeDecodeProvider.UrlEncode(container), _encodeDecodeProvider.UrlEncode(objectName))); - - ExecuteRESTRequest(identity, urlPath, HttpMethod.DELETE, headers: headers); - } - } - - public new Dictionary GetObjectHeaders(string container, string objectName, string region = null, bool useInternalUrl = false, CloudIdentity identity = null) - { - if (container == null) - throw new ArgumentNullException(nameof(container)); - if (objectName == null) - throw new ArgumentNullException(nameof(objectName)); - if (string.IsNullOrEmpty(container)) - throw new ArgumentException("container cannot be empty"); - if (string.IsNullOrEmpty(objectName)) - throw new ArgumentException("objectName cannot be empty"); - CheckIdentity(identity); - - _cloudFilesValidator.ValidateContainerName(container); - _cloudFilesValidator.ValidateObjectName(objectName); - var urlPath = new Uri(string.Format("{0}/{1}/{2}", GetServiceEndpointCloudFiles(identity, region, useInternalUrl), _encodeDecodeProvider.UrlEncode(container), _encodeDecodeProvider.UrlEncode(objectName))); - - var response = ExecuteRESTRequest(identity, urlPath, HttpMethod.HEAD); - - var processedHeaders = _cloudFilesMetadataProcessor.ProcessMetadata(response.Headers); - - return processedHeaders[ProcessedHeadersHeaderKey]; - } - - public new IEnumerable ListObjects(string container, int? limit = null, string marker = null, string markerEnd = null, string prefix = null, string region = null, bool useInternalUrl = false, CloudIdentity identity = null) - { - if (container == null) - throw new ArgumentNullException(nameof(container)); - if (string.IsNullOrEmpty(container)) - throw new ArgumentException("container cannot be empty"); - if (limit < 0) - throw new ArgumentOutOfRangeException(nameof(limit)); - CheckIdentity(identity); - - _cloudFilesValidator.ValidateContainerName(container); - var urlPath = new Uri(string.Format("{0}/{1}", GetServiceEndpointCloudFiles(identity, region, useInternalUrl), _encodeDecodeProvider.UrlEncode(container))); - - var queryStringParameter = new Dictionary(); - - if (limit != null) - queryStringParameter.Add("limit", limit.ToString()); - - if (!string.IsNullOrEmpty(marker)) - queryStringParameter.Add("marker", marker); - - if (!string.IsNullOrEmpty(markerEnd)) - queryStringParameter.Add("end_marker", markerEnd); - - if (!string.IsNullOrEmpty(prefix)) - queryStringParameter.Add("prefix", prefix); - - var response = ExecuteRESTRequest(identity, urlPath, HttpMethod.GET, null, queryStringParameter); - - if (response == null || response.Data == null) - return null; - - return response.Data; - } - - public new void CopyObject(string sourceContainer, string sourceObjectName, string destinationContainer, string destinationObjectName, string destinationContentType = null, Dictionary headers = null, string region = null, bool useInternalUrl = false, CloudIdentity identity = null) - { - if (sourceContainer == null) - throw new ArgumentNullException(nameof(sourceContainer)); - if (sourceObjectName == null) - throw new ArgumentNullException(nameof(sourceObjectName)); - if (destinationContainer == null) - throw new ArgumentNullException(nameof(destinationContainer)); - if (destinationObjectName == null) - throw new ArgumentNullException(nameof(destinationObjectName)); - if (string.IsNullOrEmpty(sourceContainer)) - throw new ArgumentException("sourceContainer cannot be empty"); - if (string.IsNullOrEmpty(sourceObjectName)) - throw new ArgumentException("sourceObjectName cannot be empty"); - if (string.IsNullOrEmpty(destinationContainer)) - throw new ArgumentException("destinationContainer cannot be empty"); - if (string.IsNullOrEmpty(destinationObjectName)) - throw new ArgumentException("destinationObjectName cannot be empty"); - CheckIdentity(identity); - - _cloudFilesValidator.ValidateContainerName(sourceContainer); - _cloudFilesValidator.ValidateObjectName(sourceObjectName); - - _cloudFilesValidator.ValidateContainerName(destinationContainer); - _cloudFilesValidator.ValidateObjectName(destinationObjectName); - - var urlPath = new Uri(string.Format("{0}/{1}/{2}", GetServiceEndpointCloudFiles(identity, region, useInternalUrl), _encodeDecodeProvider.UrlEncode(sourceContainer), _encodeDecodeProvider.UrlEncode(sourceObjectName))); - - if (headers == null) - headers = new Dictionary(StringComparer.OrdinalIgnoreCase); - - headers.Add(Destination, string.Format("{0}/{1}", UriUtility.UriEncode(destinationContainer, UriPart.AnyUrl, Encoding.UTF8), UriUtility.UriEncode(destinationObjectName, UriPart.AnyUrl, Encoding.UTF8))); - - RequestSettings settings = BuildDefaultRequestSettings(); - if (destinationContentType != null) - { - settings.ContentType = destinationContentType; - } - else - { - // make sure to preserve the content type during the copy operation - settings.ContentType = null; - } - - ExecuteRESTRequest(identity, urlPath, HttpMethod.COPY, headers: headers, settings: settings); - } - - public new void DeleteObjects(string container, IEnumerable objects, Dictionary headers = null, string region = null, bool useInternalUrl = false, CloudIdentity identity = null) - { - if (container == null) - throw new ArgumentNullException(nameof(container)); - if (objects == null) - throw new ArgumentNullException(nameof(objects)); - if (string.IsNullOrEmpty(container)) - throw new ArgumentException("container cannot be empty"); - - BulkDelete(objects.Select(o => new KeyValuePair(container, o)), headers, region, useInternalUrl, identity); - } - - public new void BulkDelete(IEnumerable> items, Dictionary headers = null, string region = null, bool useInternalUrl = false, CloudIdentity identity = null) - { - var urlPath = new Uri(string.Format("{0}/?bulk-delete", GetServiceEndpointCloudFiles(identity, region, useInternalUrl))); - - var encoded = items.Select( - pair => - { - if (string.IsNullOrEmpty(pair.Key)) - throw new ArgumentException("items", "items cannot contain any entries with a null or empty key (container name)"); - if (string.IsNullOrEmpty(pair.Value)) - throw new ArgumentException("items", "items cannot contain any entries with a null or empty value (object name)"); - _cloudFilesValidator.ValidateContainerName(pair.Key); - _cloudFilesValidator.ValidateObjectName(pair.Value); - - return string.Format("/{0}/{1}", _encodeDecodeProvider.UrlEncode(pair.Key), _encodeDecodeProvider.UrlEncode(pair.Value)); - }); - var body = string.Join("\n", encoded.ToArray()); - - var response = ExecuteRESTRequest(identity, urlPath, HttpMethod.POST, body: body, headers: headers, settings: new JSIStudios.SimpleRESTServices.Client.Json.JsonRequestSettings { ContentType = "text/plain" }); - - Status status; - if (_statusParser.TryParse(response.Data.Status, out status)) - { - if (status.Code != 200 && !response.Data.Errors.Any()) - { - response.Data.AllItems = encoded; - throw new BulkDeletionException(response.Data.Status, _bulkDeletionResultMapper.Map(response.Data)); - } - } - } - - public new Uri CreateTemporaryPublicUri(HttpMethod method, string container, string objectName, string key, DateTimeOffset expiration, string region = null, bool useInternalUrl = false, CloudIdentity identity = null) - { - if (container == null) - throw new ArgumentNullException(nameof(container)); - if (objectName == null) - throw new ArgumentNullException(nameof(objectName)); - if (key == null) - throw new ArgumentNullException(nameof(key)); - if (string.IsNullOrEmpty(container)) - throw new ArgumentException("container cannot be empty"); - if (string.IsNullOrEmpty(objectName)) - throw new ArgumentException("objectName cannot be empty"); - if (string.IsNullOrEmpty(key)) - throw new ArgumentException("key cannot be empty"); - CheckIdentity(identity); - - _cloudFilesValidator.ValidateContainerName(container); - _cloudFilesValidator.ValidateObjectName(objectName); - - Uri baseAddress = new Uri(GetServiceEndpointCloudFiles(identity, region, useInternalUrl), UriKind.Absolute); - - StringBuilder body = new StringBuilder(); - body.Append(method.ToString().ToUpperInvariant()).Append('\n'); - body.Append(ToTimestamp(expiration) / 1000).Append('\n'); - body.Append(baseAddress.PathAndQuery).Append('/').Append(container).Append('/').Append(objectName); - - using (HMAC hmac = HMAC.Create()) - { - hmac.HashName = "SHA1"; - hmac.Key = Encoding.UTF8.GetBytes(key); - byte[] hash = hmac.ComputeHash(Encoding.UTF8.GetBytes(body.ToString())); - string sig = string.Join(string.Empty, Array.ConvertAll(hash, i => i.ToString("x2"))); - string expires = (ToTimestamp(expiration) / 1000).ToString(); - - return new Uri(String.Format("{0}/{1}/{2}?temp_url_sig={3}&temp_url_expires={4}", baseAddress, container, objectName, sig, expires)); - } - } - private long ToTimestamp(DateTimeOffset dateTimeOffset) - { - DateTimeOffset Epoch = new DateTimeOffset(new DateTime(1970, 1, 1), TimeSpan.Zero); - return (long)(dateTimeOffset - Epoch).TotalMilliseconds; - } - - } - internal class BulkDeletionResultMapper : IObjectMapper - { - private readonly IStatusParser _statusParser; - - public BulkDeletionResultMapper(IStatusParser statusParser) - { - _statusParser = statusParser; - } - - public BulkDeletionResults Map(BulkDeleteResponse from) - { - var successfulObjects = from.AllItems.Where(i => !from.IsItemError(i)); - var failedObjects = from.Errors.Select(e => - { - var eParts = e.ToArray(); - Status errorStatus; - string errorItem; - - if (eParts.Length != 2) - { - errorStatus = new Status(0, "Unknown"); - errorItem = string.Format("The error array has an unexpected length. Array: {0}", string.Join("||", eParts)); - } - else - { - errorItem = eParts[1]; - if (!_statusParser.TryParse(eParts[0], out errorStatus)) - { - errorItem = eParts[0]; - if (!_statusParser.TryParse(eParts[1], out errorStatus)) - { - errorStatus = new Status(0, "Unknown"); - errorItem = string.Format("The error array is in an unknown format. Array: {0}", string.Join("||", eParts)); - } - } - } - - return new BulkDeletionFailedObject(errorItem, errorStatus); - }); - - return new BulkDeletionResults(successfulObjects, failedObjects); - } - - public BulkDeleteResponse Map(BulkDeletionResults to) - { - throw new NotImplementedException(); - } - } - internal class CloudFilesMetadataProcessor : IObjectStorageMetadataProcessor - { - - private static readonly CloudFilesMetadataProcessor _default = new CloudFilesMetadataProcessor(); - - public static CloudFilesMetadataProcessor Default - { - get - { - return _default; - } - } - - public virtual Dictionary> ProcessMetadata(IList httpHeaders) - { - if (httpHeaders == null) - throw new ArgumentNullException(nameof(httpHeaders)); - - var pheaders = new Dictionary(StringComparer.OrdinalIgnoreCase); - var metadata = new Dictionary(StringComparer.OrdinalIgnoreCase); - foreach (var header in httpHeaders) - { - if (header == null) - throw new ArgumentException("httpHeaders cannot contain any null values"); - if (string.IsNullOrEmpty(header.Key)) - throw new ArgumentException("httpHeaders cannot contain any values with a null or empty key"); - - if (header.Key.StartsWith(CloudFilesProvider.AccountMetaDataPrefix, StringComparison.OrdinalIgnoreCase)) - { - metadata.Add(header.Key.Substring(CloudFilesProvider.AccountMetaDataPrefix.Length), DecodeUnicodeValue(header.Value)); - } - else if (header.Key.StartsWith(CloudFilesProvider.ContainerMetaDataPrefix, StringComparison.OrdinalIgnoreCase)) - { - metadata.Add(header.Key.Substring(CloudFilesProvider.ContainerMetaDataPrefix.Length), DecodeUnicodeValue(header.Value)); - } - else if (header.Key.StartsWith(CloudFilesProvider.ObjectMetaDataPrefix, StringComparison.OrdinalIgnoreCase)) - { - metadata.Add(header.Key.Substring(CloudFilesProvider.ObjectMetaDataPrefix.Length), DecodeUnicodeValue(header.Value)); - } - else - { - pheaders.Add(header.Key, header.Value); - } - } - - var processedHeaders = new Dictionary>(StringComparer.OrdinalIgnoreCase) - { - {CloudFilesProvider.ProcessedHeadersHeaderKey, pheaders}, - {CloudFilesProvider.ProcessedHeadersMetadataKey, metadata} - }; - - return processedHeaders; - } - - private string DecodeUnicodeValue(string value) - { - return Encoding.UTF8.GetString(Encoding.GetEncoding("ISO-8859-1").GetBytes(value)); - } - } - - [JsonObject(MemberSerialization.OptIn)] - internal class BulkDeleteResponse - { - [JsonProperty("Number Not Found")] - public int NumberNotFound { get; set; } - - [JsonProperty("Response Status")] - public string Status { get; set; } - - [JsonProperty("Errors")] - public IEnumerable> Errors { get; set; } - - [JsonProperty("Number Deleted")] - public int NumberDeleted { get; set; } - - [JsonProperty("Response Body")] - public string ResponseBody { get; set; } - - public IEnumerable AllItems { get; set; } - - public bool IsItemError(string s) - { - return Errors.Any(e => e.Any(e2 => e2.Equals(s))); - } - } - #region Json Classes - [JsonObject(MemberSerialization.OptIn)] - public class Endpoint - { - [JsonProperty("region_id")] - public string Region_id { get; set; } - [JsonProperty("url")] - public string Url { get; set; } - [JsonProperty("region")] - public string Region { get; set; } - [JsonProperty("interface")] - public string @Interface { get; set; } - [JsonProperty("id")] - public string Id { get; set; } - } - [JsonObject(MemberSerialization.OptIn)] - public class Catalog - { - [JsonProperty("endpoints")] - public Endpoint[] Endpoints { get; set; } - [JsonProperty("type")] - public string Type { get; set; } - [JsonProperty("id")] - public string Id { get; set; } - [JsonProperty("name")] - public string Name { get; set; } - } - - [JsonObject(MemberSerialization.OptIn)] - public class UserAccessV3 : UserAccess - { - [JsonProperty("catalog")] - public Catalog[] Catalog { get; set; } - - [JsonExtensionData] - public IDictionary _additionalData; - } - #endregion - #endregion - - } -} - diff --git a/dotnet/src/dotnetframework/Providers/Storage/GXBluemix/GXBluemix.csproj b/dotnet/src/dotnetframework/Providers/Storage/GXBluemix/GXBluemix.csproj deleted file mode 100644 index 37f002ed7..000000000 --- a/dotnet/src/dotnetframework/Providers/Storage/GXBluemix/GXBluemix.csproj +++ /dev/null @@ -1,28 +0,0 @@ - - - net462 - GXBluemix - GXBluemix - CA1501;CA1812 - Bluemix - GeneXus.Bluemix - - - - - - - - - - 0.2.5 - - - - - - - - - - \ No newline at end of file diff --git a/dotnet/src/dotnetframework/Providers/Storage/GXBluemix/GlobalSuppressions.cs b/dotnet/src/dotnetframework/Providers/Storage/GXBluemix/GlobalSuppressions.cs deleted file mode 100644 index 6ca697c8d..000000000 --- a/dotnet/src/dotnetframework/Providers/Storage/GXBluemix/GlobalSuppressions.cs +++ /dev/null @@ -1,18 +0,0 @@ -// This file is used by Code Analysis to maintain SuppressMessage -// attributes that are applied to this project. -// Project-level suppressions either have no target or are given -// a specific target and scoped to a namespace, type, member, etc. - -[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Design", "CA1034:Nested types should not be visible", Justification = "", Scope = "type", Target = "~T:GeneXus.Storage.GXBluemix.ExternalProviderBluemix.Authorization")] -[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Design", "CA1034:Nested types should not be visible", Justification = "", Scope = "type", Target = "~T:GeneXus.Storage.GXBluemix.ExternalProviderBluemix.BluemixFilesProvider")] -[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Design", "CA1034:Nested types should not be visible", Justification = "", Scope = "type", Target = "~T:GeneXus.Storage.GXBluemix.ExternalProviderBluemix.BlueMixIdentityProvider")] -[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Design", "CA1034:Nested types should not be visible", Justification = "", Scope = "type", Target = "~T:GeneXus.Storage.GXBluemix.ExternalProviderBluemix.Catalog")] -[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Design", "CA1034:Nested types should not be visible", Justification = "", Scope = "type", Target = "~T:GeneXus.Storage.GXBluemix.ExternalProviderBluemix.Endpoint")] -[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Design", "CA1034:Nested types should not be visible", Justification = "", Scope = "type", Target = "~T:GeneXus.Storage.GXBluemix.ExternalProviderBluemix.IdentityTokenV3")] -[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Design", "CA1034:Nested types should not be visible", Justification = "", Scope = "type", Target = "~T:GeneXus.Storage.GXBluemix.ExternalProviderBluemix.UserAccessV3")] -[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Naming", "CA1707:Identifiers should not contain underscores", Justification = "", Scope = "member", Target = "~P:GeneXus.Storage.GXBluemix.ExternalProviderBluemix.Endpoint.Region_id")] - -[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Performance", "CA1819:Properties should not return arrays", Justification = "", Scope = "member", Target = "~P:GeneXus.Storage.GXBluemix.ExternalProviderBluemix.Catalog.Endpoints")] -[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Performance", "CA1819:Properties should not return arrays", Justification = "", Scope = "member", Target = "~P:GeneXus.Storage.GXBluemix.ExternalProviderBluemix.UserAccessV3.Catalog")] -[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Usage", "CA2208:Instantiate argument exceptions correctly", Justification = "", Scope = "member", Target = "~M:GeneXus.Storage.GXBluemix.ExternalProviderBluemix.BluemixFilesProvider.BulkDelete(System.Collections.Generic.IEnumerable{System.Collections.Generic.KeyValuePair{System.String,System.String}},System.Collections.Generic.Dictionary{System.String,System.String},System.String,System.Boolean,net.openstack.Core.Domain.CloudIdentity)")] -[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Usage", "CA2208:Instantiate argument exceptions correctly", Justification = "", Scope = "member", Target = "~M:GeneXus.Storage.GXBluemix.ExternalProviderBluemix.BluemixFilesProvider.GetServiceEndpointCloudFiles(net.openstack.Core.Domain.CloudIdentity,System.String,System.Boolean)~System.String")] \ No newline at end of file diff --git a/dotnet/src/dotnetframework/Providers/Storage/GXBluemix/Properties/AssemblyInfo.cs b/dotnet/src/dotnetframework/Providers/Storage/GXBluemix/Properties/AssemblyInfo.cs deleted file mode 100644 index bb7c4dab9..000000000 --- a/dotnet/src/dotnetframework/Providers/Storage/GXBluemix/Properties/AssemblyInfo.cs +++ /dev/null @@ -1,27 +0,0 @@ -using System.Reflection; -using System.Resources; -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; - -// General Information about an assembly is controlled through the following -// set of attributes. Change these attribute values to modify the information -// associated with an assembly. -[assembly: AssemblyTitle("GXBluemix")] - - -// Setting ComVisible to false makes the types in this assembly not visible -// to COM components. If you need to access a type in this assembly from -// COM, set the ComVisible attribute to true on that type. - -// The following GUID is for the ID of the typelib if this project is exposed to COM -// Version information for an assembly consists of the following four values: -// -// Major Version -// Minor Version -// Build Number -// Revision -// -// You can specify all the values or you can default the Build and Revision Numbers -// by using the '*' as shown below: -// [assembly: AssemblyVersion("1.0.*")] -[assembly: NeutralResourcesLanguageAttribute("en")] diff --git a/dotnet/src/dotnetframework/Providers/Storage/GXBluemix/app.config b/dotnet/src/dotnetframework/Providers/Storage/GXBluemix/app.config deleted file mode 100644 index 1cd55066a..000000000 --- a/dotnet/src/dotnetframework/Providers/Storage/GXBluemix/app.config +++ /dev/null @@ -1,15 +0,0 @@ - - - - - - - - - - - - - - - From 23f7ba44c0f815963dcf5282ea6b79cd2033ed13 Mon Sep 17 00:00:00 2001 From: Gonzalo Date: Tue, 1 Jun 2021 19:52:20 -0300 Subject: [PATCH 03/42] External Provider improvements first bits --- dotnet/DotNetStandardClasses.sln | 7 -- .../src/dotnetcore/GxClasses/GxClasses.csproj | 1 + .../GxClasses/Core/GXApplication.cs | 2 +- .../GxClasses/Core/GXUtilsCommon.cs | 2 +- .../GxClasses/Data/GXDataNTierADO.cs | 18 +-- .../GxClasses/Domain/GXFileIO.cs | 12 +- .../Services/Storage/ExternalProviderBase.cs | 111 ++++++++++++++++++ .../Storage/GXAmazonS3/ExternalProviderS3.cs | 92 +++++++++++---- 8 files changed, 195 insertions(+), 50 deletions(-) create mode 100644 dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderBase.cs diff --git a/dotnet/DotNetStandardClasses.sln b/dotnet/DotNetStandardClasses.sln index e69ec0517..38eba23a8 100644 --- a/dotnet/DotNetStandardClasses.sln +++ b/dotnet/DotNetStandardClasses.sln @@ -65,8 +65,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "GXAmazonS3", "src\dotnetfra EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "GXAzureStorage", "src\dotnetframework\Providers\Storage\GXAzureStorage\GXAzureStorage.csproj", "{F6372249-AF37-4455-B572-5BDF8DE2ACC8}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "GXBluemix", "src\dotnetframework\Providers\Storage\GXBluemix\GXBluemix.csproj", "{FCA5536D-542E-480E-92B0-316E0290553B}" -EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "GXGoogleCloud", "src\dotnetframework\Providers\Storage\GXGoogleCloud\GXGoogleCloud.csproj", "{E9072D95-D116-4D4B-B981-46146BCDE052}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "GXOpenStack", "src\dotnetframework\Providers\Storage\GXOpenStack\GXOpenStack.csproj", "{64E958D8-CE7B-482D-B339-3B52E26CA4AC}" @@ -233,10 +231,6 @@ Global {F6372249-AF37-4455-B572-5BDF8DE2ACC8}.Debug|Any CPU.Build.0 = Debug|Any CPU {F6372249-AF37-4455-B572-5BDF8DE2ACC8}.Release|Any CPU.ActiveCfg = Release|Any CPU {F6372249-AF37-4455-B572-5BDF8DE2ACC8}.Release|Any CPU.Build.0 = Release|Any CPU - {FCA5536D-542E-480E-92B0-316E0290553B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {FCA5536D-542E-480E-92B0-316E0290553B}.Debug|Any CPU.Build.0 = Debug|Any CPU - {FCA5536D-542E-480E-92B0-316E0290553B}.Release|Any CPU.ActiveCfg = Release|Any CPU - {FCA5536D-542E-480E-92B0-316E0290553B}.Release|Any CPU.Build.0 = Release|Any CPU {E9072D95-D116-4D4B-B981-46146BCDE052}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {E9072D95-D116-4D4B-B981-46146BCDE052}.Debug|Any CPU.Build.0 = Debug|Any CPU {E9072D95-D116-4D4B-B981-46146BCDE052}.Release|Any CPU.ActiveCfg = Release|Any CPU @@ -366,7 +360,6 @@ Global {F82842DA-F15E-49C5-993E-4C269818FF1F} = {F900A4AD-7249-41B4-B918-CB9E8C73747C} {3701565F-1A95-4592-B90D-291CCDE5505D} = {F82842DA-F15E-49C5-993E-4C269818FF1F} {F6372249-AF37-4455-B572-5BDF8DE2ACC8} = {F82842DA-F15E-49C5-993E-4C269818FF1F} - {FCA5536D-542E-480E-92B0-316E0290553B} = {F82842DA-F15E-49C5-993E-4C269818FF1F} {E9072D95-D116-4D4B-B981-46146BCDE052} = {F82842DA-F15E-49C5-993E-4C269818FF1F} {64E958D8-CE7B-482D-B339-3B52E26CA4AC} = {F82842DA-F15E-49C5-993E-4C269818FF1F} {A14C2C2C-ACE3-4712-A527-E4E5F02729FA} = {F900A4AD-7249-41B4-B918-CB9E8C73747C} diff --git a/dotnet/src/dotnetcore/GxClasses/GxClasses.csproj b/dotnet/src/dotnetcore/GxClasses/GxClasses.csproj index dbf7ef182..ca6c14b45 100644 --- a/dotnet/src/dotnetcore/GxClasses/GxClasses.csproj +++ b/dotnet/src/dotnetcore/GxClasses/GxClasses.csproj @@ -77,6 +77,7 @@ + diff --git a/dotnet/src/dotnetframework/GxClasses/Core/GXApplication.cs b/dotnet/src/dotnetframework/GxClasses/Core/GXApplication.cs index dc52dd35d..86b530b55 100644 --- a/dotnet/src/dotnetframework/GxClasses/Core/GXApplication.cs +++ b/dotnet/src/dotnetframework/GxClasses/Core/GXApplication.cs @@ -690,7 +690,7 @@ public static bool GetHttpRequestPostedFile(IGxContext gxContext, string varName ext = ext.TrimStart('.'); string filePath = FileUtil.getTempFileName(tempDir, "BLOB", ext); GXLogging.Debug(log, "cgiGet(" + varName + "), fileName:" + filePath); - GxFile file = new GxFile(tempDir, filePath, GxFileType.PrivateAttribute); + GxFile file = new GxFile(tempDir, filePath, GxFileType.Private); filePath = file.Create(pf.InputStream); string fileGuid = GxUploadHelper.GetUploadFileGuid(); fileToken = GxUploadHelper.GetUploadFileId(fileGuid); diff --git a/dotnet/src/dotnetframework/GxClasses/Core/GXUtilsCommon.cs b/dotnet/src/dotnetframework/GxClasses/Core/GXUtilsCommon.cs index 64ffe7ed1..b7f5f9478 100644 --- a/dotnet/src/dotnetframework/GxClasses/Core/GXUtilsCommon.cs +++ b/dotnet/src/dotnetframework/GxClasses/Core/GXUtilsCommon.cs @@ -5265,7 +5265,7 @@ public static string ResolveUri(string uriString, bool absUrl, IGxContext contex string basePath = Path.Combine(Path.Combine(Preferences.getBLOB_PATH(), MultimediaDirectory)); try { - GxFile file = new GxFile(string.Empty, PathUtil.SafeCombine(basePath, fileName), GxFileType.PublicAttribute); + GxFile file = new GxFile(string.Empty, PathUtil.SafeCombine(basePath, fileName), GxFileType.Public); return PathToUrl(file.GetURI(), absUrl, context); } catch (ArgumentException ex) diff --git a/dotnet/src/dotnetframework/GxClasses/Data/GXDataNTierADO.cs b/dotnet/src/dotnetframework/GxClasses/Data/GXDataNTierADO.cs index 0219ffb9d..e1904d8bd 100644 --- a/dotnet/src/dotnetframework/GxClasses/Data/GXDataNTierADO.cs +++ b/dotnet/src/dotnetframework/GxClasses/Data/GXDataNTierADO.cs @@ -105,7 +105,7 @@ public string getBLOBFile(int id) public string getBLOBFile(int id, string extension, string name) { - string fileName = FileUtil.getTempFileName(_gxDbCommand.Conn.BlobPath, name, extension, GxFileType.PrivateAttribute); + string fileName = FileUtil.getTempFileName(_gxDbCommand.Conn.BlobPath, name, extension, GxFileType.Private); return getBLOBFile(id, extension, name, fileName, true); } @@ -127,7 +127,7 @@ private string getBLOBFile(int id, string extension, string name, string fileNam } } if (temporary) - GXFileWatcher.Instance.AddTemporaryFile(new GxFile(_gxDbCommand.Conn.BlobPath, new GxFileInfo(fileName, _gxDbCommand.Conn.BlobPath), GxFileType.PrivateAttribute), _gxDbCommand.Conn.DataStore.Context); + GXFileWatcher.Instance.AddTemporaryFile(new GxFile(_gxDbCommand.Conn.BlobPath, new GxFileInfo(fileName, _gxDbCommand.Conn.BlobPath), GxFileType.Private), _gxDbCommand.Conn.DataStore.Context); fileName = new FileInfo(fileName).FullName; } catch (IOException e) @@ -149,7 +149,7 @@ public string getMultimediaFile(int id, string gxdbFileUri) string filePath = PathUtil.SafeCombine(_gxDbCommand.Conn.MultimediaPath, fileName); try { - GxFile file = new GxFile(string.Empty, filePath, GxFileType.PublicAttribute); + GxFile file = new GxFile(string.Empty, filePath, GxFileType.Public); if (file.Exists()) { return filePath; @@ -331,13 +331,13 @@ public string getBLOBFile(int id) public string getBLOBFile(int id, string extension, string name) { - string fileName = FileUtil.getTempFileName(_gxDbCommand.Conn.BlobPath, name, extension, GxFileType.PrivateAttribute); + string fileName = FileUtil.getTempFileName(_gxDbCommand.Conn.BlobPath, name, extension, GxFileType.Private); String value = getBLOBFile(id, extension, name, fileName, true); TraceRow("getBLOBFile - index : ", id.ToString(), " value:", (value!=null ? value.ToString() : string.Empty)); return value; } - private string getBLOBFile(int id, string extension, string name, string fileName, bool temporary, GxFileType fileType = GxFileType.PrivateAttribute) + private string getBLOBFile(int id, string extension, string name, string fileName, bool temporary, GxFileType fileType = GxFileType.Private) { GxFile file = null; Stream fs = null; @@ -433,7 +433,7 @@ public string getMultimediaFile(int id, string gxdbFileUri) try { - GxFile file = new GxFile(string.Empty, filePath, GxFileType.PublicAttribute); + GxFile file = new GxFile(string.Empty, filePath, GxFileType.Public); if (file.Exists()) { @@ -441,7 +441,7 @@ public string getMultimediaFile(int id, string gxdbFileUri) } else { - return getBLOBFile(id, FileUtil.GetFileType(gxdbFileUri), FileUtil.GetFileName(gxdbFileUri), filePath, false, GxFileType.PublicAttribute); + return getBLOBFile(id, FileUtil.GetFileType(gxdbFileUri), FileUtil.GetFileName(gxdbFileUri), filePath, false, GxFileType.Public); } } catch (ArgumentException) @@ -617,7 +617,7 @@ public void SetParameterMultimedia(int id, string image_gxi, string image, strin { try { - multimediaUri = ServiceFactory.GetExternalProvider().Copy(image_gxi, GXDbFile.GenerateUri(image_gxi, !GXDbFile.HasToken(image_gxi), false), tableName, fieldName, GxFileType.PublicAttribute); + multimediaUri = ServiceFactory.GetExternalProvider().Copy(image_gxi, GXDbFile.GenerateUri(image_gxi, !GXDbFile.HasToken(image_gxi), false), tableName, fieldName, GxFileType.Public); GXLogging.Debug(log, "Copy file already in ExternalProvider:", multimediaUri); } catch (Exception ex) @@ -726,7 +726,7 @@ private static string PushToExternalProvider(Stream fileStream, string externalF string multimediaUri; using (fileStream) { - multimediaUri = ServiceFactory.GetExternalProvider().Save(fileStream, externalFileName, tableName, fieldName, GxFileType.PublicAttribute); + multimediaUri = ServiceFactory.GetExternalProvider().Save(fileStream, externalFileName, tableName, fieldName, GxFileType.Public); GXLogging.Debug(log, "Upload file to ExternalProvider:", multimediaUri); } diff --git a/dotnet/src/dotnetframework/GxClasses/Domain/GXFileIO.cs b/dotnet/src/dotnetframework/GxClasses/Domain/GXFileIO.cs index 8fe6a1f61..bfb0243e8 100644 --- a/dotnet/src/dotnetframework/GxClasses/Domain/GXFileIO.cs +++ b/dotnet/src/dotnetframework/GxClasses/Domain/GXFileIO.cs @@ -432,12 +432,14 @@ public GxExternalFileInfo(string storageObjectFullname, ExternalProvider provide _url = storageObjectFullname; } else { - if (fileType.HasFlag(GxFileType.Attribute)) //Attributes multimedia consider Storage Provider Folder + if (fileType.HasFlag(GxFileType.Private)) //Attributes multimedia consider Storage Provider Folder { _url = provider.GetBaseURL() + storageObjectFullname; _name = _url.Replace(provider.StorageUri, string.Empty); if (_name.StartsWith("/")) + { _name = _name.Substring(1, _name.Length - 1); + } } } _fileTypeAtt = fileType; @@ -640,11 +642,9 @@ public Stream GetStream() [Flags] public enum GxFileType { - Public = 0, - Private = 1, - Attribute = 2, - PublicAttribute = Attribute | Public, - PrivateAttribute = Attribute | Private, + Default = 0, + Public = 1, + Private = 2 } public class GxFile diff --git a/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderBase.cs b/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderBase.cs new file mode 100644 index 000000000..1bef72a61 --- /dev/null +++ b/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderBase.cs @@ -0,0 +1,111 @@ +using System; +using GeneXus.Encryption; +using log4net; + +namespace GeneXus.Services +{ + public abstract class ExternalProviderBase + { + static readonly ILog logger = log4net.LogManager.GetLogger(typeof(ExternalProviderBase)); + + private GXService service; + + protected static String DEFAULT_ACL = "DEFAULT_ACL"; + protected static String DEFAULT_EXPIRATION = "DEFAULT_EXPIRATION"; + protected static String FOLDER = "FOLDER_NAME"; + protected static String DEFAULT_ACL_DEPRECATED = "STORAGE_PROVIDER_DEFAULT_ACL"; + protected static String DEFAULT_EXPIRATION_DEPRECATED = "STORAGE_PROVIDER_DEFAULT_EXPIRATION"; + protected const int DefaultExpirationMinutes = 24 * 60; + + + protected GxFileType defaultAcl = GxFileType.Private; + + public ExternalProviderBase() + { + Initialize(); + } + + public ExternalProviderBase(GXService s) + { + this.service = s; + Initialize(); + } + + public abstract String GetName(); + + private void Initialize() + { + String aclS = GetPropertyValue(DEFAULT_ACL, DEFAULT_ACL_DEPRECATED, ""); + if (!String.IsNullOrEmpty(aclS)) + { + GxFileType.TryParse(aclS, out this.defaultAcl); + } + } + + protected String GetEncryptedPropertyValue(String propertyName, String alternativePropertyName) + { + String value = GetEncryptedPropertyValue(propertyName, alternativePropertyName, null); + if (value == null) + { + String errorMessage = String.Format($"Service configuration error - Property name {ResolvePropertyName(propertyName)} must be defined"); + logger.Fatal(errorMessage); + throw new Exception(errorMessage); + } + return value; + } + + protected String GetEncryptedPropertyValue(String propertyName, String alternativePropertyName, String defaultValue) + { + String value = GetPropertyValue(propertyName, alternativePropertyName, defaultValue); + if (!String.IsNullOrEmpty(value)) + { + try + { + value = CryptoImpl.Decrypt(value); + } + catch (Exception) + { + logger.Warn($"Could not decrypt property name: {ResolvePropertyName(propertyName)}"); + } + } + return value; + } + + protected String GetPropertyValue(String propertyName, String alternativePropertyName) + { + String value = GetPropertyValue(propertyName, alternativePropertyName, null); + if (value == null) + { + String errorMessage = String.Format($"Service configuration error - Property name {ResolvePropertyName(propertyName)} must be defined"); + logger.Fatal(errorMessage); + throw new Exception(errorMessage); + } + return value; + } + + protected String GetPropertyValue(String propertyName, String alternativePropertyName, String defaultValue) + { + propertyName = ResolvePropertyName(propertyName); + String value = Environment.GetEnvironmentVariable(propertyName); + if (String.IsNullOrEmpty(value)) + { + value = Environment.GetEnvironmentVariable(alternativePropertyName); + } + if (this.service != null) + { + value = this.service.Properties.Get(propertyName); + if (String.IsNullOrEmpty(value)) + { + value = this.service.Properties.Get(alternativePropertyName); + } + } + return !String.IsNullOrEmpty(value) ? value : defaultValue; + } + + protected String ResolvePropertyName(String propertyName) + { + return String.Format("STORAGE_%s_%s", GetName(), propertyName); + } + + } +} diff --git a/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs b/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs index 23d3aa8ad..cd5c179a7 100644 --- a/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs +++ b/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs @@ -10,26 +10,47 @@ using System.Collections.Generic; using System.IO; using System.Threading; -using System.Threading.Tasks; + namespace GeneXus.Storage.GXAmazonS3 { - public class ExternalProviderS3 : ExternalProvider + public class ExternalProviderS3 : ExternalProviderBase, ExternalProvider { - const int PRIVATE_URL_MINUTES_EXPIRATION = 24 * 60; // 24 hours - const string ACCESS_KEY_ID = "STORAGE_PROVIDER_ACCESSKEYID"; - const string SECRET_ACCESS_KEY = "STORAGE_PROVIDER_SECRETACCESSKEY"; - const string REGION = "STORAGE_PROVIDER_REGION"; - const string ENDPOINT = "STORAGE_ENDPOINT"; - const string STORAGE_CUSTOM_ENDPOINT = "STORAGE_CUSTOM_ENDPOINT"; - const string STORAGE_CUSTOM_ENDPOINT_VALUE = "custom"; + const string Name = "AWSS3"; + + const string ACCESS_KEY = "ACCESS_KEY"; + const string SECRET_ACCESS_KEY = "SECRET_KEY"; + const string STORAGE_CUSTOM_ENDPOINT = "CUSTOM_ENDPOINT"; + const string STORAGE_ENDPOINT = "ENDPOINT"; const string BUCKET = "BUCKET_NAME"; - const string FOLDER = "FOLDER_NAME"; + const string REGION = "REGION"; + const string STORAGE_CUSTOM_ENDPOINT_VALUE = "custom"; + + const string DEFAULT_REGION = "us-east-1"; + + [Obsolete("Use Property ACCESS_KEY instead", false)] + const string ACCESS_KEY_ID_DEPRECATED = "STORAGE_PROVIDER_ACCESSKEYID"; + [Obsolete("Use Property SECRET_ACCESS_KEY instead", false)] + const string SECRET_ACCESS_KEY_DEPRECATED = "STORAGE_PROVIDER_SECRETACCESSKEY"; + [Obsolete("Use Property REGION instead", false)] + const string REGION_DEPRECATED = "STORAGE_PROVIDER_REGION"; + [Obsolete("Use Property STORAGE_ENDPOINT instead", false)] + const string ENDPOINT_DEPRECATED = "STORAGE_ENDPOINT"; + [Obsolete("Use Property STORAGE_CUSTOM_ENDPOINT instead", false)] + const string STORAGE_CUSTOM_ENDPOINT_DEPRECATED = "STORAGE_CUSTOM_ENDPOINT"; + + [Obsolete("Use Property BUCKET instead", false)] + const string BUCKET_DEPRECATED = "BUCKET_NAME"; + [Obsolete("Use Property FOLDER instead", false)] + const string FOLDER_DEPRECATED = "FOLDER_NAME"; + + string _storageUri; IAmazonS3 Client { get; set; } string Bucket { get; set; } string Folder { get; set; } string Endpoint { get; set; } + string Region { get; set; } bool ForcePathStyle = false; @@ -52,20 +73,20 @@ public ExternalProviderS3() public ExternalProviderS3(GXService providerService) { - string keyId = CryptoImpl.Decrypt(providerService.Properties.Get(ACCESS_KEY_ID)); - string keySecret = CryptoImpl.Decrypt(providerService.Properties.Get(SECRET_ACCESS_KEY)); + string keyId = GetEncryptedPropertyValue(ACCESS_KEY, ACCESS_KEY_ID_DEPRECATED); + string keySecret = GetEncryptedPropertyValue(SECRET_ACCESS_KEY, SECRET_ACCESS_KEY_DEPRECATED); AWSCredentials credentials = null; if (!string.IsNullOrEmpty(keyId) && !string.IsNullOrEmpty(keySecret)) { credentials = new BasicAWSCredentials(keyId, keySecret); } - var region = Amazon.RegionEndpoint.GetBySystemName(providerService.Properties.Get(REGION)); + var region = Amazon.RegionEndpoint.GetBySystemName(GetPropertyValue(REGION, REGION_DEPRECATED)); - Endpoint = providerService.Properties.Get(ENDPOINT); + Endpoint = GetPropertyValue( STORAGE_ENDPOINT , ENDPOINT_DEPRECATED); if (Endpoint == STORAGE_CUSTOM_ENDPOINT_VALUE) { - Endpoint = providerService.Properties.Get(STORAGE_CUSTOM_ENDPOINT); + Endpoint = GetPropertyValue(STORAGE_CUSTOM_ENDPOINT, STORAGE_CUSTOM_ENDPOINT_DEPRECATED); ForcePathStyle = true; } @@ -96,13 +117,27 @@ public ExternalProviderS3(GXService providerService) } #endif - Bucket = CryptoImpl.Decrypt(providerService.Properties.Get(BUCKET)); - Folder = providerService.Properties.Get(FOLDER); + Bucket = GetEncryptedPropertyValue(BUCKET, BUCKET_DEPRECATED); + Folder = GetPropertyValue(FOLDER, FOLDER_DEPRECATED); + Region = region.SystemName; + SetURI(); CreateBucket(); CreateFolder(Folder); } + private void SetURI() + { + if (Region == DEFAULT_REGION) + { + _storageUri = $"https://{Bucket}.{Endpoint}/"; + } + else + { + _storageUri = $"https://{Bucket}.{Endpoint.Replace("s3.amazonaws.com", $"s3.{Region.ToLower()}.amazonaws.com")}/"; + } + } + private void AddObjectMetadata(MetadataCollection metadata, string tableName, string fieldName, string key) { metadata.Add("Table", tableName); @@ -173,11 +208,11 @@ private void CreateBucket() PutBucketRequest request = new PutBucketRequest { BucketName = Bucket, - UseClientRegion = true, - // Every bucket is public - CannedACL = S3CannedACL.PublicRead + UseClientRegion = true }; - + if (defaultAcl == GxFileType.Public) { + request.CannedACL = S3CannedACL.PublicRead; + } PutBucket(request); } } @@ -200,7 +235,7 @@ private static bool IsPrivateUpload(GxFileType fileType) return fileType.HasFlag(GxFileType.Private); } - public string Get(string objectName, GxFileType fileType, int urlMinutes = PRIVATE_URL_MINUTES_EXPIRATION) + public string Get(string objectName, GxFileType fileType, int urlMinutes = DefaultExpirationMinutes) { bool isPrivate = IsPrivateUpload(fileType); if (Exists(objectName, fileType)) @@ -247,7 +282,7 @@ public void Delete(string objectName, GxFileType fileType) //https://github.com/aws/aws-sdk-net/blob/master/sdk/src/Services/S3/Custom/_bcl/IO/S3FileInfo.cs public bool Exists(string objectName, GxFileType fileType) { - bool exists = true; + bool exists; try { exists = new S3FileInfo(Client, Bucket, objectName).Exists; @@ -280,9 +315,9 @@ public string Copy(string objectName, GxFileType sourceFileType, string newName, return StorageUri + StorageUtils.EncodeUrl(newName); } - private static S3CannedACL GetCannedACL(GxFileType destFileType) + private S3CannedACL GetCannedACL(GxFileType destFileType) { - return (destFileType.HasFlag(GxFileType.Private)) ? S3CannedACL.Private : S3CannedACL.PublicRead; + return defaultAcl.HasFlag(GxFileType.Private) || destFileType.HasFlag(GxFileType.Private) ? S3CannedACL.Private : S3CannedACL.PublicRead; } public string Upload(string fileName, Stream stream, GxFileType destFileType) @@ -338,7 +373,7 @@ public string Save(Stream fileStream, string fileName, string tableName, string AddObjectMetadata(objectRequest.Metadata, tableName, fieldName, resourceKey); PutObjectResponse result = PutObject(objectRequest); - return "https://" + Bucket + ".s3.amazonaws.com/" + resourceKey; + return StorageUri + resourceKey; } catch (Exception ex) { @@ -469,5 +504,10 @@ public bool GetObjectNameFromURL(string url, out string objectName) objectName = null; return false; } + + public override string GetName() + { + return Name; + } } } From 182c1b46b7437638bb6902b1023fd286973dc4aa Mon Sep 17 00:00:00 2001 From: Gonzalo Date: Thu, 10 Jun 2021 18:38:42 -0300 Subject: [PATCH 04/42] Initial bits with Unit Tests --- .../Configuration/ExternalStorage.cs | 40 +-- .../dotnetframework/GxClasses/Core/GXUtils.cs | 6 +- .../GxClasses/Core/GXUtilsCommon.cs | 4 +- .../GxClasses/Data/GXDataNTierADO.cs | 16 +- .../GxClasses/Domain/GXFileIO.cs | 12 +- .../Services/Storage/ExternalProviderBase.cs | 33 +- .../src/dotnetframework/GxExcel/GxExcelI.cs | 2 +- .../Storage/GXAmazonS3/ExternalProviderS3.cs | 51 ++- .../AzureStorageExternalProvider.cs | 37 +- .../GXGoogleCloud/ExternalProviderGoogle.cs | 72 ++-- .../test/DotNetUnitTest/DotNetUnitTest.csproj | 14 + .../ExternalProviderAzureTest.cs | 14 + .../ExternalProviderGoogleTest.cs | 15 + .../ExternalProviderS3Test.cs | 13 + .../ExternalProvider/ExternalProviderTest.cs | 339 ++++++++++++++++++ .../Properties/launchSettings.json | 11 + dotnet/test/DotNetUnitTest/resources/text.txt | 1 + 17 files changed, 562 insertions(+), 118 deletions(-) create mode 100644 dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderAzureTest.cs create mode 100644 dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderGoogleTest.cs create mode 100644 dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderS3Test.cs create mode 100644 dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs create mode 100644 dotnet/test/DotNetUnitTest/Properties/launchSettings.json create mode 100644 dotnet/test/DotNetUnitTest/resources/text.txt diff --git a/dotnet/src/dotnetframework/GxClasses/Configuration/ExternalStorage.cs b/dotnet/src/dotnetframework/GxClasses/Configuration/ExternalStorage.cs index 37b103595..d37cfcfbb 100644 --- a/dotnet/src/dotnetframework/GxClasses/Configuration/ExternalStorage.cs +++ b/dotnet/src/dotnetframework/GxClasses/Configuration/ExternalStorage.cs @@ -95,7 +95,7 @@ public bool Connect(string profileName, GXProperties properties, ref GxStoragePr private void preprocess(String name, GXProperties properties) { - string className = null; + string className; switch (name) { @@ -104,37 +104,27 @@ private void preprocess(String name, GXProperties properties) className = "GeneXus.Storage.GXAmazonS3.ExternalProviderS3"; SetDefaultProperty(properties, "STORAGE_PROVIDER_REGION", "us-east-1"); SetDefaultProperty(properties, "STORAGE_ENDPOINT", "s3.amazonaws.com"); - SetEncryptProperty(properties, "STORAGE_PROVIDER_ACCESSKEYID"); - SetEncryptProperty(properties, "STORAGE_PROVIDER_SECRETACCESSKEY"); - SetEncryptProperty(properties, "BUCKET_NAME"); + SetEncryptedProperty(properties, "STORAGE_PROVIDER_ACCESSKEYID"); + SetEncryptedProperty(properties, "STORAGE_PROVIDER_SECRETACCESSKEY"); + SetEncryptedProperty(properties, "BUCKET_NAME"); break; case "AZURESTORAGE": className = "GeneXus.Storage.GXAzureStorage.AzureStorageExternalProvider"; - SetEncryptProperty(properties, "PUBLIC_CONTAINER_NAME"); - SetEncryptProperty(properties, "PRIVATE_CONTAINER_NAME"); - SetEncryptProperty(properties, "ACCOUNT_NAME"); - SetEncryptProperty(properties, "ACCESS_KEY"); + SetEncryptedProperty(properties, "PUBLIC_CONTAINER_NAME"); + SetEncryptedProperty(properties, "PRIVATE_CONTAINER_NAME"); + SetEncryptedProperty(properties, "ACCOUNT_NAME"); + SetEncryptedProperty(properties, "ACCESS_KEY"); break; - - case "BLUEMIXSTORAGE": - className = "GeneXus.Storage.GXBluemix.ExternalProviderBluemix"; - SetDefaultProperty(properties, "SERVER_URL", "https://identity.open.softlayer.com"); - SetDefaultProperty(properties, "STORAGE_PROVIDER_REGION", "dallas"); - SetEncryptProperty(properties, "PUBLIC_BUCKET_NAME"); - SetEncryptProperty(properties, "PRIVATE_BUCKET_NAME"); - SetEncryptProperty(properties, "STORAGE_PROVIDER_USER"); - SetEncryptProperty(properties, "STORAGE_PROVIDER_PASSWORD"); - break; - + //case "BOX": // className = "{class}"; // break; case "GOOGLE": className = "GeneXus.Storage.GXGoogleCloud.ExternalProviderGoogle"; - SetEncryptProperty(properties, "KEY"); - SetEncryptProperty(properties, "BUCKET_NAME"); + SetEncryptedProperty(properties, "KEY"); + SetEncryptedProperty(properties, "BUCKET_NAME"); break; //case "IBMCOS": @@ -146,9 +136,9 @@ private void preprocess(String name, GXProperties properties) case "OPENSTACKSTORAGE": className = "GeneXus.Storage.GXOpenStack.ExternalProviderOpenStack"; - SetEncryptProperty(properties, "BUCKET_NAME"); - SetEncryptProperty(properties, "STORAGE_PROVIDER_USER"); - SetEncryptProperty(properties, "STORAGE_PROVIDER_PASSWORD"); + SetEncryptedProperty(properties, "BUCKET_NAME"); + SetEncryptedProperty(properties, "STORAGE_PROVIDER_USER"); + SetEncryptedProperty(properties, "STORAGE_PROVIDER_PASSWORD"); break; default: @@ -171,7 +161,7 @@ private void SetDefaultProperty(GXProperties properties, String prop, String val properties.Set(prop, value); } - private void SetEncryptProperty(GXProperties properties, String prop) + private void SetEncryptedProperty(GXProperties properties, String prop) { String value = properties.Get(prop); if (string.IsNullOrEmpty(value)) diff --git a/dotnet/src/dotnetframework/GxClasses/Core/GXUtils.cs b/dotnet/src/dotnetframework/GxClasses/Core/GXUtils.cs index 5d9e5dba6..bdd815c9a 100644 --- a/dotnet/src/dotnetframework/GxClasses/Core/GXUtils.cs +++ b/dotnet/src/dotnetframework/GxClasses/Core/GXUtils.cs @@ -1117,7 +1117,7 @@ public bool Upload(string filefullpath, string storageobjectfullname, GxFile upl { storageobjectfullname = Path.GetFileName(filefullpath); } - string url = provider.Upload(filefullpath, storageobjectfullname, GxFileType.Public); + string url = provider.Upload(filefullpath, storageobjectfullname, GxFileType.PublicRead); uploadedFile.FileInfo = new GxExternalFileInfo(storageobjectfullname, url, provider); return true; } @@ -1157,7 +1157,7 @@ public bool Download(string storageobjectfullname, GxFile localFile, GXBaseColle destFileName = localFile.GetAbsoluteName(); else destFileName = Path.Combine(GxContext.StaticPhysicalPath(), localFile.Source); - provider.Download(storageobjectfullname, destFileName, GxFileType.Public); + provider.Download(storageobjectfullname, destFileName, GxFileType.PublicRead); return true; } catch (Exception ex) @@ -1194,7 +1194,7 @@ public bool Get(string storageobjectfullname, GxFile externalFile, GXBaseCollect try { ValidProvider(); - string url = provider.Get(storageobjectfullname, GxFileType.Public, 0); + string url = provider.Get(storageobjectfullname, GxFileType.PublicRead, 0); if (String.IsNullOrEmpty(url)) { GXUtil.ErrorToMessages("Get Error", "File doesn't exists", messages); diff --git a/dotnet/src/dotnetframework/GxClasses/Core/GXUtilsCommon.cs b/dotnet/src/dotnetframework/GxClasses/Core/GXUtilsCommon.cs index b7f5f9478..a3dc44cb7 100644 --- a/dotnet/src/dotnetframework/GxClasses/Core/GXUtilsCommon.cs +++ b/dotnet/src/dotnetframework/GxClasses/Core/GXUtilsCommon.cs @@ -3366,7 +3366,7 @@ public static string UriToPath(string uriString) return uriString; } } - public static string getTempFileName(string baseDir, string name, string extension, GxFileType fileType = GxFileType.Public) + public static string getTempFileName(string baseDir, string name, string extension, GxFileType fileType = GxFileType.PublicRead) { String fileName; name = FileUtil.FileNamePrettify(name); @@ -5265,7 +5265,7 @@ public static string ResolveUri(string uriString, bool absUrl, IGxContext contex string basePath = Path.Combine(Path.Combine(Preferences.getBLOB_PATH(), MultimediaDirectory)); try { - GxFile file = new GxFile(string.Empty, PathUtil.SafeCombine(basePath, fileName), GxFileType.Public); + GxFile file = new GxFile(string.Empty, PathUtil.SafeCombine(basePath, fileName), GxFileType.PublicRead); return PathToUrl(file.GetURI(), absUrl, context); } catch (ArgumentException ex) diff --git a/dotnet/src/dotnetframework/GxClasses/Data/GXDataNTierADO.cs b/dotnet/src/dotnetframework/GxClasses/Data/GXDataNTierADO.cs index e1904d8bd..10befd758 100644 --- a/dotnet/src/dotnetframework/GxClasses/Data/GXDataNTierADO.cs +++ b/dotnet/src/dotnetframework/GxClasses/Data/GXDataNTierADO.cs @@ -149,7 +149,7 @@ public string getMultimediaFile(int id, string gxdbFileUri) string filePath = PathUtil.SafeCombine(_gxDbCommand.Conn.MultimediaPath, fileName); try { - GxFile file = new GxFile(string.Empty, filePath, GxFileType.Public); + GxFile file = new GxFile(string.Empty, filePath, GxFileType.PublicRead); if (file.Exists()) { return filePath; @@ -433,7 +433,7 @@ public string getMultimediaFile(int id, string gxdbFileUri) try { - GxFile file = new GxFile(string.Empty, filePath, GxFileType.Public); + GxFile file = new GxFile(string.Empty, filePath, GxFileType.PublicRead); if (file.Exists()) { @@ -441,7 +441,7 @@ public string getMultimediaFile(int id, string gxdbFileUri) } else { - return getBLOBFile(id, FileUtil.GetFileType(gxdbFileUri), FileUtil.GetFileName(gxdbFileUri), filePath, false, GxFileType.Public); + return getBLOBFile(id, FileUtil.GetFileType(gxdbFileUri), FileUtil.GetFileName(gxdbFileUri), filePath, false, GxFileType.PublicRead); } } catch (ArgumentException) @@ -617,7 +617,7 @@ public void SetParameterMultimedia(int id, string image_gxi, string image, strin { try { - multimediaUri = ServiceFactory.GetExternalProvider().Copy(image_gxi, GXDbFile.GenerateUri(image_gxi, !GXDbFile.HasToken(image_gxi), false), tableName, fieldName, GxFileType.Public); + multimediaUri = ServiceFactory.GetExternalProvider().Copy(image_gxi, GXDbFile.GenerateUri(image_gxi, !GXDbFile.HasToken(image_gxi), false), tableName, fieldName, GxFileType.PublicRead); GXLogging.Debug(log, "Copy file already in ExternalProvider:", multimediaUri); } catch (Exception ex) @@ -635,7 +635,7 @@ public void SetParameterMultimedia(int id, string image_gxi, string image, strin using (var fileStream = new MemoryStream(new WebClient().DownloadData(image_gxi))) { //Cannot pass Http Stream directly, because some Providers (AWS S3) does not support Http Stream. - multimediaUri = ServiceFactory.GetExternalProvider().Save(fileStream, GXDbFile.GenerateUri(image_gxi, !GXDbFile.HasToken(image_gxi), false), tableName, fieldName, GxFileType.Public); + multimediaUri = ServiceFactory.GetExternalProvider().Save(fileStream, GXDbFile.GenerateUri(image_gxi, !GXDbFile.HasToken(image_gxi), false), tableName, fieldName, GxFileType.PublicRead); GXLogging.Debug(log, "Upload external file to ExternalProvider:", multimediaUri); } #pragma warning disable SYSLIB0014 // WebClient @@ -657,7 +657,7 @@ public void SetParameterMultimedia(int id, string image_gxi, string image, strin String fileName = PathUtil.GetValidFileName(fileFullName, "_"); using (fileStream) { - multimediaUri = ServiceFactory.GetExternalProvider().Save(fileStream, GXDbFile.GenerateUri(fileName, !GXDbFile.HasToken(fileName), false), tableName, fieldName, GxFileType.Public); + multimediaUri = ServiceFactory.GetExternalProvider().Save(fileStream, GXDbFile.GenerateUri(fileName, !GXDbFile.HasToken(fileName), false), tableName, fieldName, GxFileType.PublicRead); GXLogging.Debug(log, "Upload file (_gxi) to ExternalProvider:", multimediaUri); } } @@ -665,7 +665,7 @@ public void SetParameterMultimedia(int id, string image_gxi, string image, strin { try { - multimediaUri = ServiceFactory.GetExternalProvider().Copy(image, GXDbFile.GenerateUri(image_gxi, !GXDbFile.HasToken(image_gxi), false), tableName, fieldName, GxFileType.Public); + multimediaUri = ServiceFactory.GetExternalProvider().Copy(image, GXDbFile.GenerateUri(image_gxi, !GXDbFile.HasToken(image_gxi), false), tableName, fieldName, GxFileType.PublicRead); GXLogging.Debug(log, "Copy external file in ExternalProvider:", multimediaUri); } catch(Exception e) @@ -726,7 +726,7 @@ private static string PushToExternalProvider(Stream fileStream, string externalF string multimediaUri; using (fileStream) { - multimediaUri = ServiceFactory.GetExternalProvider().Save(fileStream, externalFileName, tableName, fieldName, GxFileType.Public); + multimediaUri = ServiceFactory.GetExternalProvider().Save(fileStream, externalFileName, tableName, fieldName, GxFileType.PublicRead); GXLogging.Debug(log, "Upload file to ExternalProvider:", multimediaUri); } diff --git a/dotnet/src/dotnetframework/GxClasses/Domain/GXFileIO.cs b/dotnet/src/dotnetframework/GxClasses/Domain/GXFileIO.cs index bfb0243e8..963328abf 100644 --- a/dotnet/src/dotnetframework/GxClasses/Domain/GXFileIO.cs +++ b/dotnet/src/dotnetframework/GxClasses/Domain/GXFileIO.cs @@ -209,7 +209,7 @@ public IGxFileInfo[] GetFiles(string searchPattern) searchPattern = ""; List files = _provider.GetFiles(_name, searchPattern); - GxExternalFileInfo[] externalFiles = files.Select(elem => new GxExternalFileInfo(elem, _provider, GxFileType.Public)).ToArray(); + GxExternalFileInfo[] externalFiles = files.Select(elem => new GxExternalFileInfo(elem, _provider, GxFileType.PublicRead)).ToArray(); return externalFiles; } @@ -412,7 +412,7 @@ public class GxExternalFileInfo : IGxFileInfo private string _name; private ExternalProvider _provider; private string _url; - private GxFileType _fileTypeAtt = GxFileType.Public; + private GxFileType _fileTypeAtt = GxFileType.PublicRead; public GxExternalFileInfo(ExternalProvider provider) { @@ -445,7 +445,7 @@ public GxExternalFileInfo(string storageObjectFullname, ExternalProvider provide _fileTypeAtt = fileType; } - public GxExternalFileInfo(string storageObjectFullname, string url, ExternalProvider provider, GxFileType fileType = GxFileType.Public) + public GxExternalFileInfo(string storageObjectFullname, string url, ExternalProvider provider, GxFileType fileType = GxFileType.PublicRead) { _name = storageObjectFullname; _provider = provider; @@ -643,7 +643,7 @@ public Stream GetStream() public enum GxFileType { Default = 0, - Public = 1, + PublicRead = 1, Private = 2 } @@ -670,13 +670,13 @@ public GxFile(string baseDirectory) _baseDirectory = _baseDirectory.Substring(0, _baseDirectory.Length - 1); } - public GxFile(string baseDirectory, IGxFileInfo file, GxFileType fileType = GxFileType.Public) + public GxFile(string baseDirectory, IGxFileInfo file, GxFileType fileType = GxFileType.PublicRead) : this(baseDirectory) { _file = file; } - public GxFile(string baseDirectory, string fileName, GxFileType fileType = GxFileType.Public) + public GxFile(string baseDirectory, string fileName, GxFileType fileType = GxFileType.PublicRead) : this(baseDirectory) { if (GxUploadHelper.IsUpload(fileName)) diff --git a/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderBase.cs b/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderBase.cs index 1bef72a61..a495f88a3 100644 --- a/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderBase.cs +++ b/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderBase.cs @@ -42,7 +42,7 @@ private void Initialize() } } - protected String GetEncryptedPropertyValue(String propertyName, String alternativePropertyName) + protected String GetEncryptedPropertyValue(String propertyName, String alternativePropertyName = null) { String value = GetEncryptedPropertyValue(propertyName, alternativePropertyName, null); if (value == null) @@ -71,7 +71,7 @@ protected String GetEncryptedPropertyValue(String propertyName, String alternati return value; } - protected String GetPropertyValue(String propertyName, String alternativePropertyName) + protected String GetPropertyValue(String propertyName, String alternativePropertyName = null) { String value = GetPropertyValue(propertyName, alternativePropertyName, null); if (value == null) @@ -85,26 +85,31 @@ protected String GetPropertyValue(String propertyName, String alternativePropert protected String GetPropertyValue(String propertyName, String alternativePropertyName, String defaultValue) { - propertyName = ResolvePropertyName(propertyName); - String value = Environment.GetEnvironmentVariable(propertyName); - if (String.IsNullOrEmpty(value)) - { - value = Environment.GetEnvironmentVariable(alternativePropertyName); - } - if (this.service != null) + String value = String.Empty; + value = string.IsNullOrEmpty(value) ? GetPropertyValueImpl(ResolvePropertyName(propertyName)) : value; + value = string.IsNullOrEmpty(value) ? GetPropertyValueImpl(propertyName) : value; + value = string.IsNullOrEmpty(value) ? GetPropertyValueImpl(alternativePropertyName) : value; + value = string.IsNullOrEmpty(value) ? defaultValue : value; + return value; + } + + private string GetPropertyValueImpl(string propertyName) + { + String value = String.Empty; + if (!string.IsNullOrEmpty(propertyName)) { - value = this.service.Properties.Get(propertyName); - if (String.IsNullOrEmpty(value)) + value = Environment.GetEnvironmentVariable(propertyName); + if (this.service != null) { - value = this.service.Properties.Get(alternativePropertyName); + value = this.service.Properties.Get(propertyName); } } - return !String.IsNullOrEmpty(value) ? value : defaultValue; + return value; } protected String ResolvePropertyName(String propertyName) { - return String.Format("STORAGE_%s_%s", GetName(), propertyName); + return $"STORAGE_{GetName()}_{propertyName}"; } } diff --git a/dotnet/src/dotnetframework/GxExcel/GxExcelI.cs b/dotnet/src/dotnetframework/GxExcel/GxExcelI.cs index 8317f9bc5..79168b9f3 100644 --- a/dotnet/src/dotnetframework/GxExcel/GxExcelI.cs +++ b/dotnet/src/dotnetframework/GxExcel/GxExcelI.cs @@ -144,7 +144,7 @@ public String Template string localTemplate = value; if (!Path.IsPathRooted(localTemplate)) localTemplate = Path.Combine(GxContext.StaticPhysicalPath(), localTemplate); - ServiceFactory.GetExternalProvider().Upload(localTemplate, template, GxFileType.Public); + ServiceFactory.GetExternalProvider().Upload(localTemplate, template, GxFileType.PublicRead); } } else if (!Path.IsPathRooted(value)) diff --git a/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs b/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs index cd5c179a7..54accf387 100644 --- a/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs +++ b/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs @@ -16,7 +16,7 @@ namespace GeneXus.Storage.GXAmazonS3 { public class ExternalProviderS3 : ExternalProviderBase, ExternalProvider { - const string Name = "AWSS3"; + public const string Name = "AWSS3"; const string ACCESS_KEY = "ACCESS_KEY"; const string SECRET_ACCESS_KEY = "SECRET_KEY"; @@ -66,13 +66,16 @@ public string GetBaseURL() return StorageUri + Folder + StorageUtils.DELIMITER; } - public ExternalProviderS3() - : this(ServiceFactory.GetGXServices().Get(GXServices.STORAGE_SERVICE)) - { + public ExternalProviderS3(): this(null) + { } - public ExternalProviderS3(GXService providerService) + public ExternalProviderS3(GXService providerService): base(providerService) { + Initialize(); + } + + private void Initialize() { string keyId = GetEncryptedPropertyValue(ACCESS_KEY, ACCESS_KEY_ID_DEPRECATED); string keySecret = GetEncryptedPropertyValue(SECRET_ACCESS_KEY, SECRET_ACCESS_KEY_DEPRECATED); AWSCredentials credentials = null; @@ -83,20 +86,20 @@ public ExternalProviderS3(GXService providerService) var region = Amazon.RegionEndpoint.GetBySystemName(GetPropertyValue(REGION, REGION_DEPRECATED)); + AmazonS3Config config = new AmazonS3Config() + { + RegionEndpoint = region + }; + Endpoint = GetPropertyValue( STORAGE_ENDPOINT , ENDPOINT_DEPRECATED); if (Endpoint == STORAGE_CUSTOM_ENDPOINT_VALUE) { Endpoint = GetPropertyValue(STORAGE_CUSTOM_ENDPOINT, STORAGE_CUSTOM_ENDPOINT_DEPRECATED); ForcePathStyle = true; + config.ForcePathStyle = ForcePathStyle; + config.ServiceURL = Endpoint; } - - AmazonS3Config config = new AmazonS3Config() - { - RegionEndpoint = region, - ServiceURL = Endpoint, - ForcePathStyle = ForcePathStyle - }; - + #if NETCORE if (credentials != null) { @@ -210,7 +213,7 @@ private void CreateBucket() BucketName = Bucket, UseClientRegion = true }; - if (defaultAcl == GxFileType.Public) { + if (defaultAcl == GxFileType.PublicRead) { request.CannedACL = S3CannedACL.PublicRead; } PutBucket(request); @@ -230,9 +233,9 @@ public string Upload(string localFile, string objectName, GxFileType fileType) return Get(objectName, fileType); } - private static bool IsPrivateUpload(GxFileType fileType) + private bool IsPrivateUpload(GxFileType fileType) { - return fileType.HasFlag(GxFileType.Private); + return GetCannedACL(fileType) != S3CannedACL.PublicRead; } public string Get(string objectName, GxFileType fileType, int urlMinutes = DefaultExpirationMinutes) @@ -315,9 +318,19 @@ public string Copy(string objectName, GxFileType sourceFileType, string newName, return StorageUri + StorageUtils.EncodeUrl(newName); } - private S3CannedACL GetCannedACL(GxFileType destFileType) + private S3CannedACL GetCannedACL(GxFileType acl) { - return defaultAcl.HasFlag(GxFileType.Private) || destFileType.HasFlag(GxFileType.Private) ? S3CannedACL.Private : S3CannedACL.PublicRead; + if (acl == GxFileType.Default) + { + acl = this.defaultAcl; + } + + S3CannedACL accessControl = S3CannedACL.Private; + if (acl == GxFileType.PublicRead) + { + accessControl = S3CannedACL.PublicRead; + } + return accessControl; } public string Upload(string fileName, Stream stream, GxFileType destFileType) @@ -426,7 +439,7 @@ public void RenameDirectory(string directoryName, string newDirectoryName) if (file.Type == FileSystemType.Directory) RenameDirectory(directoryName + "\\" + file.Name, newDirectoryName + "\\" + file.Name); else - Rename(directoryName.Replace("\\", StorageUtils.DELIMITER) + StorageUtils.DELIMITER + file.Name, newDirectoryName.Replace("\\", StorageUtils.DELIMITER) + StorageUtils.DELIMITER + file.Name, GxFileType.Public); + Rename(directoryName.Replace("\\", StorageUtils.DELIMITER) + StorageUtils.DELIMITER + file.Name, newDirectoryName.Replace("\\", StorageUtils.DELIMITER) + StorageUtils.DELIMITER + file.Name, GxFileType.PublicRead); } s3DirectoryInfo.Delete(); } diff --git a/dotnet/src/dotnetframework/Providers/Storage/GXAzureStorage/AzureStorageExternalProvider.cs b/dotnet/src/dotnetframework/Providers/Storage/GXAzureStorage/AzureStorageExternalProvider.cs index 811b079db..1b46ada6c 100644 --- a/dotnet/src/dotnetframework/Providers/Storage/GXAzureStorage/AzureStorageExternalProvider.cs +++ b/dotnet/src/dotnetframework/Providers/Storage/GXAzureStorage/AzureStorageExternalProvider.cs @@ -14,8 +14,10 @@ namespace GeneXus.Storage.GXAzureStorage { - public class AzureStorageExternalProvider : ExternalProvider + public class AzureStorageExternalProvider : ExternalProviderBase, ExternalProvider { + public static String Name = "AZUREBS"; //Azure Blob Storage + const string ACCOUNT_NAME = "ACCOUNT_NAME"; const string ACCESS_KEY = "ACCESS_KEY"; const string PUBLIC_CONTAINER = "PUBLIC_CONTAINER_NAME"; @@ -26,22 +28,35 @@ public class AzureStorageExternalProvider : ExternalProvider CloudBlobContainer PublicContainer { get; set; } CloudBlobContainer PrivateContainer { get; set; } CloudBlobClient Client { get; set; } + public string StorageUri { get { return $"https://{Account}.blob.core.windows.net"; } } - public AzureStorageExternalProvider() - : this(ServiceFactory.GetGXServices().Get(GXServices.STORAGE_SERVICE)) { } + public override string GetName() + { + return Name; + } + + public AzureStorageExternalProvider() : this(null) + { + } + + public AzureStorageExternalProvider(GXService providerService) : base(providerService) + { + Initialize(); + } - public AzureStorageExternalProvider(GXService providerService) + private void Initialize() { - Account = CryptoImpl.Decrypt(providerService.Properties.Get(ACCOUNT_NAME)); - Key = CryptoImpl.Decrypt(providerService.Properties.Get(ACCESS_KEY)); + Account = GetEncryptedPropertyValue(ACCOUNT_NAME); + Key = GetEncryptedPropertyValue(ACCESS_KEY); - string publicContainer = CryptoImpl.Decrypt(providerService.Properties.Get(PUBLIC_CONTAINER)); - string privateContainer = CryptoImpl.Decrypt(providerService.Properties.Get(PRIVATE_CONTAINER)); + string publicContainer = GetEncryptedPropertyValue(PUBLIC_CONTAINER); + string privateContainer = GetEncryptedPropertyValue(PRIVATE_CONTAINER); + StorageCredentials credentials = new StorageCredentials(Account, Key); CloudStorageAccount storageAccount = new CloudStorageAccount(credentials, true); @@ -254,15 +269,15 @@ public void RenameDirectory(string directoryName, string newDirectoryName) { CloudBlockBlob blob = (CloudBlockBlob)item; fileName = Path.GetFileName(blob.Name); - Copy(blob.Name, GxFileType.Public, newDirectoryName + fileName, GxFileType.Public); - Delete(blob.Name, GxFileType.Public); + Copy(blob.Name, GxFileType.PublicRead, newDirectoryName + fileName, GxFileType.PublicRead); + Delete(blob.Name, GxFileType.PublicRead); } else if (item.GetType() == typeof(CloudPageBlob)) { CloudPageBlob pageBlob = (CloudPageBlob)item; fileName = Path.GetFileName(pageBlob.Name); - Copy(directoryName + fileName, GxFileType.Public, newDirectoryName + fileName, GxFileType.Public); + Copy(directoryName + fileName, GxFileType.PublicRead, newDirectoryName + fileName, GxFileType.PublicRead); } else if (item.GetType() == typeof(CloudBlobDirectory)) diff --git a/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs b/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs index 7e72b04b7..c53ab783f 100644 --- a/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs +++ b/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs @@ -15,15 +15,16 @@ namespace GeneXus.Storage.GXGoogleCloud { - public class ExternalProviderGoogle : ExternalProvider + public class ExternalProviderGoogle : ExternalProviderBase, ExternalProvider { - const int BUCKET_EXISTS = 409; + public static String Name = "GOOGLECS"; //Google Cloud Storage + const int BUCKET_EXISTS = 409; const int OBJECT_NOT_FOUND = 404; const string APPLICATION_NAME = "APPLICATION_NAME"; const string PROJECT_ID = "PROJECT_ID"; const string KEY = "KEY"; const string BUCKET = "BUCKET_NAME"; - const string FOLDER = "FOLDER_NAME"; + StorageClient Client { get; set; } StorageService Service { get; set; } @@ -36,35 +37,48 @@ public string StorageUri get { return $"https://{Bucket}.storage.googleapis.com/"; } } - public ExternalProviderGoogle() - : this(ServiceFactory.GetGXServices().Get(GXServices.STORAGE_SERVICE)) - { - } + public override string GetName() + { + return Name; - public ExternalProviderGoogle(GXService providerService) - { + } + + public ExternalProviderGoogle() : this(null) + { + } + + public ExternalProviderGoogle(GXService providerService) : base(providerService) + { + Initialize(); + } + + + private void Initialize() + { GoogleCredential credentials; - using (Stream stream = KeyStream(CryptoImpl.Decrypt(providerService.Properties.Get(KEY)))) - { - credentials = GoogleCredential.FromStream(stream).CreateScoped(StorageService.Scope.CloudPlatform); - } + string key = GetEncryptedPropertyValue(KEY); - using (Stream stream = KeyStream(CryptoImpl.Decrypt(providerService.Properties.Get(KEY)))) + using (Stream stream = KeyStream(key)) { - Signer = UrlSigner.FromServiceAccountData(stream); - } + credentials = GoogleCredential.FromStream(stream).CreateScoped(StorageService.Scope.CloudPlatform); + } + + using (Stream stream = KeyStream(key)) + { + Signer = UrlSigner.FromServiceAccountData(stream); + } - Client = StorageClient.Create(credentials); + Client = StorageClient.Create(credentials); Service = new StorageService(new BaseClientService.Initializer() { HttpClientInitializer = credentials, - ApplicationName = providerService.Properties.Get(APPLICATION_NAME) + ApplicationName = GetPropertyValue(APPLICATION_NAME) }); - Project = providerService.Properties.Get(PROJECT_ID); - Bucket = CryptoImpl.Decrypt(providerService.Properties.Get(BUCKET)); - Folder = providerService.Properties.Get(FOLDER); + Bucket = GetEncryptedPropertyValue(BUCKET); + Project = GetPropertyValue(PROJECT_ID); + Folder = GetPropertyValue(FOLDER); CreateBucket(); CreateFolder(Folder); @@ -176,7 +190,7 @@ public string Upload(string localFile, string objectName, GxFileType fileType) using (FileStream stream = new FileStream(localFile, FileMode.Open)) { Google.Apis.Storage.v1.Data.Object obj = Client.UploadObject(Bucket, objectName, "application/octet-stream", stream, GetUploadOptions(fileType)); - return obj.MediaLink; + return GetURL(objectName, fileType, DefaultExpirationMinutes); } } @@ -279,12 +293,12 @@ public void DeleteDirectory(string directoryName) foreach (Google.Apis.Storage.v1.Data.Object item in Client.ListObjects(Bucket, directoryName)) { if (!item.Name.EndsWith(StorageUtils.DELIMITER)) - Delete(item.Name, GxFileType.Public); + Delete(item.Name, GxFileType.PublicRead); } foreach (string subdir in GetSubDirectories(directoryName)) DeleteDirectory(subdir); - if (Exists(directoryName, GxFileType.Public)) - Delete(directoryName, GxFileType.Public); + if (Exists(directoryName, GxFileType.PublicRead)) + Delete(directoryName, GxFileType.PublicRead); } public bool ExistsDirectory(string directoryName) @@ -377,13 +391,13 @@ public void RenameDirectory(string directoryName, string newDirectoryName) { if (IsFile(item.Name)) { - Copy(item.Name, GxFileType.Public, item.Name.Replace(directoryName, newDirectoryName), GxFileType.Public); - Delete(item.Name, GxFileType.Public); + Copy(item.Name, GxFileType.PublicRead, item.Name.Replace(directoryName, newDirectoryName), GxFileType.PublicRead); + Delete(item.Name, GxFileType.PublicRead); } } CreateDirectory(newDirectoryName); - if (Exists(directoryName, GxFileType.Public)) - Delete(directoryName, GxFileType.Public); + if (Exists(directoryName, GxFileType.PublicRead)) + Delete(directoryName, GxFileType.PublicRead); request.PageToken = response.NextPageToken; } while (response.NextPageToken != null); foreach (string subdir in GetSubDirectories(directoryName)) diff --git a/dotnet/test/DotNetUnitTest/DotNetUnitTest.csproj b/dotnet/test/DotNetUnitTest/DotNetUnitTest.csproj index 144d9f9a6..be0c33de9 100644 --- a/dotnet/test/DotNetUnitTest/DotNetUnitTest.csproj +++ b/dotnet/test/DotNetUnitTest/DotNetUnitTest.csproj @@ -28,6 +28,7 @@ + all runtime; build; native; contentfiles; analyzers; buildtransitive @@ -35,9 +36,22 @@ + + + + + + Always + + + + + Always + + diff --git a/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderAzureTest.cs b/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderAzureTest.cs new file mode 100644 index 000000000..c78b22364 --- /dev/null +++ b/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderAzureTest.cs @@ -0,0 +1,14 @@ +using GeneXus.Storage.GXAmazonS3; +using GeneXus.Storage.GXAzureStorage; +using UnitTesting; + +namespace DotNetUnitTest +{ + public class ExternalProviderAzureTest : ExternalProviderTest + { + public ExternalProviderAzureTest(): base(AzureStorageExternalProvider.Name, typeof(AzureStorageExternalProvider)) + { + } + + } +} diff --git a/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderGoogleTest.cs b/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderGoogleTest.cs new file mode 100644 index 000000000..c58dbb0a1 --- /dev/null +++ b/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderGoogleTest.cs @@ -0,0 +1,15 @@ +using GeneXus.Storage.GXAmazonS3; +using GeneXus.Storage.GXAzureStorage; +using GeneXus.Storage.GXGoogleCloud; +using UnitTesting; + +namespace DotNetUnitTest +{ + public class ExternalProviderGoogleTest : ExternalProviderTest + { + public ExternalProviderGoogleTest(): base(ExternalProviderGoogle.Name, typeof(ExternalProviderGoogle)) + { + } + + } +} diff --git a/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderS3Test.cs b/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderS3Test.cs new file mode 100644 index 000000000..e53655069 --- /dev/null +++ b/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderS3Test.cs @@ -0,0 +1,13 @@ +using GeneXus.Storage.GXAmazonS3; +using UnitTesting; + +namespace DotNetUnitTest +{ + public class ExternalProviderS3Test : ExternalProviderTest + { + public ExternalProviderS3Test(): base(ExternalProviderS3.Name, typeof(ExternalProviderS3)) + { + } + + } +} diff --git a/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs b/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs new file mode 100644 index 000000000..2469d0e70 --- /dev/null +++ b/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs @@ -0,0 +1,339 @@ +using System; +using System.Globalization; +using System.IO; +using System.Net; +using GeneXus.Storage.GXAmazonS3; +using GeneXus.Storage.GXAzureStorage; +using GeneXus.Storage.GXGoogleCloud; +using Xunit; + +#pragma warning disable CA1031 // Do not catch general exception types +namespace UnitTesting +{ + public abstract class ExternalProviderTest + { + private GeneXus.Services.ExternalProvider provider; + private static String TEST_SAMPLE_FILE_NAME = "text.txt"; + private static String TEST_SAMPLE_FILE_PATH = Path.Combine("resources", TEST_SAMPLE_FILE_NAME).ToString(CultureInfo.InvariantCulture); + + + public ExternalProviderTest(string providerName, Type externalProviderType) + { + if (externalProviderType == typeof(ExternalProviderS3)) + { + Environment.SetEnvironmentVariable(providerName + "_TEST_ENABLED", "true"); + Environment.SetEnvironmentVariable("STORAGE_AWSS3_ACCESS_KEY", "AKIAJMQ6SF3Y4IULKD5A"); + Environment.SetEnvironmentVariable("STORAGE_AWSS3_SECRET_KEY", "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"); + Environment.SetEnvironmentVariable("STORAGE_AWSS3_BUCKET_NAME", "genexuss3test"); + Environment.SetEnvironmentVariable("STORAGE_AWSS3_FOLDER_NAME", "gxclasses"); + Environment.SetEnvironmentVariable("STORAGE_AWSS3_REGION", "us-east-1"); + Environment.SetEnvironmentVariable("STORAGE_AWSS3_ENDPOINT", "s3.amazonaws.com"); + } + if (externalProviderType == typeof(AzureStorageExternalProvider)) + { + Environment.SetEnvironmentVariable(providerName + "_TEST_ENABLED", "true"); + Environment.SetEnvironmentVariable($"STORAGE_{providerName}_ACCESS_KEY", "DNiutn2Evl+MNs0TsGUqgg0IDzYWMoMAhLM7Oju/PLi4BEsIsrSY917M6li3Ml7sxj8W+KFD/LZU49OGI40Slg=="); + Environment.SetEnvironmentVariable($"STORAGE_{providerName}_ACCOUNT_NAME", "luistest1"); + Environment.SetEnvironmentVariable($"STORAGE_{providerName}_FOLDER_NAME", "luistest1"); + Environment.SetEnvironmentVariable($"STORAGE_{providerName}_PUBLIC_CONTAINER_NAME", "contluispublic"); + Environment.SetEnvironmentVariable($"STORAGE_{providerName}_PRIVATE_CONTAINER_NAME", "contluisprivate"); + } + + if (externalProviderType == typeof(ExternalProviderGoogle)) + { + Environment.SetEnvironmentVariable(providerName + "_TEST_ENABLED", "true"); + Environment.SetEnvironmentVariable($"STORAGE_{providerName}_KEY", "{\"type\": \"service_account\",\"project_id\": \"gxjavacloudstorageunittests\",\"private_key_id\": \"37e8c0566cd56c7bb07542fdddd9f46ac64a5ba0\",\"private_key\": \"-----BEGIN PRIVATE KEY-----\nMIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDQFJ0NHKkmZNCP\n824SBcXHa0DeoDy1i7zToD1DYiJzQXZAZg9tm91SKEbbCZWMgslBdUP10c7UsnuU\nkvnZTUpZ2YQ680ySiykjAWildp28e/nLBDVZy0f8N7H94484kOS4fxQs2VoW8MH6\nVzsxxm+3yhHjQfQOD+Obes3YBMXUxb7rs/blp7fYWbXNzyGGcpM0kEm4mRNslTbi\nZmVnuwpkaaYfQEPPYLfxWcwfgfpbXEPIJB+bNw+EN5JOUvYvCdpwdPgNyA6MvQFj\nGLTAbsXCOtFh6TtRjE9LkjTZrLTAT0HJ0kODxCyM4pBgag87iZAJ4nsq/355DIMJ\nJtEy3ZtTAgMBAAECggEAYmS89wxMeBlH/inwLJmKMohm/l7rFjXjrnahQZHQFIwp\n7L3WIdCIUWc2SjE4BF9753YaEs2Jbk6P3Wu6taS0udP/kRinZsxjQWhTIZr7b7t4\nHSX6TGGxwnRbuGC4wtjRLuT4l1SYIyzprQU+uoTJIzFsT/hJ/bRJvqXNXI61Na0J\n/ahzB640y75xe8H5Yw06yWXisqD+eiAxX8TU2SRdcZgcIVWaWLDiKhERk7sBctol\nWYgxm3qzpkA+dcvIoykrhLMZGaX9yDu7V9ueXeksqehZrQjSVx0CYSRi5ONy9bcJ\nnQIO4B0lo5oCh6EzKsYlvzJlAPCLJbs4K9Fxviu7QQKBgQDuRBiqAG9J7CR06TqW\nohdCGG6byZZ6x9wmWz3QVLsijFpiQTfu6XHDgh2A4NDau1hOzoJ21rGN5H3XNC9/\ni+8k4Je1nM14aiomlsgNFEp+gUhR7uR54da31q7Vf1DqcQ8ykqOCZitb6/AF0zhU\nlm8syJF1QAi3H7LMN0Jk3LnQsQKBgQDfkV14rRp0nUIdEp0O49bmD+qphKXFAsYn\nWCt37jq3mgZ2IVOfX94+m6bLUAjb+Ipwn9dT+3PrDlZqHRPMKTGvWdm9+41EyBK3\nvjKiNSyqnw/G/gapJn3aN/Nhp8qzOC+xNUTT9xXRf4cEuw9f19zXSm9niDgN5LS3\nE163+ZMNQwKBgB+J7gXaxuBvHKhJExNLY27BUyrV9VBNUkvVegownRDGqVQmM+Qx\nDHkHqSYdHChH8jmERmq6oogYvbuV0c+9Uyt7ezl0BxKwYuH2xYZNsEqsjEkkKSQl\nC8oL5dqm3qwZyRw1ouUo5wZk5cGvot43h4HTDsYJct3imUVE70nwmbwRAoGAQjh4\ni0oaz/fUoW/l/YcXHEYSp+uWfmh38Sd4mKmD0uZYi50Le+WVms3X9djbBuzzdLCj\nw0hz6WfxyLScLJj3Eo12pYNhMMJiaPJ5ZPqDJHbA4ZxUtL2mAYEZIg/lRniaB89T\nd8V0PP2dLJWL1EPIMizmGrCKifL4ZFHkeHIAUKkCgYAhJsPSmUc852/arUFqzhzM\nmtI/bWUUFeULygoARTSN4SVz/RjDj2MJYSV2N9MAskUKFXmmwNMq/NpJ2KkxlHrV\ned1O6cGAg8VYPpQvf6GOmpnHauePQXWXR3xVB++omRH4lD+J4gMT73dOU5o70V5e\nwY7LJ21G5lim4X1G3zRPOQ==\n-----END PRIVATE KEY-----\n\",\"client_email\": \"github-access@gxjavacloudstorageunittests.iam.gserviceaccount.com\",\"client_id\": \"110489372256464678127\",\"auth_uri\": \"https://accounts.google.com/o/oauth2/auth\",\"token_uri\": \"https://oauth2.googleapis.com/token\",\"auth_provider_x509_cert_url\": \"https://www.googleapis.com/oauth2/v1/certs\",\"client_x509_cert_url\": \"https://www.googleapis.com/robot/v1/metadata/x509/github-access%40gxjavacloudstorageunittests.iam.gserviceaccount.com\"}"); + Environment.SetEnvironmentVariable($"STORAGE_{providerName}_PROJECT_ID", "gxjavacloudstorageunittests"); + Environment.SetEnvironmentVariable($"STORAGE_{providerName}_BUCKET_NAME", "javaclasses-unittests"); + Environment.SetEnvironmentVariable($"STORAGE_{providerName}_FOLDER_NAME", "gxclasses"); + Environment.SetEnvironmentVariable($"STORAGE_{providerName}_APPLICATION_NAME", "gxjavacloudstorageunittests"); + } + + bool testEnabled = Environment.GetEnvironmentVariable(providerName + "_TEST_ENABLED") == "true"; + + Assume.True(testEnabled); + + provider = (GeneXus.Services.ExternalProvider)Activator.CreateInstance(externalProviderType); + + Assert.NotNull(provider); + } + + [Fact] + public void TestUploadPublicMethod() + { + String upload = provider.Upload(TEST_SAMPLE_FILE_PATH, TEST_SAMPLE_FILE_NAME, GxFileType.PublicRead); + EnsureUrl(upload, GxFileType.PublicRead); + } + + [Fact] + public void TestUploadDefaultMethod() + { + String upload = provider.Upload(TEST_SAMPLE_FILE_PATH, TEST_SAMPLE_FILE_NAME, GxFileType.Default); + EnsureUrl(upload, GxFileType.Default); + } + + [Fact] + public void TestUploadAndCopyDefault() + { + TestUploadAndCopyByAcl(GxFileType.Default, GxFileType.Default); + } + + [Fact] + public void TestUploadAndCopyPrivate() + { + TestUploadAndCopyByAcl(GxFileType.Private, GxFileType.Private); + } + + [Fact] + public void TestUploadAndCopyPublic() + { + TestUploadAndCopyByAcl(GxFileType.PublicRead, GxFileType.PublicRead); + } + + [Fact] + public void TestUploadAndCopyMixed() + { + TestUploadAndCopyByAcl(GxFileType.Default, GxFileType.Private); + } + + [Fact] + public void TestUploadAndCopyPrivateToPublic() + { + TestUploadAndCopyByAcl(GxFileType.Private, GxFileType.PublicRead); + } + + [Fact] + public void TestUploadAndCopyPublicToPrivate() + { + TestUploadAndCopyByAcl(GxFileType.PublicRead, GxFileType.Private); + } + + public void TestUploadAndCopyByAcl(GxFileType aclUpload, GxFileType aclCopy) + { + String copyFileName = "test-upload-and-copy.txt"; + DeleteSafe(TEST_SAMPLE_FILE_PATH); + DeleteSafe(copyFileName); + String upload = provider.Upload(TEST_SAMPLE_FILE_PATH, TEST_SAMPLE_FILE_NAME, aclUpload); + Assert.True(UrlExists(upload), "Not found URL: " + upload); + + String copyUrl = TryGet(copyFileName, aclCopy); + Assert.False(UrlExists(copyUrl), "URL cannot exist: " + copyUrl); + + provider.Copy(TEST_SAMPLE_FILE_NAME, aclUpload, copyFileName, aclCopy); + upload = provider.Get(copyFileName, aclCopy, 100); + EnsureUrl(upload, aclCopy); + } + + [Fact] + public void TestCopyMethod() + { + String copyFileName = "copy-text.txt"; + Copy(copyFileName, GxFileType.PublicRead); + } + + [Fact] + public void TestCopyPrivateMethod() + { + String copyFileName = "copy-text-private.txt"; + Copy(copyFileName, GxFileType.Private); + } + + private void Copy(String copyFileName, GxFileType acl) + { + provider.Upload(TEST_SAMPLE_FILE_PATH, TEST_SAMPLE_FILE_NAME, acl); + String upload = provider.Get(TEST_SAMPLE_FILE_NAME, acl, 100); + EnsureUrl(upload, acl); + + DeleteSafe(copyFileName); + Wait(1000); //Google CDN replication seems to be delayed. + + String urlCopy = TryGet(copyFileName, GxFileType.PublicRead); + Assert.False(UrlExists(urlCopy), "URL cannot exist: " + urlCopy); + + provider.Copy("text.txt", acl, copyFileName, acl); + upload = provider.Get(copyFileName, acl, 100); + EnsureUrl(upload, acl); + } + + [Fact] + public void TestMultimediaUpload() + { + String copyFileName = "copy-text-private.txt"; + GxFileType acl = GxFileType.Private; + + provider.Upload(TEST_SAMPLE_FILE_PATH, TEST_SAMPLE_FILE_NAME, acl); + String upload = provider.Get(TEST_SAMPLE_FILE_NAME, acl, 100); + EnsureUrl(upload, acl); + + provider.Delete(copyFileName, acl); + provider.Copy("text.txt", acl, copyFileName, acl); + upload = TryGet(copyFileName, acl); + EnsureUrl(upload, acl); + } + + [Fact] + public void TestGetMethod() + { + TestUploadPublicMethod(); + String url = provider.Get("text.txt", GxFileType.PublicRead, 10); + EnsureUrl(url, GxFileType.PublicRead); + } + + [Fact] + public void TestGetObjectName() + { + TestUploadPublicMethod(); + String url = provider.Get(TEST_SAMPLE_FILE_NAME, GxFileType.PublicRead, 10); + Assert.True(UrlExists(url)); + String objectName = ""; + provider.GetObjectNameFromURL(url, out objectName); + Assert.Equal("text.txt", objectName); + } + + [Fact] + public void TestDownloadMethod() + { + String downloadPath = Path.Combine("resources", "test", TEST_SAMPLE_FILE_NAME); + TestUploadPublicMethod(); + + try + { + File.Delete(downloadPath); + } + catch (Exception) { } + + provider.Download(TEST_SAMPLE_FILE_NAME, downloadPath, GxFileType.PublicRead); + Assert.True(File.Exists(downloadPath)); + } + + [Fact] + public void TestDeleteFile() + { + GxFileType acl = GxFileType.PublicRead; + TestUploadPublicMethod(); + String url = TryGet(TEST_SAMPLE_FILE_NAME, acl); + EnsureUrl(url, acl); + provider.Delete(TEST_SAMPLE_FILE_NAME, acl); + + url = TryGet(TEST_SAMPLE_FILE_NAME, acl); + Assert.False(UrlExists(url)); + } + + [Fact] + public void TestDeleteFilePrivate() + { + GxFileType acl = GxFileType.Private; + TestUploadPrivateMethod(); + String url = TryGet(TEST_SAMPLE_FILE_NAME, acl); + EnsureUrl(url, acl); + provider.Delete(TEST_SAMPLE_FILE_NAME, acl); + + url = TryGet(TEST_SAMPLE_FILE_NAME, acl); + Assert.False(UrlExists(url)); + } + + [Fact] + public void TestUploadPrivateMethod() + { + GxFileType acl = GxFileType.Private; + String externalFileName = "text-private-2.txt"; + + DeleteSafe(externalFileName); + String signedUrl = provider.Upload(TEST_SAMPLE_FILE_PATH, externalFileName, acl); + EnsureUrl(signedUrl, acl); + signedUrl = provider.Get(externalFileName, acl, 10); + EnsureUrl(signedUrl, acl); + + } + + + private String TryGet(String objectName, GxFileType acl) + { + String getValue = ""; + try + { + getValue = provider.Get(objectName, acl, 5); + } + catch (Exception) + { + + } + return getValue; + } + + private String GetSafe(String objectName, GxFileType acl) + { + try + { + return provider.Get(objectName, acl, 100); + } + catch (Exception) + { + + } + return String.Empty; + } + + private bool DeleteSafe(String objectName) + { + DeleteSafeImpl(objectName, GxFileType.Private); + DeleteSafeImpl(objectName, GxFileType.PublicRead); + return true; + } + + private bool DeleteSafeImpl(String objectName, GxFileType acl) + { + try + { + provider.Delete(objectName, acl); + } + catch (Exception) + { + + } + return true; + } + + private static void Wait(int milliseconds) + { + System.Threading.Thread.Sleep(milliseconds); + } + + private static void EnsureUrl(String signedOrUnsignedUrl, GxFileType acl) + { + Assert.True(UrlExists(signedOrUnsignedUrl), "Resource not found: " + signedOrUnsignedUrl); + if (acl == GxFileType.Private) + { + String noSignedUrl = signedOrUnsignedUrl.Substring(0, signedOrUnsignedUrl.IndexOf('?') + 1); + Assert.False(UrlExists(noSignedUrl), "Resource must be private: " + noSignedUrl); + } + } + +#pragma warning disable CA1054 // Uri parameters should not be strings + protected static bool UrlExists(string url) + { + if (string.IsNullOrEmpty(url)) + { + return false; + } + bool exists = false; + HttpWebRequest request = (HttpWebRequest)WebRequest.Create(new Uri(url)); + request.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate; + + try + { + using (HttpWebResponse response = (HttpWebResponse)request.GetResponse()) + { + exists = response.StatusCode == HttpStatusCode.OK; + } + } + catch (WebException) + { + + } + return exists; + } +#pragma warning restore CA1054 // Uri parameters should not be strings + } +} +#pragma warning restore CA1031 // Do not catch general exception types \ No newline at end of file diff --git a/dotnet/test/DotNetUnitTest/Properties/launchSettings.json b/dotnet/test/DotNetUnitTest/Properties/launchSettings.json new file mode 100644 index 000000000..6d2484989 --- /dev/null +++ b/dotnet/test/DotNetUnitTest/Properties/launchSettings.json @@ -0,0 +1,11 @@ +{ + "profiles": { + "DotNetUnitTest": { + "commandName": "Project", + "environmentVariables": { + "test": "test" + }, + "remoteDebugEnabled": false + } + } +} \ No newline at end of file diff --git a/dotnet/test/DotNetUnitTest/resources/text.txt b/dotnet/test/DotNetUnitTest/resources/text.txt new file mode 100644 index 000000000..e04a19003 --- /dev/null +++ b/dotnet/test/DotNetUnitTest/resources/text.txt @@ -0,0 +1 @@ +My uploaded file test \ No newline at end of file From 48fe32c4abc8f373117c9f9ce6812ded495dac3e Mon Sep 17 00:00:00 2001 From: Gonzalo Date: Thu, 18 Jun 2020 15:14:32 -0300 Subject: [PATCH 05/42] Support for Path Style urls --- .../Storage/GXAmazonS3/ExternalProviderS3.cs | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs b/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs index 54accf387..64c66fdd1 100644 --- a/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs +++ b/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs @@ -27,7 +27,7 @@ public class ExternalProviderS3 : ExternalProviderBase, ExternalProvider const string STORAGE_CUSTOM_ENDPOINT_VALUE = "custom"; const string DEFAULT_REGION = "us-east-1"; - + [Obsolete("Use Property ACCESS_KEY instead", false)] const string ACCESS_KEY_ID_DEPRECATED = "STORAGE_PROVIDER_ACCESSKEYID"; [Obsolete("Use Property SECRET_ACCESS_KEY instead", false)] @@ -57,7 +57,7 @@ public class ExternalProviderS3 : ExternalProviderBase, ExternalProvider public string StorageUri { get { - return (ForcePathStyle)? $"{Endpoint}/": $"https://{Bucket}.{Endpoint}/"; + return _storageUri; } } @@ -133,7 +133,7 @@ private void SetURI() { if (Region == DEFAULT_REGION) { - _storageUri = $"https://{Bucket}.{Endpoint}/"; + _storageUri = (ForcePathStyle) ? $"{Endpoint}/" : $"https://{Bucket}.{Endpoint}/"; } else { @@ -240,16 +240,20 @@ private bool IsPrivateUpload(GxFileType fileType) public string Get(string objectName, GxFileType fileType, int urlMinutes = DefaultExpirationMinutes) { - bool isPrivate = IsPrivateUpload(fileType); if (Exists(objectName, fileType)) - if (isPrivate) - return GetPreSignedUrl(objectName, urlMinutes); - else - return StorageUri + StorageUtils.EncodeUrl(objectName); + { + return GetUrlImpl(objectName, fileType, urlMinutes); + } else return string.Empty; } + private string GetUrlImpl(string objectName, GxFileType fileType, int urlMinutes = DefaultExpirationMinutes) + { + bool isPrivate = IsPrivateUpload(fileType); + return (isPrivate)? GetPreSignedUrl(objectName, urlMinutes): StorageUri + StorageUtils.EncodeUrl(objectName); + } + private string GetPreSignedUrl(string objectName, int urlMinutes) { GetPreSignedUrlRequest request = new GetPreSignedUrlRequest From cb9d170e76347aa45cf66f809aaf064b1994fa61 Mon Sep 17 00:00:00 2001 From: Gonzalo Date: Thu, 10 Jun 2021 19:29:16 -0300 Subject: [PATCH 06/42] Add Minio Unit Tests --- .../Storage/GXAmazonS3/ExternalProviderS3.cs | 32 ++++++++++++------- .../GXGoogleCloud/ExternalProviderGoogle.cs | 2 ++ .../ExternalProviderMinioTest.cs | 13 ++++++++ .../ExternalProvider/ExternalProviderTest.cs | 18 +++++++++-- 4 files changed, 51 insertions(+), 14 deletions(-) create mode 100644 dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderMinioTest.cs diff --git a/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs b/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs index 64c66fdd1..28ca58f00 100644 --- a/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs +++ b/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs @@ -52,8 +52,9 @@ public class ExternalProviderS3 : ExternalProviderBase, ExternalProvider string Endpoint { get; set; } string Region { get; set; } - bool ForcePathStyle = false; - + bool forcePathStyle = false; + bool customEndpoint = false; + public string StorageUri { get { @@ -84,7 +85,7 @@ private void Initialize() { credentials = new BasicAWSCredentials(keyId, keySecret); } - var region = Amazon.RegionEndpoint.GetBySystemName(GetPropertyValue(REGION, REGION_DEPRECATED)); + var region = Amazon.RegionEndpoint.GetBySystemName(GetPropertyValue(REGION, REGION_DEPRECATED, DEFAULT_REGION)); AmazonS3Config config = new AmazonS3Config() { @@ -95,11 +96,12 @@ private void Initialize() { if (Endpoint == STORAGE_CUSTOM_ENDPOINT_VALUE) { Endpoint = GetPropertyValue(STORAGE_CUSTOM_ENDPOINT, STORAGE_CUSTOM_ENDPOINT_DEPRECATED); - ForcePathStyle = true; - config.ForcePathStyle = ForcePathStyle; + forcePathStyle = true; + config.ForcePathStyle = forcePathStyle; config.ServiceURL = Endpoint; + customEndpoint = true; } - + #if NETCORE if (credentials != null) { @@ -131,13 +133,20 @@ private void Initialize() { private void SetURI() { - if (Region == DEFAULT_REGION) + if (customEndpoint) { - _storageUri = (ForcePathStyle) ? $"{Endpoint}/" : $"https://{Bucket}.{Endpoint}/"; + _storageUri = !Endpoint.EndsWith("/") ? $"{Endpoint}/{Bucket}/": $"{Endpoint}{Bucket}/"; } else { - _storageUri = $"https://{Bucket}.{Endpoint.Replace("s3.amazonaws.com", $"s3.{Region.ToLower()}.amazonaws.com")}/"; + if (Region == DEFAULT_REGION) + { + _storageUri = (forcePathStyle) ? $"{Endpoint}/" : $"https://{Bucket}.{Endpoint}/"; + } + else + { + _storageUri = $"https://{Bucket}.{Endpoint.Replace("s3.amazonaws.com", $"s3.{Region.ToLower()}.amazonaws.com")}/"; + } } } @@ -229,8 +238,8 @@ public string Upload(string localFile, string objectName, GxFileType fileType) FilePath = localFile, CannedACL = GetCannedACL(fileType) }; - PutObjectResponse result = PutObject(objectRequest); - return Get(objectName, fileType); + PutObject(objectRequest); + return GetUrlImpl(objectName, fileType); } private bool IsPrivateUpload(GxFileType fileType) @@ -252,6 +261,7 @@ private string GetUrlImpl(string objectName, GxFileType fileType, int urlMinutes { bool isPrivate = IsPrivateUpload(fileType); return (isPrivate)? GetPreSignedUrl(objectName, urlMinutes): StorageUri + StorageUtils.EncodeUrl(objectName); + } private string GetPreSignedUrl(string objectName, int urlMinutes) diff --git a/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs b/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs index c53ab783f..040ca64b8 100644 --- a/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs +++ b/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs @@ -32,6 +32,8 @@ public class ExternalProviderGoogle : ExternalProviderBase, ExternalProvider String Bucket { get; set; } String Folder { get; set; } UrlSigner Signer { get; set; } + + public string StorageUri { get { return $"https://{Bucket}.storage.googleapis.com/"; } diff --git a/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderMinioTest.cs b/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderMinioTest.cs new file mode 100644 index 000000000..2695bd232 --- /dev/null +++ b/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderMinioTest.cs @@ -0,0 +1,13 @@ +using GeneXus.Storage.GXAmazonS3; +using UnitTesting; + +namespace DotNetUnitTest +{ + public class ExternalProviderMinioTest : ExternalProviderTest + { + public ExternalProviderMinioTest(): base(ExternalProviderS3.Name, typeof(ExternalProviderS3)) + { + } + + } +} diff --git a/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs b/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs index 2469d0e70..5c6f0fcee 100644 --- a/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs +++ b/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs @@ -2,6 +2,7 @@ using System.Globalization; using System.IO; using System.Net; +using DotNetUnitTest; using GeneXus.Storage.GXAmazonS3; using GeneXus.Storage.GXAzureStorage; using GeneXus.Storage.GXGoogleCloud; @@ -19,7 +20,7 @@ public abstract class ExternalProviderTest public ExternalProviderTest(string providerName, Type externalProviderType) { - if (externalProviderType == typeof(ExternalProviderS3)) + if (this.GetType() == typeof(ExternalProviderS3Test)) { Environment.SetEnvironmentVariable(providerName + "_TEST_ENABLED", "true"); Environment.SetEnvironmentVariable("STORAGE_AWSS3_ACCESS_KEY", "AKIAJMQ6SF3Y4IULKD5A"); @@ -29,7 +30,18 @@ public ExternalProviderTest(string providerName, Type externalProviderType) Environment.SetEnvironmentVariable("STORAGE_AWSS3_REGION", "us-east-1"); Environment.SetEnvironmentVariable("STORAGE_AWSS3_ENDPOINT", "s3.amazonaws.com"); } - if (externalProviderType == typeof(AzureStorageExternalProvider)) + + if (this.GetType() == typeof(ExternalProviderMinioTest)) + { + Environment.SetEnvironmentVariable(providerName + "_TEST_ENABLED", "true"); + Environment.SetEnvironmentVariable("STORAGE_AWSS3_ACCESS_KEY", "desaint"); + Environment.SetEnvironmentVariable("STORAGE_AWSS3_SECRET_KEY", "6YafTT3U2YtHS7am"); + Environment.SetEnvironmentVariable("STORAGE_AWSS3_BUCKET_NAME", "java-classes-unittests"); + Environment.SetEnvironmentVariable("STORAGE_AWSS3_FOLDER_NAME", "test-minio"); + Environment.SetEnvironmentVariable("STORAGE_AWSS3_ENDPOINT", "custom"); + Environment.SetEnvironmentVariable("STORAGE_AWSS3_CUSTOM_ENDPOINT", "http://192.168.254.78:9000"); + } + if (this.GetType() == typeof(ExternalProviderAzureTest)) { Environment.SetEnvironmentVariable(providerName + "_TEST_ENABLED", "true"); Environment.SetEnvironmentVariable($"STORAGE_{providerName}_ACCESS_KEY", "DNiutn2Evl+MNs0TsGUqgg0IDzYWMoMAhLM7Oju/PLi4BEsIsrSY917M6li3Ml7sxj8W+KFD/LZU49OGI40Slg=="); @@ -39,7 +51,7 @@ public ExternalProviderTest(string providerName, Type externalProviderType) Environment.SetEnvironmentVariable($"STORAGE_{providerName}_PRIVATE_CONTAINER_NAME", "contluisprivate"); } - if (externalProviderType == typeof(ExternalProviderGoogle)) + if (this.GetType() == typeof(ExternalProviderGoogleTest)) { Environment.SetEnvironmentVariable(providerName + "_TEST_ENABLED", "true"); Environment.SetEnvironmentVariable($"STORAGE_{providerName}_KEY", "{\"type\": \"service_account\",\"project_id\": \"gxjavacloudstorageunittests\",\"private_key_id\": \"37e8c0566cd56c7bb07542fdddd9f46ac64a5ba0\",\"private_key\": \"-----BEGIN PRIVATE KEY-----\nMIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDQFJ0NHKkmZNCP\n824SBcXHa0DeoDy1i7zToD1DYiJzQXZAZg9tm91SKEbbCZWMgslBdUP10c7UsnuU\nkvnZTUpZ2YQ680ySiykjAWildp28e/nLBDVZy0f8N7H94484kOS4fxQs2VoW8MH6\nVzsxxm+3yhHjQfQOD+Obes3YBMXUxb7rs/blp7fYWbXNzyGGcpM0kEm4mRNslTbi\nZmVnuwpkaaYfQEPPYLfxWcwfgfpbXEPIJB+bNw+EN5JOUvYvCdpwdPgNyA6MvQFj\nGLTAbsXCOtFh6TtRjE9LkjTZrLTAT0HJ0kODxCyM4pBgag87iZAJ4nsq/355DIMJ\nJtEy3ZtTAgMBAAECggEAYmS89wxMeBlH/inwLJmKMohm/l7rFjXjrnahQZHQFIwp\n7L3WIdCIUWc2SjE4BF9753YaEs2Jbk6P3Wu6taS0udP/kRinZsxjQWhTIZr7b7t4\nHSX6TGGxwnRbuGC4wtjRLuT4l1SYIyzprQU+uoTJIzFsT/hJ/bRJvqXNXI61Na0J\n/ahzB640y75xe8H5Yw06yWXisqD+eiAxX8TU2SRdcZgcIVWaWLDiKhERk7sBctol\nWYgxm3qzpkA+dcvIoykrhLMZGaX9yDu7V9ueXeksqehZrQjSVx0CYSRi5ONy9bcJ\nnQIO4B0lo5oCh6EzKsYlvzJlAPCLJbs4K9Fxviu7QQKBgQDuRBiqAG9J7CR06TqW\nohdCGG6byZZ6x9wmWz3QVLsijFpiQTfu6XHDgh2A4NDau1hOzoJ21rGN5H3XNC9/\ni+8k4Je1nM14aiomlsgNFEp+gUhR7uR54da31q7Vf1DqcQ8ykqOCZitb6/AF0zhU\nlm8syJF1QAi3H7LMN0Jk3LnQsQKBgQDfkV14rRp0nUIdEp0O49bmD+qphKXFAsYn\nWCt37jq3mgZ2IVOfX94+m6bLUAjb+Ipwn9dT+3PrDlZqHRPMKTGvWdm9+41EyBK3\nvjKiNSyqnw/G/gapJn3aN/Nhp8qzOC+xNUTT9xXRf4cEuw9f19zXSm9niDgN5LS3\nE163+ZMNQwKBgB+J7gXaxuBvHKhJExNLY27BUyrV9VBNUkvVegownRDGqVQmM+Qx\nDHkHqSYdHChH8jmERmq6oogYvbuV0c+9Uyt7ezl0BxKwYuH2xYZNsEqsjEkkKSQl\nC8oL5dqm3qwZyRw1ouUo5wZk5cGvot43h4HTDsYJct3imUVE70nwmbwRAoGAQjh4\ni0oaz/fUoW/l/YcXHEYSp+uWfmh38Sd4mKmD0uZYi50Le+WVms3X9djbBuzzdLCj\nw0hz6WfxyLScLJj3Eo12pYNhMMJiaPJ5ZPqDJHbA4ZxUtL2mAYEZIg/lRniaB89T\nd8V0PP2dLJWL1EPIMizmGrCKifL4ZFHkeHIAUKkCgYAhJsPSmUc852/arUFqzhzM\nmtI/bWUUFeULygoARTSN4SVz/RjDj2MJYSV2N9MAskUKFXmmwNMq/NpJ2KkxlHrV\ned1O6cGAg8VYPpQvf6GOmpnHauePQXWXR3xVB++omRH4lD+J4gMT73dOU5o70V5e\nwY7LJ21G5lim4X1G3zRPOQ==\n-----END PRIVATE KEY-----\n\",\"client_email\": \"github-access@gxjavacloudstorageunittests.iam.gserviceaccount.com\",\"client_id\": \"110489372256464678127\",\"auth_uri\": \"https://accounts.google.com/o/oauth2/auth\",\"token_uri\": \"https://oauth2.googleapis.com/token\",\"auth_provider_x509_cert_url\": \"https://www.googleapis.com/oauth2/v1/certs\",\"client_x509_cert_url\": \"https://www.googleapis.com/robot/v1/metadata/x509/github-access%40gxjavacloudstorageunittests.iam.gserviceaccount.com\"}"); From c7f0550b12de7623db882c6421f9944141c25fbf Mon Sep 17 00:00:00 2001 From: Gonzalo Date: Thu, 10 Jun 2021 19:32:26 -0300 Subject: [PATCH 07/42] Remove launchSettings.json --- .../DotNetUnitTest/Properties/launchSettings.json | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 dotnet/test/DotNetUnitTest/Properties/launchSettings.json diff --git a/dotnet/test/DotNetUnitTest/Properties/launchSettings.json b/dotnet/test/DotNetUnitTest/Properties/launchSettings.json deleted file mode 100644 index 6d2484989..000000000 --- a/dotnet/test/DotNetUnitTest/Properties/launchSettings.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "profiles": { - "DotNetUnitTest": { - "commandName": "Project", - "environmentVariables": { - "test": "test" - }, - "remoteDebugEnabled": false - } - } -} \ No newline at end of file From d759a2a412bf9fed5574e4550fd89e9e0e061647 Mon Sep 17 00:00:00 2001 From: Gonzalo Date: Thu, 10 Jun 2021 21:45:21 -0300 Subject: [PATCH 08/42] Remove Secrets --- .../ExternalProvider/ExternalProviderTest.cs | 41 +------------------ 1 file changed, 1 insertion(+), 40 deletions(-) diff --git a/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs b/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs index 5c6f0fcee..b7dae0295 100644 --- a/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs +++ b/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs @@ -20,46 +20,7 @@ public abstract class ExternalProviderTest public ExternalProviderTest(string providerName, Type externalProviderType) { - if (this.GetType() == typeof(ExternalProviderS3Test)) - { - Environment.SetEnvironmentVariable(providerName + "_TEST_ENABLED", "true"); - Environment.SetEnvironmentVariable("STORAGE_AWSS3_ACCESS_KEY", "AKIAJMQ6SF3Y4IULKD5A"); - Environment.SetEnvironmentVariable("STORAGE_AWSS3_SECRET_KEY", "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"); - Environment.SetEnvironmentVariable("STORAGE_AWSS3_BUCKET_NAME", "genexuss3test"); - Environment.SetEnvironmentVariable("STORAGE_AWSS3_FOLDER_NAME", "gxclasses"); - Environment.SetEnvironmentVariable("STORAGE_AWSS3_REGION", "us-east-1"); - Environment.SetEnvironmentVariable("STORAGE_AWSS3_ENDPOINT", "s3.amazonaws.com"); - } - - if (this.GetType() == typeof(ExternalProviderMinioTest)) - { - Environment.SetEnvironmentVariable(providerName + "_TEST_ENABLED", "true"); - Environment.SetEnvironmentVariable("STORAGE_AWSS3_ACCESS_KEY", "desaint"); - Environment.SetEnvironmentVariable("STORAGE_AWSS3_SECRET_KEY", "6YafTT3U2YtHS7am"); - Environment.SetEnvironmentVariable("STORAGE_AWSS3_BUCKET_NAME", "java-classes-unittests"); - Environment.SetEnvironmentVariable("STORAGE_AWSS3_FOLDER_NAME", "test-minio"); - Environment.SetEnvironmentVariable("STORAGE_AWSS3_ENDPOINT", "custom"); - Environment.SetEnvironmentVariable("STORAGE_AWSS3_CUSTOM_ENDPOINT", "http://192.168.254.78:9000"); - } - if (this.GetType() == typeof(ExternalProviderAzureTest)) - { - Environment.SetEnvironmentVariable(providerName + "_TEST_ENABLED", "true"); - Environment.SetEnvironmentVariable($"STORAGE_{providerName}_ACCESS_KEY", "DNiutn2Evl+MNs0TsGUqgg0IDzYWMoMAhLM7Oju/PLi4BEsIsrSY917M6li3Ml7sxj8W+KFD/LZU49OGI40Slg=="); - Environment.SetEnvironmentVariable($"STORAGE_{providerName}_ACCOUNT_NAME", "luistest1"); - Environment.SetEnvironmentVariable($"STORAGE_{providerName}_FOLDER_NAME", "luistest1"); - Environment.SetEnvironmentVariable($"STORAGE_{providerName}_PUBLIC_CONTAINER_NAME", "contluispublic"); - Environment.SetEnvironmentVariable($"STORAGE_{providerName}_PRIVATE_CONTAINER_NAME", "contluisprivate"); - } - - if (this.GetType() == typeof(ExternalProviderGoogleTest)) - { - Environment.SetEnvironmentVariable(providerName + "_TEST_ENABLED", "true"); - Environment.SetEnvironmentVariable($"STORAGE_{providerName}_KEY", "{\"type\": \"service_account\",\"project_id\": \"gxjavacloudstorageunittests\",\"private_key_id\": \"37e8c0566cd56c7bb07542fdddd9f46ac64a5ba0\",\"private_key\": \"-----BEGIN PRIVATE KEY-----\nMIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDQFJ0NHKkmZNCP\n824SBcXHa0DeoDy1i7zToD1DYiJzQXZAZg9tm91SKEbbCZWMgslBdUP10c7UsnuU\nkvnZTUpZ2YQ680ySiykjAWildp28e/nLBDVZy0f8N7H94484kOS4fxQs2VoW8MH6\nVzsxxm+3yhHjQfQOD+Obes3YBMXUxb7rs/blp7fYWbXNzyGGcpM0kEm4mRNslTbi\nZmVnuwpkaaYfQEPPYLfxWcwfgfpbXEPIJB+bNw+EN5JOUvYvCdpwdPgNyA6MvQFj\nGLTAbsXCOtFh6TtRjE9LkjTZrLTAT0HJ0kODxCyM4pBgag87iZAJ4nsq/355DIMJ\nJtEy3ZtTAgMBAAECggEAYmS89wxMeBlH/inwLJmKMohm/l7rFjXjrnahQZHQFIwp\n7L3WIdCIUWc2SjE4BF9753YaEs2Jbk6P3Wu6taS0udP/kRinZsxjQWhTIZr7b7t4\nHSX6TGGxwnRbuGC4wtjRLuT4l1SYIyzprQU+uoTJIzFsT/hJ/bRJvqXNXI61Na0J\n/ahzB640y75xe8H5Yw06yWXisqD+eiAxX8TU2SRdcZgcIVWaWLDiKhERk7sBctol\nWYgxm3qzpkA+dcvIoykrhLMZGaX9yDu7V9ueXeksqehZrQjSVx0CYSRi5ONy9bcJ\nnQIO4B0lo5oCh6EzKsYlvzJlAPCLJbs4K9Fxviu7QQKBgQDuRBiqAG9J7CR06TqW\nohdCGG6byZZ6x9wmWz3QVLsijFpiQTfu6XHDgh2A4NDau1hOzoJ21rGN5H3XNC9/\ni+8k4Je1nM14aiomlsgNFEp+gUhR7uR54da31q7Vf1DqcQ8ykqOCZitb6/AF0zhU\nlm8syJF1QAi3H7LMN0Jk3LnQsQKBgQDfkV14rRp0nUIdEp0O49bmD+qphKXFAsYn\nWCt37jq3mgZ2IVOfX94+m6bLUAjb+Ipwn9dT+3PrDlZqHRPMKTGvWdm9+41EyBK3\nvjKiNSyqnw/G/gapJn3aN/Nhp8qzOC+xNUTT9xXRf4cEuw9f19zXSm9niDgN5LS3\nE163+ZMNQwKBgB+J7gXaxuBvHKhJExNLY27BUyrV9VBNUkvVegownRDGqVQmM+Qx\nDHkHqSYdHChH8jmERmq6oogYvbuV0c+9Uyt7ezl0BxKwYuH2xYZNsEqsjEkkKSQl\nC8oL5dqm3qwZyRw1ouUo5wZk5cGvot43h4HTDsYJct3imUVE70nwmbwRAoGAQjh4\ni0oaz/fUoW/l/YcXHEYSp+uWfmh38Sd4mKmD0uZYi50Le+WVms3X9djbBuzzdLCj\nw0hz6WfxyLScLJj3Eo12pYNhMMJiaPJ5ZPqDJHbA4ZxUtL2mAYEZIg/lRniaB89T\nd8V0PP2dLJWL1EPIMizmGrCKifL4ZFHkeHIAUKkCgYAhJsPSmUc852/arUFqzhzM\nmtI/bWUUFeULygoARTSN4SVz/RjDj2MJYSV2N9MAskUKFXmmwNMq/NpJ2KkxlHrV\ned1O6cGAg8VYPpQvf6GOmpnHauePQXWXR3xVB++omRH4lD+J4gMT73dOU5o70V5e\nwY7LJ21G5lim4X1G3zRPOQ==\n-----END PRIVATE KEY-----\n\",\"client_email\": \"github-access@gxjavacloudstorageunittests.iam.gserviceaccount.com\",\"client_id\": \"110489372256464678127\",\"auth_uri\": \"https://accounts.google.com/o/oauth2/auth\",\"token_uri\": \"https://oauth2.googleapis.com/token\",\"auth_provider_x509_cert_url\": \"https://www.googleapis.com/oauth2/v1/certs\",\"client_x509_cert_url\": \"https://www.googleapis.com/robot/v1/metadata/x509/github-access%40gxjavacloudstorageunittests.iam.gserviceaccount.com\"}"); - Environment.SetEnvironmentVariable($"STORAGE_{providerName}_PROJECT_ID", "gxjavacloudstorageunittests"); - Environment.SetEnvironmentVariable($"STORAGE_{providerName}_BUCKET_NAME", "javaclasses-unittests"); - Environment.SetEnvironmentVariable($"STORAGE_{providerName}_FOLDER_NAME", "gxclasses"); - Environment.SetEnvironmentVariable($"STORAGE_{providerName}_APPLICATION_NAME", "gxjavacloudstorageunittests"); - } + bool testEnabled = Environment.GetEnvironmentVariable(providerName + "_TEST_ENABLED") == "true"; From 32542a28643ec7ff6f51126397077648d6022990 Mon Sep 17 00:00:00 2001 From: Gonzalo Date: Fri, 11 Jun 2021 14:42:48 -0300 Subject: [PATCH 09/42] Every file is now private --- .../dotnetframework/GxClasses/Core/GXUtils.cs | 15 ++++---- .../GxClasses/Data/GXDataNTierADO.cs | 8 ++--- .../GxClasses/Domain/GXFileIO.cs | 15 ++++---- .../Services/Storage/ExternalProviderBase.cs | 18 ++++++++-- .../Storage/ExternalProviderCommon.cs | 36 +++++++++++++++++++ .../test/DotNetUnitTest/DotNetUnitTest.csproj | 3 ++ 6 files changed, 74 insertions(+), 21 deletions(-) create mode 100644 dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderCommon.cs diff --git a/dotnet/src/dotnetframework/GxClasses/Core/GXUtils.cs b/dotnet/src/dotnetframework/GxClasses/Core/GXUtils.cs index e4354f935..0cb3458a0 100644 --- a/dotnet/src/dotnetframework/GxClasses/Core/GXUtils.cs +++ b/dotnet/src/dotnetframework/GxClasses/Core/GXUtils.cs @@ -1139,8 +1139,9 @@ public bool UploadPrivate(string filefullpath, string storageobjectfullname, GxF { storageobjectfullname=Path.GetFileName(filefullpath); } - string url = provider.Upload(filefullpath, storageobjectfullname, GxFileType.Private); - uploadedFile.FileInfo = new GxExternalFileInfo(storageobjectfullname, url, provider); + GxFileType acl = GxFileType.Private; + string url = provider.Upload(filefullpath, storageobjectfullname, acl); + uploadedFile.FileInfo = new GxExternalFileInfo(storageobjectfullname, url, provider, acl); return true; } catch (Exception ex) @@ -1197,7 +1198,8 @@ public bool Get(string storageobjectfullname, GxFile externalFile, GXBaseCollect try { ValidProvider(); - string url = provider.Get(storageobjectfullname, GxFileType.PublicRead, 0); + GxFileType acl = GxFileType.PublicRead; + string url = provider.Get(storageobjectfullname, acl, 0); if (String.IsNullOrEmpty(url)) { GXUtil.ErrorToMessages("Get Error", "File doesn't exists", messages); @@ -1205,7 +1207,7 @@ public bool Get(string storageobjectfullname, GxFile externalFile, GXBaseCollect } else { - externalFile.FileInfo = new GxExternalFileInfo(storageobjectfullname, url, provider); + externalFile.FileInfo = new GxExternalFileInfo(storageobjectfullname, url, provider, acl); return true; } } @@ -1222,7 +1224,8 @@ public bool GetPrivate(string storageobjectfullname, GxFile externalFile, int ex try { ValidProvider(); - string url = provider.Get(storageobjectfullname, GxFileType.Private, expirationMinutes); + GxFileType acl = GxFileType.Private; + string url = provider.Get(storageobjectfullname, acl, expirationMinutes); if (String.IsNullOrEmpty(url)) { GXUtil.ErrorToMessages("Get Error", "File doesn't exists", messages); @@ -1230,7 +1233,7 @@ public bool GetPrivate(string storageobjectfullname, GxFile externalFile, int ex } else { - externalFile.FileInfo = new GxExternalFileInfo(storageobjectfullname, url, provider, GxFileType.Private); + externalFile.FileInfo = new GxExternalFileInfo(storageobjectfullname, url, provider, acl); return true; } } diff --git a/dotnet/src/dotnetframework/GxClasses/Data/GXDataNTierADO.cs b/dotnet/src/dotnetframework/GxClasses/Data/GXDataNTierADO.cs index 10befd758..4ea624d2b 100644 --- a/dotnet/src/dotnetframework/GxClasses/Data/GXDataNTierADO.cs +++ b/dotnet/src/dotnetframework/GxClasses/Data/GXDataNTierADO.cs @@ -617,7 +617,7 @@ public void SetParameterMultimedia(int id, string image_gxi, string image, strin { try { - multimediaUri = ServiceFactory.GetExternalProvider().Copy(image_gxi, GXDbFile.GenerateUri(image_gxi, !GXDbFile.HasToken(image_gxi), false), tableName, fieldName, GxFileType.PublicRead); + multimediaUri = ServiceFactory.GetExternalProvider().Copy(image_gxi, GXDbFile.GenerateUri(image_gxi, !GXDbFile.HasToken(image_gxi), false), tableName, fieldName, GxFileType.Default); GXLogging.Debug(log, "Copy file already in ExternalProvider:", multimediaUri); } catch (Exception ex) @@ -635,7 +635,7 @@ public void SetParameterMultimedia(int id, string image_gxi, string image, strin using (var fileStream = new MemoryStream(new WebClient().DownloadData(image_gxi))) { //Cannot pass Http Stream directly, because some Providers (AWS S3) does not support Http Stream. - multimediaUri = ServiceFactory.GetExternalProvider().Save(fileStream, GXDbFile.GenerateUri(image_gxi, !GXDbFile.HasToken(image_gxi), false), tableName, fieldName, GxFileType.PublicRead); + multimediaUri = ServiceFactory.GetExternalProvider().Save(fileStream, GXDbFile.GenerateUri(image_gxi, !GXDbFile.HasToken(image_gxi), false), tableName, fieldName, GxFileType.Default); GXLogging.Debug(log, "Upload external file to ExternalProvider:", multimediaUri); } #pragma warning disable SYSLIB0014 // WebClient @@ -657,7 +657,7 @@ public void SetParameterMultimedia(int id, string image_gxi, string image, strin String fileName = PathUtil.GetValidFileName(fileFullName, "_"); using (fileStream) { - multimediaUri = ServiceFactory.GetExternalProvider().Save(fileStream, GXDbFile.GenerateUri(fileName, !GXDbFile.HasToken(fileName), false), tableName, fieldName, GxFileType.PublicRead); + multimediaUri = ServiceFactory.GetExternalProvider().Save(fileStream, GXDbFile.GenerateUri(fileName, !GXDbFile.HasToken(fileName), false), tableName, fieldName, GxFileType.Default); GXLogging.Debug(log, "Upload file (_gxi) to ExternalProvider:", multimediaUri); } } @@ -665,7 +665,7 @@ public void SetParameterMultimedia(int id, string image_gxi, string image, strin { try { - multimediaUri = ServiceFactory.GetExternalProvider().Copy(image, GXDbFile.GenerateUri(image_gxi, !GXDbFile.HasToken(image_gxi), false), tableName, fieldName, GxFileType.PublicRead); + multimediaUri = ServiceFactory.GetExternalProvider().Copy(image, GXDbFile.GenerateUri(image_gxi, !GXDbFile.HasToken(image_gxi), false), tableName, fieldName, GxFileType.Default); GXLogging.Debug(log, "Copy external file in ExternalProvider:", multimediaUri); } catch(Exception e) diff --git a/dotnet/src/dotnetframework/GxClasses/Domain/GXFileIO.cs b/dotnet/src/dotnetframework/GxClasses/Domain/GXFileIO.cs index 963328abf..34cee20b8 100644 --- a/dotnet/src/dotnetframework/GxClasses/Domain/GXFileIO.cs +++ b/dotnet/src/dotnetframework/GxClasses/Domain/GXFileIO.cs @@ -17,6 +17,7 @@ using GeneXus; using GeneXus.Storage; using GeneXus.Attributes; +using GxClasses.Services.Storage; public interface IGxDirectoryInfo { @@ -412,7 +413,7 @@ public class GxExternalFileInfo : IGxFileInfo private string _name; private ExternalProvider _provider; private string _url; - private GxFileType _fileTypeAtt = GxFileType.PublicRead; + private GxFileType _fileTypeAtt = GxFileType.Private; public GxExternalFileInfo(ExternalProvider provider) { @@ -432,20 +433,16 @@ public GxExternalFileInfo(string storageObjectFullname, ExternalProvider provide _url = storageObjectFullname; } else { - if (fileType.HasFlag(GxFileType.Private)) //Attributes multimedia consider Storage Provider Folder + /*if (fileType.HasFlag(GxFileType.Private)) //Attributes multimedia consider Storage Provider Folder { _url = provider.GetBaseURL() + storageObjectFullname; - _name = _url.Replace(provider.StorageUri, string.Empty); - if (_name.StartsWith("/")) - { - _name = _name.Substring(1, _name.Length - 1); - } - } + _name = ExternalProviderCommon.getProviderObjectName(provider, storageObjectFullname); + }*/ } _fileTypeAtt = fileType; } - public GxExternalFileInfo(string storageObjectFullname, string url, ExternalProvider provider, GxFileType fileType = GxFileType.PublicRead) + public GxExternalFileInfo(string storageObjectFullname, string url, ExternalProvider provider, GxFileType fileType = GxFileType.Private) { _name = storageObjectFullname; _provider = provider; diff --git a/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderBase.cs b/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderBase.cs index a495f88a3..c80108011 100644 --- a/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderBase.cs +++ b/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderBase.cs @@ -27,6 +27,16 @@ public ExternalProviderBase() public ExternalProviderBase(GXService s) { + if (s == null) { + try + { + s = ServiceFactory.GetGXServices().Get(GXServices.STORAGE_SERVICE); + } + catch (Exception) { + logger.Warn("STORAGE_SERVICE is not activated in CloudServices.config"); + } + } + this.service = s; Initialize(); } @@ -39,6 +49,10 @@ private void Initialize() if (!String.IsNullOrEmpty(aclS)) { GxFileType.TryParse(aclS, out this.defaultAcl); + } + if (this.defaultAcl == GxFileType.Default) + { + this.defaultAcl = GxFileType.PublicRead; } } @@ -85,7 +99,7 @@ protected String GetPropertyValue(String propertyName, String alternativePropert protected String GetPropertyValue(String propertyName, String alternativePropertyName, String defaultValue) { - String value = String.Empty; + String value = null; value = string.IsNullOrEmpty(value) ? GetPropertyValueImpl(ResolvePropertyName(propertyName)) : value; value = string.IsNullOrEmpty(value) ? GetPropertyValueImpl(propertyName) : value; value = string.IsNullOrEmpty(value) ? GetPropertyValueImpl(alternativePropertyName) : value; @@ -95,7 +109,7 @@ protected String GetPropertyValue(String propertyName, String alternativePropert private string GetPropertyValueImpl(string propertyName) { - String value = String.Empty; + String value = null; if (!string.IsNullOrEmpty(propertyName)) { value = Environment.GetEnvironmentVariable(propertyName); diff --git a/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderCommon.cs b/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderCommon.cs new file mode 100644 index 000000000..c4c625709 --- /dev/null +++ b/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderCommon.cs @@ -0,0 +1,36 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using GeneXus.Services; + +namespace GxClasses.Services.Storage +{ + public static class ExternalProviderCommon + { + public static String getProviderObjectName(ExternalProvider provider, String objectNameOrUrl) + { + String providerObjectName = null; + if (provider != null) + { + //providerObjectName = provider.GetObjectNameFromURL(objectNameOrUrl); + if (providerObjectName != null && providerObjectName.IndexOf("?") > 0) + { + providerObjectName = providerObjectName.Substring(0, providerObjectName.IndexOf("?")); + } + } + return providerObjectName; + } + + public static String getProviderObjectAbsoluteUriSafe(ExternalProvider provider, String rawObjectUri) + { + String providerObjectName = getProviderObjectName(provider, rawObjectUri); + if (providerObjectName != null && rawObjectUri.IndexOf("?") > 0) + { + rawObjectUri = rawObjectUri.Substring(0, rawObjectUri.IndexOf("?")); + } + return rawObjectUri; + } + + } +} diff --git a/dotnet/test/DotNetUnitTest/DotNetUnitTest.csproj b/dotnet/test/DotNetUnitTest/DotNetUnitTest.csproj index abd6f8be1..085f4125b 100644 --- a/dotnet/test/DotNetUnitTest/DotNetUnitTest.csproj +++ b/dotnet/test/DotNetUnitTest/DotNetUnitTest.csproj @@ -52,6 +52,9 @@ PreserveNewest + + Always + From 57988be16a30a68a24cbc3e291d06a3bdf311db0 Mon Sep 17 00:00:00 2001 From: Claudia Murialdo Date: Fri, 4 Jun 2021 10:39:02 -0300 Subject: [PATCH 10/42] Always add a GUID suffix to the name of uploaded image files. --- .../dotnetframework/GxClasses/Core/GXUtilsCommon.cs | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/dotnet/src/dotnetframework/GxClasses/Core/GXUtilsCommon.cs b/dotnet/src/dotnetframework/GxClasses/Core/GXUtilsCommon.cs index a3dc44cb7..6f56fa6f1 100644 --- a/dotnet/src/dotnetframework/GxClasses/Core/GXUtilsCommon.cs +++ b/dotnet/src/dotnetframework/GxClasses/Core/GXUtilsCommon.cs @@ -3373,16 +3373,8 @@ public static string getTempFileName(string baseDir, string name, string extensi lock (syncObj) { - name = FixFileName(name, string.Empty); - if (string.IsNullOrEmpty(name)) - { - fileName = tempFileName(baseDir, name, extension); - } - else - { - fileName = PathUtil.SafeCombine(baseDir, name + "." + extension); - } + fileName = tempFileName(baseDir, name, extension); GxFile file = new GxFile(baseDir, fileName, fileType); while (file.FileInfo.Exist(fileName)) { From 08dcb7fb577e91980b882ecfd2a695bd389d488e Mon Sep 17 00:00:00 2001 From: Claudia Murialdo Date: Mon, 7 Jun 2021 16:28:00 -0300 Subject: [PATCH 11/42] Support for Private storage --- .../GxClasses/Core/GXApplication.cs | 15 ++---- .../dotnetframework/GxClasses/Core/GXUtils.cs | 5 +- .../GxClasses/Core/GXUtilsCommon.cs | 48 ++++++++++--------- .../GxClasses/Data/GXDataNTierADO.cs | 15 +++--- .../GxClasses/Domain/GXFileIO.cs | 16 +++---- .../GxClasses/Helpers/HttpHelper.cs | 2 +- .../GxClasses/Middleware/GXHttpServices.cs | 11 +++-- .../Services/Storage/ExternalProviderBase.cs | 15 +++++- .../Storage/ExternalProviderCommon.cs | 36 -------------- .../GxClasses/Services/Storage/GXServices.cs | 4 +- .../GxClasses/Storage/StorageFactory.cs | 29 ++++++++++- .../src/dotnetframework/GxExcel/GxExcelI.cs | 2 +- .../Storage/GXAmazonS3/ExternalProviderS3.cs | 9 ++-- .../AzureStorageExternalProvider.cs | 4 +- .../GXGoogleCloud/ExternalProviderGoogle.cs | 8 ++-- .../GXOpenStack/ExternalProviderOpenStack.cs | 2 +- .../ExternalProvider/ExternalProviderTest.cs | 6 +-- 17 files changed, 112 insertions(+), 115 deletions(-) delete mode 100644 dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderCommon.cs diff --git a/dotnet/src/dotnetframework/GxClasses/Core/GXApplication.cs b/dotnet/src/dotnetframework/GxClasses/Core/GXApplication.cs index 86b530b55..ffa334553 100644 --- a/dotnet/src/dotnetframework/GxClasses/Core/GXApplication.cs +++ b/dotnet/src/dotnetframework/GxClasses/Core/GXApplication.cs @@ -679,16 +679,13 @@ public static bool GetHttpRequestPostedFile(IGxContext gxContext, string varName if (httpContext != null) { HttpPostedFile pf = httpContext.Request.GetFile(varName); - if (pf != null) + if (pf != null && pf.ContentLength > 0) { -#pragma warning disable SCS0018 // Path traversal: injection possible in {1} argument passed to '{0}' - FileInfo fi = new FileInfo(pf.FileName); -#pragma warning restore SCS0018 // Path traversal: injection possible in {1} argument passed to '{0}' string tempDir = Preferences.getTMP_MEDIA_PATH(); - string ext = fi.Extension; + string ext = Path.GetExtension(pf.FileName); if (ext != null) ext = ext.TrimStart('.'); - string filePath = FileUtil.getTempFileName(tempDir, "BLOB", ext); + string filePath = FileUtil.getTempFileName(tempDir); GXLogging.Debug(log, "cgiGet(" + varName + "), fileName:" + filePath); GxFile file = new GxFile(tempDir, filePath, GxFileType.Private); filePath = file.Create(pf.InputStream); @@ -3752,8 +3749,7 @@ public string FileToBase64(string filePath) } public string FileFromBase64(string b64) { - string tmpFileName = Guid.NewGuid().ToString(); - string filePath = Path.Combine(Preferences.getTMP_MEDIA_PATH(), "Blob" + tmpFileName); + string filePath = FileUtil.getTempFileName(Preferences.getTMP_MEDIA_PATH()); GxFile auxFile = new GxFile(GetPhysicalPath(), filePath, GxFileType.Private); auxFile.FromBase64(b64); GXFileWatcher.Instance.AddTemporaryFile(new GxFile("", new GxFileInfo(filePath, "")), this); @@ -3769,8 +3765,7 @@ public byte[] FileToByteArray(string filePath) } public string FileFromByteArray(byte[] bArray) { - string tmpFileName = Guid.NewGuid().ToString(); - string filePath = Path.Combine(Preferences.getTMP_MEDIA_PATH(), "Blob" + tmpFileName); + string filePath = FileUtil.getTempFileName(Preferences.getTMP_MEDIA_PATH()); GxFile auxFile = new GxFile(GetPhysicalPath(), filePath, GxFileType.Private); auxFile.FromByteArray(bArray); GXFileWatcher.Instance.AddTemporaryFile(new GxFile("", new GxFileInfo(filePath, "")), this); diff --git a/dotnet/src/dotnetframework/GxClasses/Core/GXUtils.cs b/dotnet/src/dotnetframework/GxClasses/Core/GXUtils.cs index 0cb3458a0..23d3d8c80 100644 --- a/dotnet/src/dotnetframework/GxClasses/Core/GXUtils.cs +++ b/dotnet/src/dotnetframework/GxClasses/Core/GXUtils.cs @@ -1116,12 +1116,13 @@ public bool Upload(string filefullpath, string storageobjectfullname, GxFile upl try { ValidProvider(); + GxFileType acl = GxFileType.PublicRead; if (String.IsNullOrEmpty(storageobjectfullname)) { storageobjectfullname = Path.GetFileName(filefullpath); } - string url = provider.Upload(filefullpath, storageobjectfullname, GxFileType.PublicRead); - uploadedFile.FileInfo = new GxExternalFileInfo(storageobjectfullname, url, provider); + string url = provider.Upload(filefullpath, storageobjectfullname, acl); + uploadedFile.FileInfo = new GxExternalFileInfo(storageobjectfullname, url, provider, acl); return true; } catch (Exception ex) diff --git a/dotnet/src/dotnetframework/GxClasses/Core/GXUtilsCommon.cs b/dotnet/src/dotnetframework/GxClasses/Core/GXUtilsCommon.cs index 6f56fa6f1..2ec2b3ecd 100644 --- a/dotnet/src/dotnetframework/GxClasses/Core/GXUtilsCommon.cs +++ b/dotnet/src/dotnetframework/GxClasses/Core/GXUtilsCommon.cs @@ -37,6 +37,8 @@ using System.Security.Cryptography; using System.Collections.Concurrent; using System.Drawing.Drawing2D; +using GeneXus.Storage; +using GeneXus.Services; namespace GeneXus.Utils { @@ -3272,8 +3274,6 @@ static public DateTime FromTimeZone(DateTime dt, String sTZ, IGxContext context) public class FileUtil { static readonly ILog log = log4net.LogManager.GetLogger(typeof(GeneXus.Utils.FileUtil)); - - private static object syncObj = new object(); public static byte DeleteFile(string fileName) { try @@ -3366,22 +3366,10 @@ public static string UriToPath(string uriString) return uriString; } } - public static string getTempFileName(string baseDir, string name, string extension, GxFileType fileType = GxFileType.PublicRead) + public static string getTempFileName(string baseDir, string name="", string extension="tmp", GxFileType fileType = GxFileType.Private) { - String fileName; - name = FileUtil.FileNamePrettify(name); - - lock (syncObj) - { - name = FixFileName(name, string.Empty); - fileName = tempFileName(baseDir, name, extension); - GxFile file = new GxFile(baseDir, fileName, fileType); - while (file.FileInfo.Exist(fileName)) - { - fileName = tempFileName(baseDir, name, extension); - } - } - return fileName.Trim(); + name = FixFileName(FileUtil.FileNamePrettify(name), string.Empty); + return tempFileName(baseDir, name, extension); } private static string tempFileName(string baseDir, string name, string extension) @@ -3442,10 +3430,17 @@ public static string GetFileType(string FileName) public static string FileNamePrettify(string s) { - string str = GXUtil.RemoveDiacritics(s.Trim().ToLower()); //remove accents - str = Regex.Replace(str, @"\s+", " ").Trim(); // convert multiple spaces into one space - str = Regex.Replace(str, @"\s", "-"); // //Replace spaces by dashes - return str; + if (!string.IsNullOrEmpty(s)) + { + string str = GXUtil.RemoveDiacritics(s.Trim().ToLower()); //remove accents + str = Regex.Replace(str, @"\s+", " ").Trim(); // convert multiple spaces into one space + str = Regex.Replace(str, @"\s", "-"); // //Replace spaces by dashes + return str; + } + else + { + return s; + } } public static string GetFileName(string FileName) @@ -5249,7 +5244,14 @@ public static string ResolveUri(string uriString, IGxContext context = null) public static string ResolveUri(string uriString, bool absUrl, IGxContext context = null) { if (String.IsNullOrEmpty(uriString)) - return ""; + return string.Empty; + + string providerObjectName; + if (PathUtil.IsAbsoluteUrl(uriString) && StorageFactory.TryGetProviderObjectName(ServiceFactory.GetExternalProvider(), uriString, out providerObjectName)) + { + return new GxFile(string.Empty, providerObjectName).GetURI(); + } + if (schemeRegex.IsMatch(uriString)) { string fileName = schemeRegex.Replace(uriString, ""); @@ -5257,7 +5259,7 @@ public static string ResolveUri(string uriString, bool absUrl, IGxContext contex string basePath = Path.Combine(Path.Combine(Preferences.getBLOB_PATH(), MultimediaDirectory)); try { - GxFile file = new GxFile(string.Empty, PathUtil.SafeCombine(basePath, fileName), GxFileType.PublicRead); + GxFile file = new GxFile(string.Empty, PathUtil.SafeCombine(basePath, fileName)); return PathToUrl(file.GetURI(), absUrl, context); } catch (ArgumentException ex) diff --git a/dotnet/src/dotnetframework/GxClasses/Data/GXDataNTierADO.cs b/dotnet/src/dotnetframework/GxClasses/Data/GXDataNTierADO.cs index 4ea624d2b..8aa134c91 100644 --- a/dotnet/src/dotnetframework/GxClasses/Data/GXDataNTierADO.cs +++ b/dotnet/src/dotnetframework/GxClasses/Data/GXDataNTierADO.cs @@ -10,8 +10,7 @@ using System.Collections.Generic; using GeneXus.Services; using System.Net; -using System.Diagnostics; -using System.Linq; +using GeneXus.Storage; namespace GeneXus.Data.NTier.ADO { @@ -149,7 +148,7 @@ public string getMultimediaFile(int id, string gxdbFileUri) string filePath = PathUtil.SafeCombine(_gxDbCommand.Conn.MultimediaPath, fileName); try { - GxFile file = new GxFile(string.Empty, filePath, GxFileType.PublicRead); + GxFile file = new GxFile(string.Empty, filePath, GxFileType.Default); if (file.Exists()) { return filePath; @@ -433,7 +432,7 @@ public string getMultimediaFile(int id, string gxdbFileUri) try { - GxFile file = new GxFile(string.Empty, filePath, GxFileType.PublicRead); + GxFile file = new GxFile(string.Empty, filePath, GxFileType.Default); if (file.Exists()) { @@ -441,7 +440,7 @@ public string getMultimediaFile(int id, string gxdbFileUri) } else { - return getBLOBFile(id, FileUtil.GetFileType(gxdbFileUri), FileUtil.GetFileName(gxdbFileUri), filePath, false, GxFileType.PublicRead); + return getBLOBFile(id, FileUtil.GetFileType(gxdbFileUri), FileUtil.GetFileName(gxdbFileUri), filePath, false, GxFileType.Default); } } catch (ArgumentException) @@ -456,7 +455,7 @@ public string getMultimediaFile(int id, string gxdbFileUri) public string getMultimediaUri(int id) { - return GXDbFile.ResolveUri(getVarchar(id), true, _gxDbCommand.Conn.DataStore.Context); + return getMultimediaUri(id, true); } public string getMultimediaUri(int id, bool absUrl) @@ -613,7 +612,7 @@ public void SetParameterMultimedia(int id, string image_gxi, string image, strin string objectName; //file is already on the cloud p.e. https://s3.amazonaws.com/Test/PublicTempStorage/multimedia/Image_ad013b5b050c4bf199f544b5561d9b92.png //Must be copied to https://s3.amazonaws.com/Test/TableName/FieldName/Image_ad013b5b050c4bf199f544b5561d9b92.png - if (ServiceFactory.GetExternalProvider().GetObjectNameFromURL(image_gxi, out objectName)) + if (ServiceFactory.GetExternalProvider().TryGetObjectNameFromURL(image_gxi, out objectName)) { try { @@ -726,7 +725,7 @@ private static string PushToExternalProvider(Stream fileStream, string externalF string multimediaUri; using (fileStream) { - multimediaUri = ServiceFactory.GetExternalProvider().Save(fileStream, externalFileName, tableName, fieldName, GxFileType.PublicRead); + multimediaUri = ServiceFactory.GetExternalProvider().Save(fileStream, externalFileName, tableName, fieldName, GxFileType.Default); GXLogging.Debug(log, "Upload file to ExternalProvider:", multimediaUri); } diff --git a/dotnet/src/dotnetframework/GxClasses/Domain/GXFileIO.cs b/dotnet/src/dotnetframework/GxClasses/Domain/GXFileIO.cs index 34cee20b8..d2fd14d78 100644 --- a/dotnet/src/dotnetframework/GxClasses/Domain/GXFileIO.cs +++ b/dotnet/src/dotnetframework/GxClasses/Domain/GXFileIO.cs @@ -17,7 +17,6 @@ using GeneXus; using GeneXus.Storage; using GeneXus.Attributes; -using GxClasses.Services.Storage; public interface IGxDirectoryInfo { @@ -210,7 +209,7 @@ public IGxFileInfo[] GetFiles(string searchPattern) searchPattern = ""; List files = _provider.GetFiles(_name, searchPattern); - GxExternalFileInfo[] externalFiles = files.Select(elem => new GxExternalFileInfo(elem, _provider, GxFileType.PublicRead)).ToArray(); + GxExternalFileInfo[] externalFiles = files.Select(elem => new GxExternalFileInfo(elem, _provider, GxFileType.Default)).ToArray(); return externalFiles; } @@ -443,9 +442,9 @@ public GxExternalFileInfo(string storageObjectFullname, ExternalProvider provide } public GxExternalFileInfo(string storageObjectFullname, string url, ExternalProvider provider, GxFileType fileType = GxFileType.Private) - { - _name = storageObjectFullname; - _provider = provider; + { + _name = StorageFactory.GetProviderObjectAbsoluteUriSafe(provider, storageObjectFullname); + _provider = provider; _url = url; _fileTypeAtt = fileType; } @@ -667,13 +666,13 @@ public GxFile(string baseDirectory) _baseDirectory = _baseDirectory.Substring(0, _baseDirectory.Length - 1); } - public GxFile(string baseDirectory, IGxFileInfo file, GxFileType fileType = GxFileType.PublicRead) + public GxFile(string baseDirectory, IGxFileInfo file, GxFileType fileType = GxFileType.Private) : this(baseDirectory) { _file = file; } - public GxFile(string baseDirectory, string fileName, GxFileType fileType = GxFileType.PublicRead) + public GxFile(string baseDirectory, string fileName, GxFileType fileType = GxFileType.Private) : this(baseDirectory) { if (GxUploadHelper.IsUpload(fileName)) @@ -739,8 +738,7 @@ public string Source if (IsAbsoluteUrl(value)) { string objName = string.Empty; - ExternalProvider p = StorageFactory.GetExternalProviderFromUrl(value, out objName); - _file = new GxExternalFileInfo(objName, value, p); + _file = new GxExternalFileInfo(objName, value, ServiceFactory.GetExternalProvider()); } else { diff --git a/dotnet/src/dotnetframework/GxClasses/Helpers/HttpHelper.cs b/dotnet/src/dotnetframework/GxClasses/Helpers/HttpHelper.cs index f96dbf91d..5bc041ace 100644 --- a/dotnet/src/dotnetframework/GxClasses/Helpers/HttpHelper.cs +++ b/dotnet/src/dotnetframework/GxClasses/Helpers/HttpHelper.cs @@ -231,7 +231,7 @@ public static bool GetHttpRequestPostedFile(IGxContext gxContext, string varName string ext = fi.Extension; if (ext != null) ext = ext.TrimStart('.'); - filePath = FileUtil.getTempFileName(tempDir, "BLOB", ext); + filePath = FileUtil.getTempFileName(tempDir); GXLogging.Debug(log, "cgiGet(" + varName + "), fileName:" + filePath); GxFile file = new GxFile(tempDir, filePath); #if NETCORE diff --git a/dotnet/src/dotnetframework/GxClasses/Middleware/GXHttpServices.cs b/dotnet/src/dotnetframework/GxClasses/Middleware/GXHttpServices.cs index 2cf810e90..a2a1a879c 100644 --- a/dotnet/src/dotnetframework/GxClasses/Middleware/GXHttpServices.cs +++ b/dotnet/src/dotnetframework/GxClasses/Middleware/GXHttpServices.cs @@ -405,8 +405,9 @@ public override void webExecute() fName = hpf.FileName; ext = FileUtil.GetFileType(fName); - savedFileName = FileUtil.getTempFileName(Preferences.getTMP_MEDIA_PATH(), FileUtil.GetFileName(fName), string.IsNullOrEmpty(ext) ? "tmp" : ext); - GxFile gxFile = new GxFile(Preferences.getTMP_MEDIA_PATH(), savedFileName); + string tempDir = Preferences.getTMP_MEDIA_PATH(); + savedFileName = FileUtil.getTempFileName(tempDir); + GxFile gxFile = new GxFile(tempDir, savedFileName); gxFile.Create(hpf.InputStream); string uri = gxFile.GetURI(); @@ -458,9 +459,9 @@ internal void WcfExecute(Stream istream, string contentType) { string savedFileName, ext, fName; ext = context.ExtensionForContentType(contentType); - - savedFileName = FileUtil.getTempFileName(Preferences.getTMP_MEDIA_PATH(), "BLOB", string.IsNullOrEmpty(ext) ? "tmp" : ext); - GxFile file = new GxFile(Preferences.getTMP_MEDIA_PATH(), savedFileName); + string tempDir = Preferences.getTMP_MEDIA_PATH(); + savedFileName = FileUtil.getTempFileName(tempDir); + GxFile file = new GxFile(tempDir, savedFileName); file.Create(istream); JObject obj = new JObject(); diff --git a/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderBase.cs b/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderBase.cs index c80108011..5a9bc4b7c 100644 --- a/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderBase.cs +++ b/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderBase.cs @@ -14,8 +14,8 @@ public abstract class ExternalProviderBase protected static String DEFAULT_EXPIRATION = "DEFAULT_EXPIRATION"; protected static String FOLDER = "FOLDER_NAME"; protected static String DEFAULT_ACL_DEPRECATED = "STORAGE_PROVIDER_DEFAULT_ACL"; - protected static String DEFAULT_EXPIRATION_DEPRECATED = "STORAGE_PROVIDER_DEFAULT_EXPIRATION"; - protected const int DefaultExpirationMinutes = 24 * 60; + protected static String DEFAULT_EXPIRATION_DEPRECATED = "STORAGE_PROVIDER_DEFAULT_EXPIRATION"; + protected int defaultExpirationMinutes = 1440; protected GxFileType defaultAcl = GxFileType.Private; @@ -54,6 +54,17 @@ private void Initialize() { this.defaultAcl = GxFileType.PublicRead; } + + String expirationS = GetPropertyValue(DEFAULT_EXPIRATION, DEFAULT_EXPIRATION, defaultExpirationMinutes.ToString()); + if (!String.IsNullOrEmpty(expirationS)) + { + Int32.TryParse(expirationS, out this.defaultExpirationMinutes); + } + } + + protected int ResolveExpirationMinutes(int expirationMinutes) + { + return expirationMinutes > 0 ? expirationMinutes : defaultExpirationMinutes; } protected String GetEncryptedPropertyValue(String propertyName, String alternativePropertyName = null) diff --git a/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderCommon.cs b/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderCommon.cs deleted file mode 100644 index c4c625709..000000000 --- a/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderCommon.cs +++ /dev/null @@ -1,36 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; -using GeneXus.Services; - -namespace GxClasses.Services.Storage -{ - public static class ExternalProviderCommon - { - public static String getProviderObjectName(ExternalProvider provider, String objectNameOrUrl) - { - String providerObjectName = null; - if (provider != null) - { - //providerObjectName = provider.GetObjectNameFromURL(objectNameOrUrl); - if (providerObjectName != null && providerObjectName.IndexOf("?") > 0) - { - providerObjectName = providerObjectName.Substring(0, providerObjectName.IndexOf("?")); - } - } - return providerObjectName; - } - - public static String getProviderObjectAbsoluteUriSafe(ExternalProvider provider, String rawObjectUri) - { - String providerObjectName = getProviderObjectName(provider, rawObjectUri); - if (providerObjectName != null && rawObjectUri.IndexOf("?") > 0) - { - rawObjectUri = rawObjectUri.Substring(0, rawObjectUri.IndexOf("?")); - } - return rawObjectUri; - } - - } -} diff --git a/dotnet/src/dotnetframework/GxClasses/Services/Storage/GXServices.cs b/dotnet/src/dotnetframework/GxClasses/Services/Storage/GXServices.cs index 399e02a56..f78e0a202 100644 --- a/dotnet/src/dotnetframework/GxClasses/Services/Storage/GXServices.cs +++ b/dotnet/src/dotnetframework/GxClasses/Services/Storage/GXServices.cs @@ -247,9 +247,9 @@ public interface ExternalProvider List GetFiles(string directoryName, string filter = ""); List GetSubDirectories(string directoryName); Stream GetStream(string objectName, GxFileType fileType); - bool GetMessageFromException(Exception ex, SdtMessages_Message msg); - bool GetObjectNameFromURL(string url, out string objectName); + bool GetMessageFromException(Exception ex, SdtMessages_Message msg); string GetBaseURL(); + bool TryGetObjectNameFromURL(string objectNameOrUrl, out string providerObjectName); string StorageUri { get; } } } diff --git a/dotnet/src/dotnetframework/GxClasses/Storage/StorageFactory.cs b/dotnet/src/dotnetframework/GxClasses/Storage/StorageFactory.cs index 0ef2a2d3f..eeb36091f 100644 --- a/dotnet/src/dotnetframework/GxClasses/Storage/StorageFactory.cs +++ b/dotnet/src/dotnetframework/GxClasses/Storage/StorageFactory.cs @@ -5,13 +5,13 @@ namespace GeneXus.Storage public class StorageFactory { const char QUESTION_MARK = '?'; - public static ExternalProvider GetExternalProviderFromUrl(string url, out string objectName) + /*public static ExternalProvider GetExternalProviderFromUrl(string url, out string objectName) { objectName = null; ExternalProvider provider = ServiceFactory.GetExternalProvider(); if (provider != null) { - if (provider.GetObjectNameFromURL(url, out objectName)) + if (provider.TryGetObjectNameFromURL(url, out objectName)) { var questionMarkIndex = objectName.IndexOf(QUESTION_MARK); objectName = questionMarkIndex >= 0 ? objectName.Substring(0, questionMarkIndex): objectName.Substring(0); @@ -20,5 +20,30 @@ public static ExternalProvider GetExternalProviderFromUrl(string url, out string } return null; } + */ + public static bool TryGetProviderObjectName(ExternalProvider provider, string objectNameOrUrl, out string providerObjectName) + { + providerObjectName = null; + if (provider != null && provider.TryGetObjectNameFromURL(objectNameOrUrl, out providerObjectName)) + { + int idx = providerObjectName.IndexOf(QUESTION_MARK); + if (idx > 0) + { + providerObjectName = providerObjectName.Substring(0, idx); + } + return true; + } + return false; + } + + public static string GetProviderObjectAbsoluteUriSafe(ExternalProvider provider, string rawObjectUri) + { + string providerObjectName; + if (TryGetProviderObjectName(provider, rawObjectUri, out providerObjectName)) + { + rawObjectUri = providerObjectName; + } + return rawObjectUri; + } } } diff --git a/dotnet/src/dotnetframework/GxExcel/GxExcelI.cs b/dotnet/src/dotnetframework/GxExcel/GxExcelI.cs index 79168b9f3..27d4c25a9 100644 --- a/dotnet/src/dotnetframework/GxExcel/GxExcelI.cs +++ b/dotnet/src/dotnetframework/GxExcel/GxExcelI.cs @@ -144,7 +144,7 @@ public String Template string localTemplate = value; if (!Path.IsPathRooted(localTemplate)) localTemplate = Path.Combine(GxContext.StaticPhysicalPath(), localTemplate); - ServiceFactory.GetExternalProvider().Upload(localTemplate, template, GxFileType.PublicRead); + ServiceFactory.GetExternalProvider().Upload(localTemplate, template, GxFileType.Private); } } else if (!Path.IsPathRooted(value)) diff --git a/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs b/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs index 28ca58f00..afebd64ad 100644 --- a/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs +++ b/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs @@ -247,7 +247,7 @@ private bool IsPrivateUpload(GxFileType fileType) return GetCannedACL(fileType) != S3CannedACL.PublicRead; } - public string Get(string objectName, GxFileType fileType, int urlMinutes = DefaultExpirationMinutes) + public string Get(string objectName, GxFileType fileType, int urlMinutes = 0) { if (Exists(objectName, fileType)) { @@ -257,10 +257,10 @@ public string Get(string objectName, GxFileType fileType, int urlMinutes = Defau return string.Empty; } - private string GetUrlImpl(string objectName, GxFileType fileType, int urlMinutes = DefaultExpirationMinutes) + private string GetUrlImpl(string objectName, GxFileType fileType, int urlMinutes = 0) { bool isPrivate = IsPrivateUpload(fileType); - return (isPrivate)? GetPreSignedUrl(objectName, urlMinutes): StorageUri + StorageUtils.EncodeUrl(objectName); + return (isPrivate)? GetPreSignedUrl(objectName, ResolveExpirationMinutes(urlMinutes)): StorageUri + StorageUtils.EncodeUrl(objectName); } @@ -521,7 +521,7 @@ public bool GetMessageFromException(Exception ex, SdtMessages_Message msg) } } - public bool GetObjectNameFromURL(string url, out string objectName) + public bool TryGetObjectNameFromURL(string url, out string objectName) { if (url.StartsWith(StorageUri)) { @@ -536,5 +536,6 @@ public override string GetName() { return Name; } + } } diff --git a/dotnet/src/dotnetframework/Providers/Storage/GXAzureStorage/AzureStorageExternalProvider.cs b/dotnet/src/dotnetframework/Providers/Storage/GXAzureStorage/AzureStorageExternalProvider.cs index 1b46ada6c..5c5351db8 100644 --- a/dotnet/src/dotnetframework/Providers/Storage/GXAzureStorage/AzureStorageExternalProvider.cs +++ b/dotnet/src/dotnetframework/Providers/Storage/GXAzureStorage/AzureStorageExternalProvider.cs @@ -125,7 +125,7 @@ private string GetURL(CloudBlockBlob blob, GxFileType fileType, int urlMinutes = { SharedAccessBlobPolicy sasConstraints = new SharedAccessBlobPolicy(); sasConstraints.SharedAccessStartTime = DateTime.UtcNow; - sasConstraints.SharedAccessExpiryTime = DateTime.UtcNow.AddMinutes((urlMinutes > 0) ? urlMinutes : 60 * 24 * 7); + sasConstraints.SharedAccessExpiryTime = DateTime.UtcNow.AddMinutes(ResolveExpirationMinutes(urlMinutes)); sasConstraints.Permissions = SharedAccessBlobPermissions.Read; url += blob.GetSharedAccessSignature(sasConstraints); } @@ -352,7 +352,7 @@ public string Save(Stream fileStream, string fileName, string tableName, string return Upload(newName, fileStream, fileType); } - public bool GetObjectNameFromURL(string url, out string objectName) + public bool TryGetObjectNameFromURL(string url, out string objectName) { string baseUrl = StorageUri + StorageUtils.DELIMITER + PublicContainer.Name + StorageUtils.DELIMITER; if (url.StartsWith(baseUrl)) diff --git a/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs b/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs index 040ca64b8..e1fcdbdb1 100644 --- a/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs +++ b/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs @@ -192,7 +192,7 @@ public string Upload(string localFile, string objectName, GxFileType fileType) using (FileStream stream = new FileStream(localFile, FileMode.Open)) { Google.Apis.Storage.v1.Data.Object obj = Client.UploadObject(Bucket, objectName, "application/octet-stream", stream, GetUploadOptions(fileType)); - return GetURL(objectName, fileType, DefaultExpirationMinutes); + return GetURL(objectName, fileType); } } @@ -245,10 +245,10 @@ public string Get(string objectName, GxFileType fileType, int urlMinutes) return string.Empty; } - private string GetURL(string objectName, GxFileType fileType, int urlMinutes = 60 * 24 * 7) + private string GetURL(string objectName, GxFileType fileType, int urlMinutes = 0) { if (fileType.HasFlag(GxFileType.Private)) - return Signer.Sign(Bucket, StorageUtils.EncodeUrl(objectName), TimeSpan.FromMinutes(urlMinutes), HttpMethod.Get); + return Signer.Sign(Bucket, StorageUtils.EncodeUrl(objectName), TimeSpan.FromMinutes(ResolveExpirationMinutes(urlMinutes)), HttpMethod.Get); else { return StorageUri + StorageUtils.EncodeUrl(objectName); @@ -433,7 +433,7 @@ public string Save(Stream fileStream, string fileName, string tableName, string fileName = Folder + StorageUtils.DELIMITER + tableName + StorageUtils.DELIMITER + fieldName + StorageUtils.DELIMITER + fileName; return Upload(fileName, fileStream, fileType); } - public bool GetObjectNameFromURL(string url, out string objectName) + public bool TryGetObjectNameFromURL(string url, out string objectName) { string baseUrl = StorageUri; if (url.StartsWith(baseUrl)) diff --git a/dotnet/src/dotnetframework/Providers/Storage/GXOpenStack/ExternalProviderOpenStack.cs b/dotnet/src/dotnetframework/Providers/Storage/GXOpenStack/ExternalProviderOpenStack.cs index a8aaddff1..bd9a94348 100644 --- a/dotnet/src/dotnetframework/Providers/Storage/GXOpenStack/ExternalProviderOpenStack.cs +++ b/dotnet/src/dotnetframework/Providers/Storage/GXOpenStack/ExternalProviderOpenStack.cs @@ -362,7 +362,7 @@ public string GetBaseURL() return storageUrl + StorageUtils.DELIMITER + publicBucketName + StorageUtils.DELIMITER; } - public bool GetObjectNameFromURL(string url, out string objectName) + public bool TryGetObjectNameFromURL(string url, out string objectName) { string baseUrl = storageUrl + StorageUtils.DELIMITER + publicBucketName + StorageUtils.DELIMITER; if (url.StartsWith(baseUrl)) diff --git a/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs b/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs index b7dae0295..bbe764a12 100644 --- a/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs +++ b/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs @@ -156,10 +156,10 @@ public void TestGetMethod() public void TestGetObjectName() { TestUploadPublicMethod(); - String url = provider.Get(TEST_SAMPLE_FILE_NAME, GxFileType.PublicRead, 10); + string url = provider.Get(TEST_SAMPLE_FILE_NAME, GxFileType.PublicRead, 10); Assert.True(UrlExists(url)); - String objectName = ""; - provider.GetObjectNameFromURL(url, out objectName); + string objectName; + provider.TryGetObjectNameFromURL(url, out objectName); Assert.Equal("text.txt", objectName); } From de2ba310e66c0df1efd33de32fcfe67fe17a11e9 Mon Sep 17 00:00:00 2001 From: Gonzalo Date: Mon, 14 Jun 2021 14:51:14 -0300 Subject: [PATCH 12/42] Add Workflow for External Storage Unit Tests --- .github/workflows/External-Storage-Tests.yml | 99 ++++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 .github/workflows/External-Storage-Tests.yml diff --git a/.github/workflows/External-Storage-Tests.yml b/.github/workflows/External-Storage-Tests.yml new file mode 100644 index 000000000..c9ca8874c --- /dev/null +++ b/.github/workflows/External-Storage-Tests.yml @@ -0,0 +1,99 @@ +name: Build + +on: + pull_request: + branches: + - 'master' + - 'release-*' + push: + branches: + - 'master' + - 'beta' + - 'release-*' + +jobs: + build: + env: + GIT_REF: ${{ github.ref }} + GIT_SHA: ${{ github.sha }} + Configuration: Release + SolutionFile: dotnet\DotNetStandardClasses.sln + + runs-on: windows-latest + + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Install .NET Core 3.1 + uses: actions/setup-dotnet@v1 + with: + dotnet-version: '3.1.x' + + - name: Install .NET 6 + uses: actions/setup-dotnet@v1.7.2 + with: + dotnet-version: '6.0.100-preview.3.21202.5' + + - uses: actions/setup-dotnet@v1 + with: + source-url: https://nuget.pkg.github.com/genexuslabs/index.json + env: + NUGET_AUTH_TOKEN: ${{secrets.GITHUB_TOKEN}} + + - name: Calculate environment variables + run: | + $IsPrerelease = !($Env:GIT_REF -match 'release-[0-9]+(?:\.[0-9]+)?$') + echo "IsPrerelease=$IsPrerelease" >> $env:GITHUB_ENV + + $COMMIT_NUMBER = @($(git rev-list --count origin/master..), $(git rev-list --count HEAD))[$IsPrerelease] + + echo "COMMIT_NUMBER=$COMMIT_NUMBER" >> $env:GITHUB_ENV + + - name: Calculate package version + env: + PackageVersionString: ./.github/generatePackageVersion.ps1 + run: | + $NuGetPackageVersion = & "$Env:PackageVersionString" + + Write-Output "Packge version to be used: $NuGetPackageVersion" + + echo "NuGetPackageVersion=$NuGetPackageVersion" >> $env:GITHUB_ENV + + - name: Restore packages + run: dotnet restore $Env:SolutionFile + + - name: Build + run: dotnet build $Env:SolutionFile --no-restore --configuration $Env:Configuration + + - name: Test External Storage + run: | + export AWSS3_TEST_ENABLED=true + export STORAGE_AWSS3_ACCESS_KEY="${{ secrets.AWSS3_ACCESS_KEY }}" + export STORAGE_AWSS3_SECRET_KEY="${{ secrets.AWSS3_SECRET_KEY }}" + export STORAGE_AWSS3_BUCKET_NAME=genexuss3test + export STORAGE_AWSS3_FOLDER_NAME=gxclasses + export STORAGE_AWSS3_REGION=us-east-1 + export IBMCOS_TEST_ENABLED=true + export STORAGE_IBMCOS_ACCESS_KEY="${{ secrets.IBMCOS_ACCESS_KEY }}" + export STORAGE_IBMCOS_SECRET_KEY="${{ secrets.IBMCOS_SECRET_KEY }}" + export STORAGE_IBMCOS_BUCKET_NAME=gxclasses-unit-tests + export STORAGE_IBMCOS_FOLDER_NAME=tests + export STORAGE_IBMCOS_REGION=us-south + export AZUREBS_TEST_ENABLED=true + export STORAGE_AZUREBS_ACCESS_KEY="${{ secrets.AZUREBS_ACCESS_KEY }}" + export STORAGE_AZUREBS_ACCOUNT_NAME="${{ secrets.AZUREBS_ACCOUNT_NAME }}" + export STORAGE_AZUREBS_FOLDER_NAME=tests + export STORAGE_AZUREBS_PUBLIC_CONTAINER_NAME=contluispublic + export STORAGE_AZUREBS_PRIVATE_CONTAINER_NAME=contluisprivate + export GOOGLECS_TEST_ENABLED=true + export STORAGE_GOOGLECS_KEY="${{ secrets.GOOGLECS_KEY }}" + export STORAGE_GOOGLECS_PROJECT_ID=gxjavacloudstorageunittests + export STORAGE_GOOGLECS_BUCKET_NAME=javaclasses-unittests + export STORAGE_GOOGLECS_FOLDER_NAME=gxclasses + export STORAGE_GOOGLECS_APPLICATION_NAME=gxjavacloudstorageunittests + + dotnet test $Env:SolutionFile --no-restore --no-build --configuration $Env:Configuration + From 05b99b33a91bbe5e3eb3a7dc636207c5691018f6 Mon Sep 17 00:00:00 2001 From: Gonzalo Date: Mon, 14 Jun 2021 14:58:07 -0300 Subject: [PATCH 13/42] Workflow renamed --- .github/workflows/External-Storage-Tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/External-Storage-Tests.yml b/.github/workflows/External-Storage-Tests.yml index c9ca8874c..2ae7522de 100644 --- a/.github/workflows/External-Storage-Tests.yml +++ b/.github/workflows/External-Storage-Tests.yml @@ -1,4 +1,4 @@ -name: Build +name: External Storage Tests on: pull_request: From fa68e1b3ef4cfeed46cb2ad9ae4d767c04b82051 Mon Sep 17 00:00:00 2001 From: Gonzalo Date: Mon, 14 Jun 2021 15:24:20 -0300 Subject: [PATCH 14/42] Set Environment Variables --- .github/workflows/External-Storage-Tests.yml | 48 ++++++++++---------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/.github/workflows/External-Storage-Tests.yml b/.github/workflows/External-Storage-Tests.yml index 2ae7522de..daab0e351 100644 --- a/.github/workflows/External-Storage-Tests.yml +++ b/.github/workflows/External-Storage-Tests.yml @@ -70,30 +70,30 @@ jobs: - name: Test External Storage run: | - export AWSS3_TEST_ENABLED=true - export STORAGE_AWSS3_ACCESS_KEY="${{ secrets.AWSS3_ACCESS_KEY }}" - export STORAGE_AWSS3_SECRET_KEY="${{ secrets.AWSS3_SECRET_KEY }}" - export STORAGE_AWSS3_BUCKET_NAME=genexuss3test - export STORAGE_AWSS3_FOLDER_NAME=gxclasses - export STORAGE_AWSS3_REGION=us-east-1 - export IBMCOS_TEST_ENABLED=true - export STORAGE_IBMCOS_ACCESS_KEY="${{ secrets.IBMCOS_ACCESS_KEY }}" - export STORAGE_IBMCOS_SECRET_KEY="${{ secrets.IBMCOS_SECRET_KEY }}" - export STORAGE_IBMCOS_BUCKET_NAME=gxclasses-unit-tests - export STORAGE_IBMCOS_FOLDER_NAME=tests - export STORAGE_IBMCOS_REGION=us-south - export AZUREBS_TEST_ENABLED=true - export STORAGE_AZUREBS_ACCESS_KEY="${{ secrets.AZUREBS_ACCESS_KEY }}" - export STORAGE_AZUREBS_ACCOUNT_NAME="${{ secrets.AZUREBS_ACCOUNT_NAME }}" - export STORAGE_AZUREBS_FOLDER_NAME=tests - export STORAGE_AZUREBS_PUBLIC_CONTAINER_NAME=contluispublic - export STORAGE_AZUREBS_PRIVATE_CONTAINER_NAME=contluisprivate - export GOOGLECS_TEST_ENABLED=true - export STORAGE_GOOGLECS_KEY="${{ secrets.GOOGLECS_KEY }}" - export STORAGE_GOOGLECS_PROJECT_ID=gxjavacloudstorageunittests - export STORAGE_GOOGLECS_BUCKET_NAME=javaclasses-unittests - export STORAGE_GOOGLECS_FOLDER_NAME=gxclasses - export STORAGE_GOOGLECS_APPLICATION_NAME=gxjavacloudstorageunittests + $Env:AWSS3_TEST_ENABLED=true + $Env:STORAGE_AWSS3_ACCESS_KEY="${{ secrets.AWSS3_ACCESS_KEY }}" + $Env:STORAGE_AWSS3_SECRET_KEY="${{ secrets.AWSS3_SECRET_KEY }}" + $Env:STORAGE_AWSS3_BUCKET_NAME=genexuss3test + $Env:STORAGE_AWSS3_FOLDER_NAME=gxclasses + $Env:STORAGE_AWSS3_REGION=us-east-1 + $Env:IBMCOS_TEST_ENABLED=true + $Env:STORAGE_IBMCOS_ACCESS_KEY="${{ secrets.IBMCOS_ACCESS_KEY }}" + $Env:STORAGE_IBMCOS_SECRET_KEY="${{ secrets.IBMCOS_SECRET_KEY }}" + $Env:STORAGE_IBMCOS_BUCKET_NAME=gxclasses-unit-tests + $Env:STORAGE_IBMCOS_FOLDER_NAME=tests + $Env:STORAGE_IBMCOS_REGION=us-south + $Env:AZUREBS_TEST_ENABLED=true + $Env:STORAGE_AZUREBS_ACCESS_KEY="${{ secrets.AZUREBS_ACCESS_KEY }}" + $Env:STORAGE_AZUREBS_ACCOUNT_NAME="${{ secrets.AZUREBS_ACCOUNT_NAME }}" + $Env:STORAGE_AZUREBS_FOLDER_NAME=tests + $Env:STORAGE_AZUREBS_PUBLIC_CONTAINER_NAME=contluispublic + $Env:STORAGE_AZUREBS_PRIVATE_CONTAINER_NAME=contluisprivate + $Env:GOOGLECS_TEST_ENABLED=true + $Env:STORAGE_GOOGLECS_KEY="${{ secrets.GOOGLECS_KEY }}" + $Env:STORAGE_GOOGLECS_PROJECT_ID=gxjavacloudstorageunittests + $Env:STORAGE_GOOGLECS_BUCKET_NAME=javaclasses-unittests + $Env:STORAGE_GOOGLECS_FOLDER_NAME=gxclasses + $Env:STORAGE_GOOGLECS_APPLICATION_NAME=gxjavacloudstorageunittests dotnet test $Env:SolutionFile --no-restore --no-build --configuration $Env:Configuration From cb50197dd5fc7d53932e2ff5ca2554e42312cd23 Mon Sep 17 00:00:00 2001 From: Gonzalo Date: Mon, 14 Jun 2021 15:29:49 -0300 Subject: [PATCH 15/42] Set Environment Variables --- .github/workflows/External-Storage-Tests.yml | 34 ++++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/.github/workflows/External-Storage-Tests.yml b/.github/workflows/External-Storage-Tests.yml index daab0e351..0baca4659 100644 --- a/.github/workflows/External-Storage-Tests.yml +++ b/.github/workflows/External-Storage-Tests.yml @@ -70,30 +70,30 @@ jobs: - name: Test External Storage run: | - $Env:AWSS3_TEST_ENABLED=true + $Env:AWSS3_TEST_ENABLED="true" $Env:STORAGE_AWSS3_ACCESS_KEY="${{ secrets.AWSS3_ACCESS_KEY }}" $Env:STORAGE_AWSS3_SECRET_KEY="${{ secrets.AWSS3_SECRET_KEY }}" - $Env:STORAGE_AWSS3_BUCKET_NAME=genexuss3test - $Env:STORAGE_AWSS3_FOLDER_NAME=gxclasses - $Env:STORAGE_AWSS3_REGION=us-east-1 - $Env:IBMCOS_TEST_ENABLED=true + $Env:STORAGE_AWSS3_BUCKET_NAME="genexuss3test" + $Env:STORAGE_AWSS3_FOLDER_NAME="gxclasses" + $Env:STORAGE_AWSS3_REGION="us-east-1" + $Env:IBMCOS_TEST_ENABLED="true" $Env:STORAGE_IBMCOS_ACCESS_KEY="${{ secrets.IBMCOS_ACCESS_KEY }}" $Env:STORAGE_IBMCOS_SECRET_KEY="${{ secrets.IBMCOS_SECRET_KEY }}" - $Env:STORAGE_IBMCOS_BUCKET_NAME=gxclasses-unit-tests - $Env:STORAGE_IBMCOS_FOLDER_NAME=tests - $Env:STORAGE_IBMCOS_REGION=us-south - $Env:AZUREBS_TEST_ENABLED=true + $Env:STORAGE_IBMCOS_BUCKET_NAME="gxclasses-unit-tests" + $Env:STORAGE_IBMCOS_FOLDER_NAME="tests" + $Env:STORAGE_IBMCOS_REGION="us-south" + $Env:AZUREBS_TEST_ENABLED="true" $Env:STORAGE_AZUREBS_ACCESS_KEY="${{ secrets.AZUREBS_ACCESS_KEY }}" $Env:STORAGE_AZUREBS_ACCOUNT_NAME="${{ secrets.AZUREBS_ACCOUNT_NAME }}" - $Env:STORAGE_AZUREBS_FOLDER_NAME=tests - $Env:STORAGE_AZUREBS_PUBLIC_CONTAINER_NAME=contluispublic - $Env:STORAGE_AZUREBS_PRIVATE_CONTAINER_NAME=contluisprivate - $Env:GOOGLECS_TEST_ENABLED=true + $Env:STORAGE_AZUREBS_FOLDER_NAME="tests" + $Env:STORAGE_AZUREBS_PUBLIC_CONTAINER_NAME="contluispublic" + $Env:STORAGE_AZUREBS_PRIVATE_CONTAINER_NAME="contluisprivate" + $Env:GOOGLECS_TEST_ENABLED="true" $Env:STORAGE_GOOGLECS_KEY="${{ secrets.GOOGLECS_KEY }}" - $Env:STORAGE_GOOGLECS_PROJECT_ID=gxjavacloudstorageunittests - $Env:STORAGE_GOOGLECS_BUCKET_NAME=javaclasses-unittests - $Env:STORAGE_GOOGLECS_FOLDER_NAME=gxclasses - $Env:STORAGE_GOOGLECS_APPLICATION_NAME=gxjavacloudstorageunittests + $Env:STORAGE_GOOGLECS_PROJECT_ID="gxjavacloudstorageunittests" + $Env:STORAGE_GOOGLECS_BUCKET_NAME="javaclasses-unittests" + $Env:STORAGE_GOOGLECS_FOLDER_NAME="gxclasses" + $Env:STORAGE_GOOGLECS_APPLICATION_NAME="gxjavacloudstorageunittests" dotnet test $Env:SolutionFile --no-restore --no-build --configuration $Env:Configuration From 73dac973ca613b71ebee1fb0ba72eabea2a277a5 Mon Sep 17 00:00:00 2001 From: Gonzalo Date: Wed, 16 Jun 2021 10:18:28 -0300 Subject: [PATCH 16/42] Remove commented code --- dotnet/src/dotnetframework/GxClasses/Domain/GXFileIO.cs | 7 ------- 1 file changed, 7 deletions(-) diff --git a/dotnet/src/dotnetframework/GxClasses/Domain/GXFileIO.cs b/dotnet/src/dotnetframework/GxClasses/Domain/GXFileIO.cs index d2fd14d78..55901a29d 100644 --- a/dotnet/src/dotnetframework/GxClasses/Domain/GXFileIO.cs +++ b/dotnet/src/dotnetframework/GxClasses/Domain/GXFileIO.cs @@ -431,13 +431,6 @@ public GxExternalFileInfo(string storageObjectFullname, ExternalProvider provide { _url = storageObjectFullname; } - else { - /*if (fileType.HasFlag(GxFileType.Private)) //Attributes multimedia consider Storage Provider Folder - { - _url = provider.GetBaseURL() + storageObjectFullname; - _name = ExternalProviderCommon.getProviderObjectName(provider, storageObjectFullname); - }*/ - } _fileTypeAtt = fileType; } From ef2692d46807b442b0e8652680c4617407d3d551 Mon Sep 17 00:00:00 2001 From: Gonzalo Date: Wed, 16 Jun 2021 10:19:46 -0300 Subject: [PATCH 17/42] Remove commented code --- .../GxClasses/Storage/StorageFactory.cs | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/dotnet/src/dotnetframework/GxClasses/Storage/StorageFactory.cs b/dotnet/src/dotnetframework/GxClasses/Storage/StorageFactory.cs index eeb36091f..f4bc8fdda 100644 --- a/dotnet/src/dotnetframework/GxClasses/Storage/StorageFactory.cs +++ b/dotnet/src/dotnetframework/GxClasses/Storage/StorageFactory.cs @@ -5,22 +5,6 @@ namespace GeneXus.Storage public class StorageFactory { const char QUESTION_MARK = '?'; - /*public static ExternalProvider GetExternalProviderFromUrl(string url, out string objectName) - { - objectName = null; - ExternalProvider provider = ServiceFactory.GetExternalProvider(); - if (provider != null) - { - if (provider.TryGetObjectNameFromURL(url, out objectName)) - { - var questionMarkIndex = objectName.IndexOf(QUESTION_MARK); - objectName = questionMarkIndex >= 0 ? objectName.Substring(0, questionMarkIndex): objectName.Substring(0); - return provider; - } - } - return null; - } - */ public static bool TryGetProviderObjectName(ExternalProvider provider, string objectNameOrUrl, out string providerObjectName) { providerObjectName = null; From 58c7cfa5c276e5410816e2636fe0e602298bfd02 Mon Sep 17 00:00:00 2001 From: Gonzalo Date: Wed, 16 Jun 2021 10:33:02 -0300 Subject: [PATCH 18/42] Use TimeSpan instead of Minutes --- .../Services/Storage/ExternalProviderBase.cs | 18 +++++++++++------- .../Storage/GXAmazonS3/ExternalProviderS3.cs | 13 ++++--------- .../AzureStorageExternalProvider.cs | 2 +- .../GXGoogleCloud/ExternalProviderGoogle.cs | 2 +- 4 files changed, 17 insertions(+), 18 deletions(-) diff --git a/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderBase.cs b/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderBase.cs index 5a9bc4b7c..e6f3bbbea 100644 --- a/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderBase.cs +++ b/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderBase.cs @@ -14,10 +14,10 @@ public abstract class ExternalProviderBase protected static String DEFAULT_EXPIRATION = "DEFAULT_EXPIRATION"; protected static String FOLDER = "FOLDER_NAME"; protected static String DEFAULT_ACL_DEPRECATED = "STORAGE_PROVIDER_DEFAULT_ACL"; - protected static String DEFAULT_EXPIRATION_DEPRECATED = "STORAGE_PROVIDER_DEFAULT_EXPIRATION"; - protected int defaultExpirationMinutes = 1440; - + protected static String DEFAULT_EXPIRATION_DEPRECATED = "STORAGE_PROVIDER_DEFAULT_EXPIRATION"; + protected TimeSpan defaultExpiration = new TimeSpan(24, 0, 0); + protected GxFileType defaultAcl = GxFileType.Private; public ExternalProviderBase() @@ -55,16 +55,20 @@ private void Initialize() this.defaultAcl = GxFileType.PublicRead; } - String expirationS = GetPropertyValue(DEFAULT_EXPIRATION, DEFAULT_EXPIRATION, defaultExpirationMinutes.ToString()); + String expirationS = GetPropertyValue(DEFAULT_EXPIRATION, DEFAULT_EXPIRATION, defaultExpiration.ToString()); if (!String.IsNullOrEmpty(expirationS)) { - Int32.TryParse(expirationS, out this.defaultExpirationMinutes); + int minutes; + if (Int32.TryParse(expirationS, out minutes) && minutes > 0) + { + defaultExpiration = new TimeSpan(0, minutes, 0); + } } } - protected int ResolveExpirationMinutes(int expirationMinutes) + protected TimeSpan ResolveExpiration(int expirationMinutes) { - return expirationMinutes > 0 ? expirationMinutes : defaultExpirationMinutes; + return expirationMinutes > 0 ? new TimeSpan(0, expirationMinutes, 0) : defaultExpiration; } protected String GetEncryptedPropertyValue(String propertyName, String alternativePropertyName = null) diff --git a/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs b/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs index afebd64ad..19e756900 100644 --- a/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs +++ b/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs @@ -39,11 +39,6 @@ public class ExternalProviderS3 : ExternalProviderBase, ExternalProvider [Obsolete("Use Property STORAGE_CUSTOM_ENDPOINT instead", false)] const string STORAGE_CUSTOM_ENDPOINT_DEPRECATED = "STORAGE_CUSTOM_ENDPOINT"; - [Obsolete("Use Property BUCKET instead", false)] - const string BUCKET_DEPRECATED = "BUCKET_NAME"; - [Obsolete("Use Property FOLDER instead", false)] - const string FOLDER_DEPRECATED = "FOLDER_NAME"; - string _storageUri; IAmazonS3 Client { get; set; } @@ -92,7 +87,7 @@ private void Initialize() { RegionEndpoint = region }; - Endpoint = GetPropertyValue( STORAGE_ENDPOINT , ENDPOINT_DEPRECATED); + Endpoint = GetPropertyValue(STORAGE_ENDPOINT, ENDPOINT_DEPRECATED); if (Endpoint == STORAGE_CUSTOM_ENDPOINT_VALUE) { Endpoint = GetPropertyValue(STORAGE_CUSTOM_ENDPOINT, STORAGE_CUSTOM_ENDPOINT_DEPRECATED); @@ -122,8 +117,8 @@ private void Initialize() { } #endif - Bucket = GetEncryptedPropertyValue(BUCKET, BUCKET_DEPRECATED); - Folder = GetPropertyValue(FOLDER, FOLDER_DEPRECATED); + Bucket = GetEncryptedPropertyValue(BUCKET); + Folder = GetPropertyValue(FOLDER); Region = region.SystemName; SetURI(); @@ -260,7 +255,7 @@ public string Get(string objectName, GxFileType fileType, int urlMinutes = 0) private string GetUrlImpl(string objectName, GxFileType fileType, int urlMinutes = 0) { bool isPrivate = IsPrivateUpload(fileType); - return (isPrivate)? GetPreSignedUrl(objectName, ResolveExpirationMinutes(urlMinutes)): StorageUri + StorageUtils.EncodeUrl(objectName); + return (isPrivate)? GetPreSignedUrl(objectName, ResolveExpiration(urlMinutes).Minutes): StorageUri + StorageUtils.EncodeUrl(objectName); } diff --git a/dotnet/src/dotnetframework/Providers/Storage/GXAzureStorage/AzureStorageExternalProvider.cs b/dotnet/src/dotnetframework/Providers/Storage/GXAzureStorage/AzureStorageExternalProvider.cs index 5c5351db8..f1beae904 100644 --- a/dotnet/src/dotnetframework/Providers/Storage/GXAzureStorage/AzureStorageExternalProvider.cs +++ b/dotnet/src/dotnetframework/Providers/Storage/GXAzureStorage/AzureStorageExternalProvider.cs @@ -125,7 +125,7 @@ private string GetURL(CloudBlockBlob blob, GxFileType fileType, int urlMinutes = { SharedAccessBlobPolicy sasConstraints = new SharedAccessBlobPolicy(); sasConstraints.SharedAccessStartTime = DateTime.UtcNow; - sasConstraints.SharedAccessExpiryTime = DateTime.UtcNow.AddMinutes(ResolveExpirationMinutes(urlMinutes)); + sasConstraints.SharedAccessExpiryTime = DateTime.UtcNow.AddMinutes(ResolveExpiration(urlMinutes).Minutes); sasConstraints.Permissions = SharedAccessBlobPermissions.Read; url += blob.GetSharedAccessSignature(sasConstraints); } diff --git a/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs b/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs index e1fcdbdb1..ca19cd134 100644 --- a/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs +++ b/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs @@ -248,7 +248,7 @@ public string Get(string objectName, GxFileType fileType, int urlMinutes) private string GetURL(string objectName, GxFileType fileType, int urlMinutes = 0) { if (fileType.HasFlag(GxFileType.Private)) - return Signer.Sign(Bucket, StorageUtils.EncodeUrl(objectName), TimeSpan.FromMinutes(ResolveExpirationMinutes(urlMinutes)), HttpMethod.Get); + return Signer.Sign(Bucket, StorageUtils.EncodeUrl(objectName), TimeSpan.FromMinutes(ResolveExpiration(urlMinutes).Minutes), HttpMethod.Get); else { return StorageUri + StorageUtils.EncodeUrl(objectName); From fa818dd2a540aece3a02ef14d27655d893cca763 Mon Sep 17 00:00:00 2001 From: Gonzalo Date: Wed, 16 Jun 2021 10:40:42 -0300 Subject: [PATCH 19/42] Use TimeSpan instead of Minutes --- .../Providers/Storage/GXAmazonS3/ExternalProviderS3.cs | 4 ++-- .../Storage/GXAzureStorage/AzureStorageExternalProvider.cs | 2 +- .../Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs b/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs index 19e756900..58deb23a9 100644 --- a/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs +++ b/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs @@ -255,11 +255,11 @@ public string Get(string objectName, GxFileType fileType, int urlMinutes = 0) private string GetUrlImpl(string objectName, GxFileType fileType, int urlMinutes = 0) { bool isPrivate = IsPrivateUpload(fileType); - return (isPrivate)? GetPreSignedUrl(objectName, ResolveExpiration(urlMinutes).Minutes): StorageUri + StorageUtils.EncodeUrl(objectName); + return (isPrivate)? GetPreSignedUrl(objectName, ResolveExpiration(urlMinutes).TotalMinutes): StorageUri + StorageUtils.EncodeUrl(objectName); } - private string GetPreSignedUrl(string objectName, int urlMinutes) + private string GetPreSignedUrl(string objectName, double urlMinutes) { GetPreSignedUrlRequest request = new GetPreSignedUrlRequest { diff --git a/dotnet/src/dotnetframework/Providers/Storage/GXAzureStorage/AzureStorageExternalProvider.cs b/dotnet/src/dotnetframework/Providers/Storage/GXAzureStorage/AzureStorageExternalProvider.cs index f1beae904..4f84fdbeb 100644 --- a/dotnet/src/dotnetframework/Providers/Storage/GXAzureStorage/AzureStorageExternalProvider.cs +++ b/dotnet/src/dotnetframework/Providers/Storage/GXAzureStorage/AzureStorageExternalProvider.cs @@ -125,7 +125,7 @@ private string GetURL(CloudBlockBlob blob, GxFileType fileType, int urlMinutes = { SharedAccessBlobPolicy sasConstraints = new SharedAccessBlobPolicy(); sasConstraints.SharedAccessStartTime = DateTime.UtcNow; - sasConstraints.SharedAccessExpiryTime = DateTime.UtcNow.AddMinutes(ResolveExpiration(urlMinutes).Minutes); + sasConstraints.SharedAccessExpiryTime = DateTime.UtcNow.AddMinutes(ResolveExpiration(urlMinutes).TotalMinutes); sasConstraints.Permissions = SharedAccessBlobPermissions.Read; url += blob.GetSharedAccessSignature(sasConstraints); } diff --git a/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs b/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs index ca19cd134..94561466d 100644 --- a/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs +++ b/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs @@ -248,7 +248,7 @@ public string Get(string objectName, GxFileType fileType, int urlMinutes) private string GetURL(string objectName, GxFileType fileType, int urlMinutes = 0) { if (fileType.HasFlag(GxFileType.Private)) - return Signer.Sign(Bucket, StorageUtils.EncodeUrl(objectName), TimeSpan.FromMinutes(ResolveExpiration(urlMinutes).Minutes), HttpMethod.Get); + return Signer.Sign(Bucket, StorageUtils.EncodeUrl(objectName), ResolveExpiration(urlMinutes), HttpMethod.Get); else { return StorageUri + StorageUtils.EncodeUrl(objectName); From 9b07823a039072f8749758d65fdce08104a14665 Mon Sep 17 00:00:00 2001 From: Gonzalo Date: Wed, 16 Jun 2021 15:13:23 -0300 Subject: [PATCH 20/42] Do not require Folder Name --- .../GxClasses/Services/Storage/ExternalProviderBase.cs | 2 ++ .../Providers/Storage/GXAmazonS3/ExternalProviderS3.cs | 2 -- .../Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs | 2 -- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderBase.cs b/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderBase.cs index e6f3bbbea..7bafcdb04 100644 --- a/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderBase.cs +++ b/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderBase.cs @@ -19,6 +19,7 @@ public abstract class ExternalProviderBase protected GxFileType defaultAcl = GxFileType.Private; + protected string Folder { get; set; } public ExternalProviderBase() { @@ -64,6 +65,7 @@ private void Initialize() defaultExpiration = new TimeSpan(0, minutes, 0); } } + Folder = GetPropertyValue(FOLDER, null, string.Empty); } protected TimeSpan ResolveExpiration(int expirationMinutes) diff --git a/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs b/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs index 58deb23a9..1d781f428 100644 --- a/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs +++ b/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs @@ -43,7 +43,6 @@ public class ExternalProviderS3 : ExternalProviderBase, ExternalProvider IAmazonS3 Client { get; set; } string Bucket { get; set; } - string Folder { get; set; } string Endpoint { get; set; } string Region { get; set; } @@ -118,7 +117,6 @@ private void Initialize() { #endif Bucket = GetEncryptedPropertyValue(BUCKET); - Folder = GetPropertyValue(FOLDER); Region = region.SystemName; SetURI(); diff --git a/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs b/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs index 94561466d..a9a351f54 100644 --- a/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs +++ b/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs @@ -30,7 +30,6 @@ public class ExternalProviderGoogle : ExternalProviderBase, ExternalProvider StorageService Service { get; set; } String Project { get; set; } String Bucket { get; set; } - String Folder { get; set; } UrlSigner Signer { get; set; } @@ -80,7 +79,6 @@ private void Initialize() Bucket = GetEncryptedPropertyValue(BUCKET); Project = GetPropertyValue(PROJECT_ID); - Folder = GetPropertyValue(FOLDER); CreateBucket(); CreateFolder(Folder); From 69d95084e183b3917ac25d315e7788bb6f7e7a60 Mon Sep 17 00:00:00 2001 From: Gonzalo Date: Wed, 16 Jun 2021 17:02:51 -0300 Subject: [PATCH 21/42] Default expiration fix --- .../GxClasses/Services/Storage/ExternalProviderBase.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderBase.cs b/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderBase.cs index 7bafcdb04..78bd167ee 100644 --- a/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderBase.cs +++ b/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderBase.cs @@ -56,7 +56,7 @@ private void Initialize() this.defaultAcl = GxFileType.PublicRead; } - String expirationS = GetPropertyValue(DEFAULT_EXPIRATION, DEFAULT_EXPIRATION, defaultExpiration.ToString()); + String expirationS = GetPropertyValue(DEFAULT_EXPIRATION, DEFAULT_EXPIRATION, defaultExpiration.TotalMinutes.ToString()); if (!String.IsNullOrEmpty(expirationS)) { int minutes; From 0e630cffe2427a5271eab929f70bb9bfa559b7e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Echag=C3=BCe?= Date: Thu, 17 Jun 2021 09:11:11 -0300 Subject: [PATCH 22/42] Add an environment to the testing workflow --- .github/workflows/External-Storage-Tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/External-Storage-Tests.yml b/.github/workflows/External-Storage-Tests.yml index 0baca4659..609786786 100644 --- a/.github/workflows/External-Storage-Tests.yml +++ b/.github/workflows/External-Storage-Tests.yml @@ -20,6 +20,7 @@ jobs: SolutionFile: dotnet\DotNetStandardClasses.sln runs-on: windows-latest + environment: external-storage-tests steps: - name: Checkout From 2198d123c42680fadd2d0f0ec1af12afdb9a4536 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Echag=C3=BCe?= Date: Thu, 17 Jun 2021 09:22:28 -0300 Subject: [PATCH 23/42] Fix JSON quote --- .github/workflows/External-Storage-Tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/External-Storage-Tests.yml b/.github/workflows/External-Storage-Tests.yml index 609786786..c96b29906 100644 --- a/.github/workflows/External-Storage-Tests.yml +++ b/.github/workflows/External-Storage-Tests.yml @@ -90,7 +90,7 @@ jobs: $Env:STORAGE_AZUREBS_PUBLIC_CONTAINER_NAME="contluispublic" $Env:STORAGE_AZUREBS_PRIVATE_CONTAINER_NAME="contluisprivate" $Env:GOOGLECS_TEST_ENABLED="true" - $Env:STORAGE_GOOGLECS_KEY="${{ secrets.GOOGLECS_KEY }}" + $Env:STORAGE_GOOGLECS_KEY='${{ secrets.GOOGLECS_KEY }}' $Env:STORAGE_GOOGLECS_PROJECT_ID="gxjavacloudstorageunittests" $Env:STORAGE_GOOGLECS_BUCKET_NAME="javaclasses-unittests" $Env:STORAGE_GOOGLECS_FOLDER_NAME="gxclasses" From 6d175b9957c5e06e3aba675858773057cd9dd975 Mon Sep 17 00:00:00 2001 From: Gonzalo Date: Thu, 17 Jun 2021 09:53:16 -0300 Subject: [PATCH 24/42] Do not require S3 Endpoint. --- .../Storage/GXAmazonS3/ExternalProviderS3.cs | 5 ++++- .../ExternalProvider/ExternalProviderTest.cs | 15 ++++++++++----- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs b/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs index 1d781f428..cb0834192 100644 --- a/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs +++ b/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs @@ -26,6 +26,7 @@ public class ExternalProviderS3 : ExternalProviderBase, ExternalProvider const string REGION = "REGION"; const string STORAGE_CUSTOM_ENDPOINT_VALUE = "custom"; + const string DEFAULT_ENDPOINT = "s3.amazonaws.com"; const string DEFAULT_REGION = "us-east-1"; [Obsolete("Use Property ACCESS_KEY instead", false)] @@ -86,7 +87,7 @@ private void Initialize() { RegionEndpoint = region }; - Endpoint = GetPropertyValue(STORAGE_ENDPOINT, ENDPOINT_DEPRECATED); + Endpoint = GetPropertyValue(STORAGE_ENDPOINT, ENDPOINT_DEPRECATED, DEFAULT_ENDPOINT); if (Endpoint == STORAGE_CUSTOM_ENDPOINT_VALUE) { Endpoint = GetPropertyValue(STORAGE_CUSTOM_ENDPOINT, STORAGE_CUSTOM_ENDPOINT_DEPRECATED); @@ -350,7 +351,9 @@ public string Upload(string fileName, Stream stream, GxFileType destFileType) CannedACL = GetCannedACL(destFileType) }; if (Path.GetExtension(fileName).Equals(".tmp")) + { objectRequest.ContentType = "image/jpeg"; + } PutObjectResponse result = PutObject(objectRequest); return Get(fileName, destFileType); } diff --git a/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs b/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs index bbe764a12..00c102e2e 100644 --- a/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs +++ b/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs @@ -138,8 +138,9 @@ public void TestMultimediaUpload() String upload = provider.Get(TEST_SAMPLE_FILE_NAME, acl, 100); EnsureUrl(upload, acl); - provider.Delete(copyFileName, acl); - provider.Copy("text.txt", acl, copyFileName, acl); + DeleteSafe(copyFileName); + upload = provider.Copy(TEST_SAMPLE_FILE_NAME, copyFileName, "Table", "Field", acl); + provider.TryGetObjectNameFromURL(upload, out copyFileName); upload = TryGet(copyFileName, acl); EnsureUrl(upload, acl); } @@ -165,16 +166,20 @@ public void TestGetObjectName() [Fact] public void TestDownloadMethod() - { - String downloadPath = Path.Combine("resources", "test", TEST_SAMPLE_FILE_NAME); + { TestUploadPublicMethod(); + String downloadPath = Path.Combine("resources", "test", TEST_SAMPLE_FILE_NAME); try { File.Delete(downloadPath); } catch (Exception) { } - + try + { + Directory.CreateDirectory(Path.Combine("resources", "test")); + } + catch (Exception) { } provider.Download(TEST_SAMPLE_FILE_NAME, downloadPath, GxFileType.PublicRead); Assert.True(File.Exists(downloadPath)); } From 69a825853fe992a1710ff2e860360f17a0a1eb0a Mon Sep 17 00:00:00 2001 From: Gonzalo Date: Thu, 17 Jun 2021 13:48:53 -0300 Subject: [PATCH 25/42] Skip tests if Env Variables not set --- .../test/DotNetUnitTest/DotNetUnitTest.csproj | 2 +- .../ExternalProvider/ExternalProviderTest.cs | 77 +++++++++---------- 2 files changed, 39 insertions(+), 40 deletions(-) diff --git a/dotnet/test/DotNetUnitTest/DotNetUnitTest.csproj b/dotnet/test/DotNetUnitTest/DotNetUnitTest.csproj index 38acb52e1..e1e18d01d 100644 --- a/dotnet/test/DotNetUnitTest/DotNetUnitTest.csproj +++ b/dotnet/test/DotNetUnitTest/DotNetUnitTest.csproj @@ -26,11 +26,11 @@ - all runtime; build; native; contentfiles; analyzers; buildtransitive + diff --git a/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs b/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs index 00c102e2e..3dff9861d 100644 --- a/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs +++ b/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs @@ -16,7 +16,7 @@ public abstract class ExternalProviderTest private GeneXus.Services.ExternalProvider provider; private static String TEST_SAMPLE_FILE_NAME = "text.txt"; private static String TEST_SAMPLE_FILE_PATH = Path.Combine("resources", TEST_SAMPLE_FILE_NAME).ToString(CultureInfo.InvariantCulture); - + public ExternalProviderTest(string providerName, Type externalProviderType) { @@ -24,58 +24,58 @@ public ExternalProviderTest(string providerName, Type externalProviderType) bool testEnabled = Environment.GetEnvironmentVariable(providerName + "_TEST_ENABLED") == "true"; - Assume.True(testEnabled); - + Skip.IfNot(testEnabled, "Environment variables not set"); provider = (GeneXus.Services.ExternalProvider)Activator.CreateInstance(externalProviderType); + Skip.IfNot(testEnabled); Assert.NotNull(provider); } - [Fact] + [SkippableFact] public void TestUploadPublicMethod() - { + { String upload = provider.Upload(TEST_SAMPLE_FILE_PATH, TEST_SAMPLE_FILE_NAME, GxFileType.PublicRead); EnsureUrl(upload, GxFileType.PublicRead); } - [Fact] + [SkippableFact] public void TestUploadDefaultMethod() { String upload = provider.Upload(TEST_SAMPLE_FILE_PATH, TEST_SAMPLE_FILE_NAME, GxFileType.Default); EnsureUrl(upload, GxFileType.Default); } - [Fact] + [SkippableFact] public void TestUploadAndCopyDefault() { TestUploadAndCopyByAcl(GxFileType.Default, GxFileType.Default); } - [Fact] + [SkippableFact] public void TestUploadAndCopyPrivate() { TestUploadAndCopyByAcl(GxFileType.Private, GxFileType.Private); } - [Fact] + [SkippableFact] public void TestUploadAndCopyPublic() { TestUploadAndCopyByAcl(GxFileType.PublicRead, GxFileType.PublicRead); } - [Fact] + [SkippableFact] public void TestUploadAndCopyMixed() { TestUploadAndCopyByAcl(GxFileType.Default, GxFileType.Private); } - [Fact] + [SkippableFact] public void TestUploadAndCopyPrivateToPublic() { TestUploadAndCopyByAcl(GxFileType.Private, GxFileType.PublicRead); } - [Fact] + [SkippableFact] public void TestUploadAndCopyPublicToPrivate() { TestUploadAndCopyByAcl(GxFileType.PublicRead, GxFileType.Private); @@ -97,38 +97,21 @@ public void TestUploadAndCopyByAcl(GxFileType aclUpload, GxFileType aclCopy) EnsureUrl(upload, aclCopy); } - [Fact] + [SkippableFact] public void TestCopyMethod() { String copyFileName = "copy-text.txt"; Copy(copyFileName, GxFileType.PublicRead); } - [Fact] + [SkippableFact] public void TestCopyPrivateMethod() { String copyFileName = "copy-text-private.txt"; Copy(copyFileName, GxFileType.Private); - } - - private void Copy(String copyFileName, GxFileType acl) - { - provider.Upload(TEST_SAMPLE_FILE_PATH, TEST_SAMPLE_FILE_NAME, acl); - String upload = provider.Get(TEST_SAMPLE_FILE_NAME, acl, 100); - EnsureUrl(upload, acl); - - DeleteSafe(copyFileName); - Wait(1000); //Google CDN replication seems to be delayed. - - String urlCopy = TryGet(copyFileName, GxFileType.PublicRead); - Assert.False(UrlExists(urlCopy), "URL cannot exist: " + urlCopy); - - provider.Copy("text.txt", acl, copyFileName, acl); - upload = provider.Get(copyFileName, acl, 100); - EnsureUrl(upload, acl); - } + } - [Fact] + [SkippableFact] public void TestMultimediaUpload() { String copyFileName = "copy-text-private.txt"; @@ -145,7 +128,7 @@ public void TestMultimediaUpload() EnsureUrl(upload, acl); } - [Fact] + [SkippableFact] public void TestGetMethod() { TestUploadPublicMethod(); @@ -153,7 +136,7 @@ public void TestGetMethod() EnsureUrl(url, GxFileType.PublicRead); } - [Fact] + [SkippableFact] public void TestGetObjectName() { TestUploadPublicMethod(); @@ -164,7 +147,7 @@ public void TestGetObjectName() Assert.Equal("text.txt", objectName); } - [Fact] + [SkippableFact] public void TestDownloadMethod() { TestUploadPublicMethod(); @@ -184,7 +167,7 @@ public void TestDownloadMethod() Assert.True(File.Exists(downloadPath)); } - [Fact] + [SkippableFact] public void TestDeleteFile() { GxFileType acl = GxFileType.PublicRead; @@ -197,7 +180,7 @@ public void TestDeleteFile() Assert.False(UrlExists(url)); } - [Fact] + [SkippableFact] public void TestDeleteFilePrivate() { GxFileType acl = GxFileType.Private; @@ -210,7 +193,7 @@ public void TestDeleteFilePrivate() Assert.False(UrlExists(url)); } - [Fact] + [SkippableFact] public void TestUploadPrivateMethod() { GxFileType acl = GxFileType.Private; @@ -224,6 +207,22 @@ public void TestUploadPrivateMethod() } + private void Copy(String copyFileName, GxFileType acl) + { + provider.Upload(TEST_SAMPLE_FILE_PATH, TEST_SAMPLE_FILE_NAME, acl); + String upload = provider.Get(TEST_SAMPLE_FILE_NAME, acl, 100); + EnsureUrl(upload, acl); + + DeleteSafe(copyFileName); + Wait(1000); //Google CDN replication seems to be delayed. + + String urlCopy = TryGet(copyFileName, GxFileType.PublicRead); + Assert.False(UrlExists(urlCopy), "URL cannot exist: " + urlCopy); + + provider.Copy("text.txt", acl, copyFileName, acl); + upload = provider.Get(copyFileName, acl, 100); + EnsureUrl(upload, acl); + } private String TryGet(String objectName, GxFileType acl) { From 29a4690f620e990c87982d42bedbd38b3a706c82 Mon Sep 17 00:00:00 2001 From: Gonzalo Date: Fri, 18 Jun 2021 12:29:20 -0300 Subject: [PATCH 26/42] Change S3 Bucket Name --- .github/workflows/External-Storage-Tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/External-Storage-Tests.yml b/.github/workflows/External-Storage-Tests.yml index c96b29906..fdd2ecc3c 100644 --- a/.github/workflows/External-Storage-Tests.yml +++ b/.github/workflows/External-Storage-Tests.yml @@ -74,7 +74,7 @@ jobs: $Env:AWSS3_TEST_ENABLED="true" $Env:STORAGE_AWSS3_ACCESS_KEY="${{ secrets.AWSS3_ACCESS_KEY }}" $Env:STORAGE_AWSS3_SECRET_KEY="${{ secrets.AWSS3_SECRET_KEY }}" - $Env:STORAGE_AWSS3_BUCKET_NAME="genexuss3test" + $Env:STORAGE_AWSS3_BUCKET_NAME="genexus-s3-test" $Env:STORAGE_AWSS3_FOLDER_NAME="gxclasses" $Env:STORAGE_AWSS3_REGION="us-east-1" $Env:IBMCOS_TEST_ENABLED="true" From 77b46c89a984113d8999d01e6ca91d0550dd8274 Mon Sep 17 00:00:00 2001 From: Gonzalo Date: Fri, 18 Jun 2021 16:52:42 -0300 Subject: [PATCH 27/42] Fix error: The project exceeded the rate limit for bucket operations --- .../GXGoogleCloud/ExternalProviderGoogle.cs | 46 +++++++++++-------- 1 file changed, 27 insertions(+), 19 deletions(-) diff --git a/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs b/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs index a9a351f54..9aabf8b70 100644 --- a/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs +++ b/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs @@ -96,25 +96,33 @@ private Stream KeyStream(string key) private void CreateBucket() { - //objects in bucket are public by default - ObjectAccessControl defaultAccess = new ObjectAccessControl(); - defaultAccess.Entity = "allUsers"; - defaultAccess.Role = "READER"; - - Bucket bucket = new Bucket(); - bucket.Name = Bucket; - bucket.DefaultObjectAcl = new List { defaultAccess }; - - try - { - Client.CreateBucket(Project, bucket); - } - catch (GoogleApiException ex) - { - if (ex.Error.Code != BUCKET_EXISTS) - throw ex; - } - + try + { + //objects in bucket are public by default + ObjectAccessControl defaultAccess = new ObjectAccessControl(); + defaultAccess.Entity = "allUsers"; + defaultAccess.Role = "READER"; + + Bucket bucket = new Bucket(); + bucket.Name = Bucket; + bucket.DefaultObjectAcl = new List { defaultAccess }; + + if (Client.GetBucket(Bucket) == null) + { + try + { + Client.CreateBucket(Project, bucket); + } + catch (GoogleApiException ex) + { + if (ex.Error.Code != BUCKET_EXISTS) + throw ex; + } + } + } + catch (Exception) { + //We should not fail when Bucket cannot be created. Bucket should be created beforehand as Credentials may not allow Bucket creation. + } } private void CreateFolder(String name, string table = null, string field = null) From 65542b3d56b9a20091a2f9c3655e15a994e3af25 Mon Sep 17 00:00:00 2001 From: Gonzalo Date: Fri, 18 Jun 2021 17:22:25 -0300 Subject: [PATCH 28/42] Do not run Tests in parallel, as Google fails by rate limiter. --- .../ExternalProvider/ExternalProviderTest.cs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs b/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs index 3dff9861d..1b2862950 100644 --- a/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs +++ b/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs @@ -3,14 +3,14 @@ using System.IO; using System.Net; using DotNetUnitTest; -using GeneXus.Storage.GXAmazonS3; -using GeneXus.Storage.GXAzureStorage; -using GeneXus.Storage.GXGoogleCloud; +using GeneXus.Storage; using Xunit; + #pragma warning disable CA1031 // Do not catch general exception types namespace UnitTesting { + [Collection("Sequential")] public abstract class ExternalProviderTest { private GeneXus.Services.ExternalProvider provider; @@ -123,8 +123,12 @@ public void TestMultimediaUpload() DeleteSafe(copyFileName); upload = provider.Copy(TEST_SAMPLE_FILE_NAME, copyFileName, "Table", "Field", acl); - provider.TryGetObjectNameFromURL(upload, out copyFileName); - upload = TryGet(copyFileName, acl); + + copyFileName = StorageFactory.GetProviderObjectAbsoluteUriSafe(provider, upload); + if (!copyFileName.StartsWith("http", StringComparison.OrdinalIgnoreCase)) + { + upload = TryGet(copyFileName, acl); + } EnsureUrl(upload, acl); } From 8c1a5e9d37c64c02229b984bfd2f0d291b84a9de Mon Sep 17 00:00:00 2001 From: Gonzalo Date: Wed, 23 Jun 2021 17:12:28 -0300 Subject: [PATCH 29/42] #Fix: Multimedia files must be server with default access policy and not private. --- dotnet/src/dotnetframework/GxClasses/Core/GXUtilsCommon.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dotnet/src/dotnetframework/GxClasses/Core/GXUtilsCommon.cs b/dotnet/src/dotnetframework/GxClasses/Core/GXUtilsCommon.cs index 8671f055f..340ef6516 100644 --- a/dotnet/src/dotnetframework/GxClasses/Core/GXUtilsCommon.cs +++ b/dotnet/src/dotnetframework/GxClasses/Core/GXUtilsCommon.cs @@ -5264,7 +5264,7 @@ public static string ResolveUri(string uriString, bool absUrl, IGxContext contex string providerObjectName; if (PathUtil.IsAbsoluteUrl(uriString) && StorageFactory.TryGetProviderObjectName(ServiceFactory.GetExternalProvider(), uriString, out providerObjectName)) { - return new GxFile(string.Empty, providerObjectName).GetURI(); + return new GxFile(string.Empty, providerObjectName, GxFileType.Default).GetURI(); } if (schemeRegex.IsMatch(uriString)) From a0fb497b12b1977544693831d749659a51b39831 Mon Sep 17 00:00:00 2001 From: Gonzalo Date: Thu, 24 Jun 2021 17:51:02 -0300 Subject: [PATCH 30/42] &File support for External Provider URLs --- .../GxClasses/Domain/GXFileIO.cs | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/dotnet/src/dotnetframework/GxClasses/Domain/GXFileIO.cs b/dotnet/src/dotnetframework/GxClasses/Domain/GXFileIO.cs index 55901a29d..331f5d6b0 100644 --- a/dotnet/src/dotnetframework/GxClasses/Domain/GXFileIO.cs +++ b/dotnet/src/dotnetframework/GxClasses/Domain/GXFileIO.cs @@ -722,21 +722,26 @@ public string Source } else { + _file = null; if (GxUploadHelper.IsUpload(value)) { _uploadFileId = value; value = GxUploadHelper.UploadPath(value); + ExternalProvider provider = ServiceFactory.GetExternalProvider(); + _file = (provider != null)? new GxExternalFileInfo(value, String.Empty, provider): _file; } - - if (IsAbsoluteUrl(value)) - { - string objName = string.Empty; - _file = new GxExternalFileInfo(objName, value, ServiceFactory.GetExternalProvider()); - } - else + + if (_file == null) { - _file = new GxFileInfo(_baseDirectory); - _file.Source = value; + if (IsAbsoluteUrl(value)) + { + _file =new GxExternalFileInfo(value, value, ServiceFactory.GetExternalProvider()); + } + else + { + _file = new GxFileInfo(_baseDirectory); + _file.Source = value; + } } _lastError = 0; From 08c3ecfb2c7fe9ec00891aa802a039aefebf16f5 Mon Sep 17 00:00:00 2001 From: Gonzalo Date: Fri, 25 Jun 2021 12:09:05 -0300 Subject: [PATCH 31/42] StorageProvider: Folder name was not honored --- .../GxClasses/Core/GXApplication.cs | 2 +- .../GxClasses/Domain/GXFileIO.cs | 25 +++++++++++++------ .../GxClasses/Helpers/GXRestUtils.cs | 6 ++--- .../GxClasses/Middleware/GXHttpServices.cs | 16 ++++++------ .../Services/Storage/ExternalProviderBase.cs | 2 +- 5 files changed, 29 insertions(+), 22 deletions(-) diff --git a/dotnet/src/dotnetframework/GxClasses/Core/GXApplication.cs b/dotnet/src/dotnetframework/GxClasses/Core/GXApplication.cs index e90e307a3..61c6ee51d 100644 --- a/dotnet/src/dotnetframework/GxClasses/Core/GXApplication.cs +++ b/dotnet/src/dotnetframework/GxClasses/Core/GXApplication.cs @@ -692,7 +692,7 @@ public static bool GetHttpRequestPostedFile(IGxContext gxContext, string varName string fileGuid = GxUploadHelper.GetUploadFileGuid(); fileToken = GxUploadHelper.GetUploadFileId(fileGuid); - GxUploadHelper.CacheUploadFile(fileGuid, filePath, Path.GetFileName(pf.FileName), ext, file, gxContext); + GxUploadHelper.CacheUploadFile(fileGuid, Path.GetFileName(pf.FileName), ext, file, gxContext); return true; } diff --git a/dotnet/src/dotnetframework/GxClasses/Domain/GXFileIO.cs b/dotnet/src/dotnetframework/GxClasses/Domain/GXFileIO.cs index 331f5d6b0..795d10883 100644 --- a/dotnet/src/dotnetframework/GxClasses/Domain/GXFileIO.cs +++ b/dotnet/src/dotnetframework/GxClasses/Domain/GXFileIO.cs @@ -421,22 +421,31 @@ public GxExternalFileInfo(ExternalProvider provider) _url = ""; } - public GxExternalFileInfo(string storageObjectFullname, ExternalProvider provider, GxFileType fileType) + public GxExternalFileInfo(string objectPath, ExternalProvider provider, GxFileType fileType) { - storageObjectFullname = storageObjectFullname!=null ? storageObjectFullname.Replace('\\', '/') : storageObjectFullname; - _name = storageObjectFullname; + objectPath = !String.IsNullOrEmpty(objectPath) ? objectPath.Replace('\\', '/') : objectPath; _provider = provider; + _fileTypeAtt = fileType; + _name = objectPath; + Uri result; - if (Uri.TryCreate(storageObjectFullname, UriKind.Absolute, out result) && result.IsAbsoluteUri) + if (Uri.TryCreate(objectPath, UriKind.Absolute, out result) && result.IsAbsoluteUri) { - _url = storageObjectFullname; + _url = objectPath; + } + else { + string folderName = ((ExternalProviderBase)provider).Folder; + if (!string.IsNullOrEmpty(folderName) && !_name.StartsWith(folderName)) + { + _name = $"{folderName}{StorageUtils.DELIMITER}{_name}"; + _url = $"{provider.GetBaseURL()}{StorageUtils.DELIMITER}{_name}"; + } } - _fileTypeAtt = fileType; } - public GxExternalFileInfo(string storageObjectFullname, string url, ExternalProvider provider, GxFileType fileType = GxFileType.Private) + public GxExternalFileInfo(string objectPath, string url, ExternalProvider provider, GxFileType fileType = GxFileType.Private) { - _name = StorageFactory.GetProviderObjectAbsoluteUriSafe(provider, storageObjectFullname); + _name = StorageFactory.GetProviderObjectAbsoluteUriSafe(provider, objectPath); _provider = provider; _url = url; _fileTypeAtt = fileType; diff --git a/dotnet/src/dotnetframework/GxClasses/Helpers/GXRestUtils.cs b/dotnet/src/dotnetframework/GxClasses/Helpers/GXRestUtils.cs index 8bd57d8f9..b8ffa60af 100644 --- a/dotnet/src/dotnetframework/GxClasses/Helpers/GXRestUtils.cs +++ b/dotnet/src/dotnetframework/GxClasses/Helpers/GXRestUtils.cs @@ -29,10 +29,10 @@ internal static bool IsUploadURL(HttpContext httpContext) { return httpContext.Request.GetRawUrl().EndsWith(HttpHelper.GXOBJECT, StringComparison.OrdinalIgnoreCase); } - internal static void CacheUploadFile(string fileGuid, string savedFileName, string localfileName, string ext, GxFile file, IGxContext gxContext) + internal static void CacheUploadFile(string fileGuid, string realFileName, string realFileExtension, GxFile temporalFile, IGxContext gxContext) { - CacheAPI.FilesCache.Set(fileGuid, JSONHelper.Serialize(new UploadCachedFile() { path = savedFileName, fileExtension = ext, fileName = localfileName }), GxRestPrefix.UPLOAD_TIMEOUT); - GXFileWatcher.Instance.AddTemporaryFile(file, gxContext); + CacheAPI.FilesCache.Set(fileGuid, JSONHelper.Serialize(new UploadCachedFile() { path = temporalFile.GetAbsoluteName(), fileExtension = realFileExtension, fileName = realFileName }), GxRestPrefix.UPLOAD_TIMEOUT); + GXFileWatcher.Instance.AddTemporaryFile(temporalFile, gxContext); } internal static string GetUploadFileGuid() { diff --git a/dotnet/src/dotnetframework/GxClasses/Middleware/GXHttpServices.cs b/dotnet/src/dotnetframework/GxClasses/Middleware/GXHttpServices.cs index 1de795f0f..eab71fbb7 100644 --- a/dotnet/src/dotnetframework/GxClasses/Middleware/GXHttpServices.cs +++ b/dotnet/src/dotnetframework/GxClasses/Middleware/GXHttpServices.cs @@ -386,7 +386,7 @@ public override void webExecute() { try { - string savedFileName, ext, fName; + string ext, fName; if (context.isMultipartRequest()) { localHttpContext.Response.ContentType = MediaTypesNames.TextPlain; @@ -406,8 +406,7 @@ public override void webExecute() ext = FileUtil.GetFileType(fName); string tempDir = Preferences.getTMP_MEDIA_PATH(); - savedFileName = FileUtil.getTempFileName(tempDir); - GxFile gxFile = new GxFile(tempDir, savedFileName); + GxFile gxFile = new GxFile(tempDir, FileUtil.getTempFileName(tempDir)); gxFile.Create(hpf.InputStream); string uri = gxFile.GetURI(); @@ -423,7 +422,7 @@ public override void webExecute() thumbnailUrl = url, path = fileToken }); - GxUploadHelper.CacheUploadFile(fileGuid, savedFileName, Path.GetFileName(fName), ext, gxFile, context); + GxUploadHelper.CacheUploadFile(fileGuid, Path.GetFileName(fName), ext, gxFile, context); } UploadFilesResult result = new UploadFilesResult() { files = r }; var jsonObj = JSONHelper.Serialize(result); @@ -457,11 +456,10 @@ public override void webExecute() internal void WcfExecute(Stream istream, string contentType) { - string savedFileName, ext, fName; + string ext, fName; ext = context.ExtensionForContentType(contentType); - string tempDir = Preferences.getTMP_MEDIA_PATH(); - savedFileName = FileUtil.getTempFileName(tempDir); - GxFile file = new GxFile(tempDir, savedFileName); + string tempDir = Preferences.getTMP_MEDIA_PATH(); + GxFile file = new GxFile(tempDir, FileUtil.getTempFileName(tempDir)); file.Create(istream); JObject obj = new JObject(); @@ -475,7 +473,7 @@ internal void WcfExecute(Stream istream, string contentType) HttpHelper.SetResponseStatus(localHttpContext, ((int)HttpStatusCode.Created).ToString(), string.Empty); localHttpContext.Response.Write(obj.ToString()); - GxUploadHelper.CacheUploadFile(fileGuid, savedFileName, $"{Path.GetFileNameWithoutExtension(fName)}.{ext}", ext, file, context); + GxUploadHelper.CacheUploadFile(fileGuid, $"{Path.GetFileNameWithoutExtension(fName)}.{ext}", ext, file, context); } protected override bool IntegratedSecurityEnabled { diff --git a/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderBase.cs b/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderBase.cs index 78bd167ee..3df2785da 100644 --- a/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderBase.cs +++ b/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderBase.cs @@ -19,7 +19,7 @@ public abstract class ExternalProviderBase protected GxFileType defaultAcl = GxFileType.Private; - protected string Folder { get; set; } + public string Folder { get; set; } public ExternalProviderBase() { From 57403e25b061bb5aca31f2c805aa6d2102f3fed3 Mon Sep 17 00:00:00 2001 From: Gonzalo Date: Mon, 28 Jun 2021 18:33:47 -0300 Subject: [PATCH 32/42] FolderName duplicate --- dotnet/src/dotnetframework/GxClasses/Domain/GXFileIO.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dotnet/src/dotnetframework/GxClasses/Domain/GXFileIO.cs b/dotnet/src/dotnetframework/GxClasses/Domain/GXFileIO.cs index 795d10883..5bacf95c2 100644 --- a/dotnet/src/dotnetframework/GxClasses/Domain/GXFileIO.cs +++ b/dotnet/src/dotnetframework/GxClasses/Domain/GXFileIO.cs @@ -437,8 +437,8 @@ public GxExternalFileInfo(string objectPath, ExternalProvider provider, GxFileTy string folderName = ((ExternalProviderBase)provider).Folder; if (!string.IsNullOrEmpty(folderName) && !_name.StartsWith(folderName)) { + _url = $"{provider.GetBaseURL()}{_name}"; _name = $"{folderName}{StorageUtils.DELIMITER}{_name}"; - _url = $"{provider.GetBaseURL()}{StorageUtils.DELIMITER}{_name}"; } } } From fc59a9c9307c9476cb871b36af44d60702ef2736 Mon Sep 17 00:00:00 2001 From: Gonzalo Date: Tue, 29 Jun 2021 12:50:34 -0300 Subject: [PATCH 33/42] Azure baseURl should consider Private Container name instead of Public, as is the default. --- .../GXAzureStorage/AzureStorageExternalProvider.cs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/dotnet/src/dotnetframework/Providers/Storage/GXAzureStorage/AzureStorageExternalProvider.cs b/dotnet/src/dotnetframework/Providers/Storage/GXAzureStorage/AzureStorageExternalProvider.cs index 4f84fdbeb..56753fd82 100644 --- a/dotnet/src/dotnetframework/Providers/Storage/GXAzureStorage/AzureStorageExternalProvider.cs +++ b/dotnet/src/dotnetframework/Providers/Storage/GXAzureStorage/AzureStorageExternalProvider.cs @@ -360,13 +360,19 @@ public bool TryGetObjectNameFromURL(string url, out string objectName) objectName = url.Replace(baseUrl, string.Empty); return true; } + baseUrl = StorageUri + StorageUtils.DELIMITER + PrivateContainer.Name + StorageUtils.DELIMITER; + if (url.StartsWith(baseUrl)) + { + objectName = url.Replace(baseUrl, string.Empty); + return true; + } objectName = null; return false; } public string GetBaseURL() { - return StorageUri + StorageUtils.DELIMITER + PublicContainer.Name + StorageUtils.DELIMITER; + return StorageUri + StorageUtils.DELIMITER + PrivateContainer.Name + StorageUtils.DELIMITER; } } From 1f0360c2531685d2f44183304190abae898a2e22 Mon Sep 17 00:00:00 2001 From: Gonzalo Date: Tue, 29 Jun 2021 15:04:23 -0300 Subject: [PATCH 34/42] Fix Minio Signed Urls when https not supported --- .../Providers/Storage/GXAmazonS3/ExternalProviderS3.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs b/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs index cb0834192..8f3a4add6 100644 --- a/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs +++ b/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs @@ -266,6 +266,10 @@ private string GetPreSignedUrl(string objectName, double urlMinutes) Key = objectName, Expires = DateTime.Now.AddMinutes(urlMinutes) }; + if (customEndpoint && StorageUri.StartsWith("http://")) + { + request.Protocol = Protocol.HTTP; + } return Client.GetPreSignedURL(request); } From f69c2982318123129140ffbaddeb294c69d8cf0c Mon Sep 17 00:00:00 2001 From: Gonzalo Date: Tue, 29 Jun 2021 15:05:02 -0300 Subject: [PATCH 35/42] Fix Google Cloud Storage not returing valid signed urls in some scenarios --- .../Storage/GXGoogleCloud/ExternalProviderGoogle.cs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs b/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs index 9aabf8b70..7b6ea6b8a 100644 --- a/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs +++ b/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs @@ -157,7 +157,7 @@ public string Upload(string fileName, Stream stream, GxFileType fileType) obj.ContentType = MimeMapping.GetMimeMapping(fileName); Client.UploadObject(obj, stream, GetUploadOptions(fileType)); - return StorageUri + StorageUtils.EncodeUrl(fileName); + return GetURL(fileName, fileType); } private Dictionary CreateObjectMetadata(string tableName, string fieldName, string name) @@ -172,7 +172,7 @@ private Dictionary CreateObjectMetadata(string tableName, string public string Copy(string objectName, GxFileType sourceFileType, string newName, GxFileType targetFileType) { Client.CopyObject(Bucket, objectName, Bucket, newName, GetCopyOptions(targetFileType)); - return GetURL(objectName, targetFileType, 0); + return GetURL(objectName, targetFileType); } private static CopyObjectOptions GetCopyOptions(GxFileType fileType) @@ -447,6 +447,12 @@ public bool TryGetObjectNameFromURL(string url, out string objectName) objectName = url.Replace(baseUrl, string.Empty); return true; } + baseUrl = $"https://storage.googleapis.com/{Bucket}/"; + if (url.StartsWith(baseUrl)) + { + objectName = url.Replace(baseUrl, string.Empty); + return true; + } objectName = null; return false; } From 78e4ef5182147dcdbdc349e99072af2c64b69854 Mon Sep 17 00:00:00 2001 From: Gonzalo Date: Thu, 1 Jul 2021 13:01:33 -0300 Subject: [PATCH 36/42] Azure & Google Default ACL was not honored. --- .../Storage/GXAzureStorage/AzureStorageExternalProvider.cs | 6 +++++- .../Storage/GXGoogleCloud/ExternalProviderGoogle.cs | 5 +++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/dotnet/src/dotnetframework/Providers/Storage/GXAzureStorage/AzureStorageExternalProvider.cs b/dotnet/src/dotnetframework/Providers/Storage/GXAzureStorage/AzureStorageExternalProvider.cs index 56753fd82..a9ff43173 100644 --- a/dotnet/src/dotnetframework/Providers/Storage/GXAzureStorage/AzureStorageExternalProvider.cs +++ b/dotnet/src/dotnetframework/Providers/Storage/GXAzureStorage/AzureStorageExternalProvider.cs @@ -102,9 +102,13 @@ private CloudBlockBlob GetCloudBlockBlob(string externalFileName, GxFileType fil return blob; } - + private bool IsPrivateFile(GxFileType fileType) { + if (fileType == GxFileType.Default) + { + return this.defaultAcl == GxFileType.Private; + } return fileType.HasFlag(GxFileType.Private); } diff --git a/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs b/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs index 7b6ea6b8a..6254ab45f 100644 --- a/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs +++ b/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs @@ -202,10 +202,11 @@ public string Upload(string localFile, string objectName, GxFileType fileType) } } - private static UploadObjectOptions GetUploadOptions(GxFileType fileType) + private UploadObjectOptions GetUploadOptions(GxFileType fileType) { UploadObjectOptions options = new UploadObjectOptions(); - if (fileType.HasFlag(GxFileType.Private)) + + if (fileType.HasFlag(GxFileType.Private) || fileType == GxFileType.Default && this.defaultAcl == GxFileType.Private) options.PredefinedAcl = PredefinedObjectAcl.ProjectPrivate; else options.PredefinedAcl = PredefinedObjectAcl.PublicRead; From 2295530c08dbac0fe1dfcd4b725bb93d2fe887f6 Mon Sep 17 00:00:00 2001 From: Gonzalo Date: Thu, 1 Jul 2021 16:45:39 -0300 Subject: [PATCH 37/42] Restore Concept of Attributes in GXFileType in order to differentiate Multimedia, Blobs (Attributes) from &Files, External paths. When we use Multimedia, Blobs, it's OK to use the Storage Folder Content Property automatically. But when using programatic access (File, StroagePAI), the User does not expect that the Folder Storage Content is appended automatically. --- .../GxClasses/Core/GXApplication.cs | 2 +- .../GxClasses/Core/GXUtilsCommon.cs | 10 +++---- .../GxClasses/Data/GXDataCommon.cs | 2 +- .../GxClasses/Data/GXDataNTierADO.cs | 22 +++++++-------- .../GxClasses/Domain/GXFileIO.cs | 15 +++++------ .../GxClasses/Helpers/HttpHelper.cs | 2 +- .../GxClasses/Middleware/GXHttpServices.cs | 4 +-- .../Services/Storage/ExternalProviderBase.cs | 2 +- .../GxClasses/Services/Storage/GXServices.cs | 1 + .../Storage/GXAmazonS3/ExternalProviderS3.cs | 20 +++++++------- .../AzureStorageExternalProvider.cs | 18 ++++++++++--- .../GXGoogleCloud/ExternalProviderGoogle.cs | 27 ++++++++++++++++--- .../GXOpenStack/ExternalProviderOpenStack.cs | 5 ++++ 13 files changed, 84 insertions(+), 46 deletions(-) diff --git a/dotnet/src/dotnetframework/GxClasses/Core/GXApplication.cs b/dotnet/src/dotnetframework/GxClasses/Core/GXApplication.cs index 61c6ee51d..2cd62a6f1 100644 --- a/dotnet/src/dotnetframework/GxClasses/Core/GXApplication.cs +++ b/dotnet/src/dotnetframework/GxClasses/Core/GXApplication.cs @@ -687,7 +687,7 @@ public static bool GetHttpRequestPostedFile(IGxContext gxContext, string varName ext = ext.TrimStart('.'); string filePath = FileUtil.getTempFileName(tempDir); GXLogging.Debug(log, "cgiGet(" + varName + "), fileName:" + filePath); - GxFile file = new GxFile(tempDir, filePath, GxFileType.Private); + GxFile file = new GxFile(tempDir, filePath, GxFileType.PrivateAttribute); filePath = file.Create(pf.InputStream); string fileGuid = GxUploadHelper.GetUploadFileGuid(); fileToken = GxUploadHelper.GetUploadFileId(fileGuid); diff --git a/dotnet/src/dotnetframework/GxClasses/Core/GXUtilsCommon.cs b/dotnet/src/dotnetframework/GxClasses/Core/GXUtilsCommon.cs index 340ef6516..9b8e42068 100644 --- a/dotnet/src/dotnetframework/GxClasses/Core/GXUtilsCommon.cs +++ b/dotnet/src/dotnetframework/GxClasses/Core/GXUtilsCommon.cs @@ -3411,7 +3411,7 @@ public static string GetFileType(string FileName) if (GxUploadHelper.IsUpload(FileName)) { - return new GxFile(string.Empty, FileName, GxFileType.Private).GetExtension(); + return new GxFile(string.Empty, FileName, GxFileType.PrivateAttribute).GetExtension(); } string extension = string.Empty; @@ -3456,7 +3456,7 @@ public static string GetFileName(string FileName) if (GxUploadHelper.IsUpload(FileName)) { - FileName = new GxFile(string.Empty, FileName, GxFileType.Private).GetName(); + FileName = new GxFile(string.Empty, FileName, GxFileType.PrivateAttribute).GetName(); } try { @@ -5264,7 +5264,7 @@ public static string ResolveUri(string uriString, bool absUrl, IGxContext contex string providerObjectName; if (PathUtil.IsAbsoluteUrl(uriString) && StorageFactory.TryGetProviderObjectName(ServiceFactory.GetExternalProvider(), uriString, out providerObjectName)) { - return new GxFile(string.Empty, providerObjectName, GxFileType.Default).GetURI(); + return new GxFile(string.Empty, providerObjectName, GxFileType.DefaultAttribute).GetURI(); } if (schemeRegex.IsMatch(uriString)) @@ -5274,7 +5274,7 @@ public static string ResolveUri(string uriString, bool absUrl, IGxContext contex string basePath = Path.Combine(Path.Combine(Preferences.getBLOB_PATH(), MultimediaDirectory)); try { - GxFile file = new GxFile(string.Empty, PathUtil.SafeCombine(basePath, fileName)); + GxFile file = new GxFile(string.Empty, PathUtil.SafeCombine(basePath, fileName), GxFileType.PrivateAttribute); return PathToUrl(file.GetURI(), absUrl, context); } catch (ArgumentException ex) @@ -5327,7 +5327,7 @@ public static string GetUriFromFile(string name, string type, string path) { if (GxUploadHelper.IsUpload(path)) { - return new GxFile(string.Empty, path, GxFileType.Private).GetName(); + return new GxFile(string.Empty, path, GxFileType.PrivateAttribute).GetName(); } string fromPathType = Path.GetExtension(path); if (!String.IsNullOrEmpty(fromPathType) && fromPathType != "tmp") diff --git a/dotnet/src/dotnetframework/GxClasses/Data/GXDataCommon.cs b/dotnet/src/dotnetframework/GxClasses/Data/GXDataCommon.cs index dae28864a..122c48a7b 100644 --- a/dotnet/src/dotnetframework/GxClasses/Data/GXDataCommon.cs +++ b/dotnet/src/dotnetframework/GxClasses/Data/GXDataCommon.cs @@ -903,7 +903,7 @@ protected static byte[] GetBinary(string fileNameParm, bool dbBlob) { if (ServiceFactory.GetExternalProvider() != null) { - GxFile file = new GxFile(string.Empty, fileNameParm, GxFileType.Private); + GxFile file = new GxFile(string.Empty, fileNameParm, GxFileType.PrivateAttribute); if (file.Exists()) { binary = file.ToByteArray(); diff --git a/dotnet/src/dotnetframework/GxClasses/Data/GXDataNTierADO.cs b/dotnet/src/dotnetframework/GxClasses/Data/GXDataNTierADO.cs index 8aa134c91..eab6eebd3 100644 --- a/dotnet/src/dotnetframework/GxClasses/Data/GXDataNTierADO.cs +++ b/dotnet/src/dotnetframework/GxClasses/Data/GXDataNTierADO.cs @@ -126,7 +126,7 @@ private string getBLOBFile(int id, string extension, string name, string fileNam } } if (temporary) - GXFileWatcher.Instance.AddTemporaryFile(new GxFile(_gxDbCommand.Conn.BlobPath, new GxFileInfo(fileName, _gxDbCommand.Conn.BlobPath), GxFileType.Private), _gxDbCommand.Conn.DataStore.Context); + GXFileWatcher.Instance.AddTemporaryFile(new GxFile(_gxDbCommand.Conn.BlobPath, new GxFileInfo(fileName, _gxDbCommand.Conn.BlobPath), GxFileType.PrivateAttribute), _gxDbCommand.Conn.DataStore.Context); fileName = new FileInfo(fileName).FullName; } catch (IOException e) @@ -148,7 +148,7 @@ public string getMultimediaFile(int id, string gxdbFileUri) string filePath = PathUtil.SafeCombine(_gxDbCommand.Conn.MultimediaPath, fileName); try { - GxFile file = new GxFile(string.Empty, filePath, GxFileType.Default); + GxFile file = new GxFile(string.Empty, filePath, GxFileType.DefaultAttribute); if (file.Exists()) { return filePath; @@ -336,7 +336,7 @@ public string getBLOBFile(int id, string extension, string name) return value; } - private string getBLOBFile(int id, string extension, string name, string fileName, bool temporary, GxFileType fileType = GxFileType.Private) + private string getBLOBFile(int id, string extension, string name, string fileName, bool temporary, GxFileType fileType = GxFileType.PrivateAttribute) { GxFile file = null; Stream fs = null; @@ -344,7 +344,7 @@ private string getBLOBFile(int id, string extension, string name, string fileNam int bufferSize = 4096; byte[] outbyte = new byte[bufferSize]; long retval; - long startIndex = 0; + long startIndex; bool streamClosed = false; try { @@ -432,7 +432,7 @@ public string getMultimediaFile(int id, string gxdbFileUri) try { - GxFile file = new GxFile(string.Empty, filePath, GxFileType.Default); + GxFile file = new GxFile(string.Empty, filePath, GxFileType.DefaultAttribute); if (file.Exists()) { @@ -440,7 +440,7 @@ public string getMultimediaFile(int id, string gxdbFileUri) } else { - return getBLOBFile(id, FileUtil.GetFileType(gxdbFileUri), FileUtil.GetFileName(gxdbFileUri), filePath, false, GxFileType.Default); + return getBLOBFile(id, FileUtil.GetFileType(gxdbFileUri), FileUtil.GetFileName(gxdbFileUri), filePath, false, GxFileType.DefaultAttribute); } } catch (ArgumentException) @@ -616,7 +616,7 @@ public void SetParameterMultimedia(int id, string image_gxi, string image, strin { try { - multimediaUri = ServiceFactory.GetExternalProvider().Copy(image_gxi, GXDbFile.GenerateUri(image_gxi, !GXDbFile.HasToken(image_gxi), false), tableName, fieldName, GxFileType.Default); + multimediaUri = ServiceFactory.GetExternalProvider().Copy(image_gxi, GXDbFile.GenerateUri(image_gxi, !GXDbFile.HasToken(image_gxi), false), tableName, fieldName, GxFileType.DefaultAttribute); GXLogging.Debug(log, "Copy file already in ExternalProvider:", multimediaUri); } catch (Exception ex) @@ -634,7 +634,7 @@ public void SetParameterMultimedia(int id, string image_gxi, string image, strin using (var fileStream = new MemoryStream(new WebClient().DownloadData(image_gxi))) { //Cannot pass Http Stream directly, because some Providers (AWS S3) does not support Http Stream. - multimediaUri = ServiceFactory.GetExternalProvider().Save(fileStream, GXDbFile.GenerateUri(image_gxi, !GXDbFile.HasToken(image_gxi), false), tableName, fieldName, GxFileType.Default); + multimediaUri = ServiceFactory.GetExternalProvider().Save(fileStream, GXDbFile.GenerateUri(image_gxi, !GXDbFile.HasToken(image_gxi), false), tableName, fieldName, GxFileType.DefaultAttribute); GXLogging.Debug(log, "Upload external file to ExternalProvider:", multimediaUri); } #pragma warning disable SYSLIB0014 // WebClient @@ -656,7 +656,7 @@ public void SetParameterMultimedia(int id, string image_gxi, string image, strin String fileName = PathUtil.GetValidFileName(fileFullName, "_"); using (fileStream) { - multimediaUri = ServiceFactory.GetExternalProvider().Save(fileStream, GXDbFile.GenerateUri(fileName, !GXDbFile.HasToken(fileName), false), tableName, fieldName, GxFileType.Default); + multimediaUri = ServiceFactory.GetExternalProvider().Save(fileStream, GXDbFile.GenerateUri(fileName, !GXDbFile.HasToken(fileName), false), tableName, fieldName, GxFileType.DefaultAttribute); GXLogging.Debug(log, "Upload file (_gxi) to ExternalProvider:", multimediaUri); } } @@ -664,7 +664,7 @@ public void SetParameterMultimedia(int id, string image_gxi, string image, strin { try { - multimediaUri = ServiceFactory.GetExternalProvider().Copy(image, GXDbFile.GenerateUri(image_gxi, !GXDbFile.HasToken(image_gxi), false), tableName, fieldName, GxFileType.Default); + multimediaUri = ServiceFactory.GetExternalProvider().Copy(image, GXDbFile.GenerateUri(image_gxi, !GXDbFile.HasToken(image_gxi), false), tableName, fieldName, GxFileType.DefaultAttribute); GXLogging.Debug(log, "Copy external file in ExternalProvider:", multimediaUri); } catch(Exception e) @@ -725,7 +725,7 @@ private static string PushToExternalProvider(Stream fileStream, string externalF string multimediaUri; using (fileStream) { - multimediaUri = ServiceFactory.GetExternalProvider().Save(fileStream, externalFileName, tableName, fieldName, GxFileType.Default); + multimediaUri = ServiceFactory.GetExternalProvider().Save(fileStream, externalFileName, tableName, fieldName, GxFileType.DefaultAttribute); GXLogging.Debug(log, "Upload file to ExternalProvider:", multimediaUri); } diff --git a/dotnet/src/dotnetframework/GxClasses/Domain/GXFileIO.cs b/dotnet/src/dotnetframework/GxClasses/Domain/GXFileIO.cs index 5bacf95c2..b7f0c1d79 100644 --- a/dotnet/src/dotnetframework/GxClasses/Domain/GXFileIO.cs +++ b/dotnet/src/dotnetframework/GxClasses/Domain/GXFileIO.cs @@ -435,7 +435,7 @@ public GxExternalFileInfo(string objectPath, ExternalProvider provider, GxFileTy } else { string folderName = ((ExternalProviderBase)provider).Folder; - if (!string.IsNullOrEmpty(folderName) && !_name.StartsWith(folderName)) + if (!string.IsNullOrEmpty(folderName) && fileType.HasFlag(GxFileType.Attribute) && !_name.StartsWith(folderName)) { _url = $"{provider.GetBaseURL()}{_name}"; _name = $"{folderName}{StorageUtils.DELIMITER}{_name}"; @@ -576,12 +576,8 @@ public string AbsolutePath private string URL { - get { - if (string.IsNullOrEmpty(_url)) - { - _url = _provider.Get(_name, _fileTypeAtt, 0); - } - return _url; + get { + return _provider.GetUrl(_name, _fileTypeAtt, 0); } } @@ -642,7 +638,10 @@ public enum GxFileType { Default = 0, PublicRead = 1, - Private = 2 + Private = 2, + Attribute = 8, + DefaultAttribute = Attribute | Default, + PrivateAttribute = Attribute | Private } public class GxFile diff --git a/dotnet/src/dotnetframework/GxClasses/Helpers/HttpHelper.cs b/dotnet/src/dotnetframework/GxClasses/Helpers/HttpHelper.cs index e9e89a113..9a7948910 100644 --- a/dotnet/src/dotnetframework/GxClasses/Helpers/HttpHelper.cs +++ b/dotnet/src/dotnetframework/GxClasses/Helpers/HttpHelper.cs @@ -235,7 +235,7 @@ public static bool GetHttpRequestPostedFile(IGxContext gxContext, string varName ext = ext.TrimStart('.'); filePath = FileUtil.getTempFileName(tempDir); GXLogging.Debug(log, "cgiGet(" + varName + "), fileName:" + filePath); - GxFile file = new GxFile(tempDir, filePath); + GxFile file = new GxFile(tempDir, filePath, GxFileType.PrivateAttribute); #if NETCORE filePath = file.Create(pf.OpenReadStream()); #else diff --git a/dotnet/src/dotnetframework/GxClasses/Middleware/GXHttpServices.cs b/dotnet/src/dotnetframework/GxClasses/Middleware/GXHttpServices.cs index eab71fbb7..ada323861 100644 --- a/dotnet/src/dotnetframework/GxClasses/Middleware/GXHttpServices.cs +++ b/dotnet/src/dotnetframework/GxClasses/Middleware/GXHttpServices.cs @@ -406,7 +406,7 @@ public override void webExecute() ext = FileUtil.GetFileType(fName); string tempDir = Preferences.getTMP_MEDIA_PATH(); - GxFile gxFile = new GxFile(tempDir, FileUtil.getTempFileName(tempDir)); + GxFile gxFile = new GxFile(tempDir, FileUtil.getTempFileName(tempDir), GxFileType.PrivateAttribute); gxFile.Create(hpf.InputStream); string uri = gxFile.GetURI(); @@ -459,7 +459,7 @@ internal void WcfExecute(Stream istream, string contentType) string ext, fName; ext = context.ExtensionForContentType(contentType); string tempDir = Preferences.getTMP_MEDIA_PATH(); - GxFile file = new GxFile(tempDir, FileUtil.getTempFileName(tempDir)); + GxFile file = new GxFile(tempDir, FileUtil.getTempFileName(tempDir), GxFileType.PrivateAttribute); file.Create(istream); JObject obj = new JObject(); diff --git a/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderBase.cs b/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderBase.cs index 3df2785da..3cd3ff76f 100644 --- a/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderBase.cs +++ b/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderBase.cs @@ -51,7 +51,7 @@ private void Initialize() { GxFileType.TryParse(aclS, out this.defaultAcl); } - if (this.defaultAcl == GxFileType.Default) + if (this.defaultAcl.HasFlag(GxFileType.Default)) { this.defaultAcl = GxFileType.PublicRead; } diff --git a/dotnet/src/dotnetframework/GxClasses/Services/Storage/GXServices.cs b/dotnet/src/dotnetframework/GxClasses/Services/Storage/GXServices.cs index f78e0a202..525d95e55 100644 --- a/dotnet/src/dotnetframework/GxClasses/Services/Storage/GXServices.cs +++ b/dotnet/src/dotnetframework/GxClasses/Services/Storage/GXServices.cs @@ -231,6 +231,7 @@ public interface ExternalProvider string Upload(string localFile, string objectName, GxFileType fileType); void Download(string objectName, string localFile, GxFileType fileType); string Get(string objectName, GxFileType fileType, int urlMinutes); + string GetUrl(string objectName, GxFileType fileType, int urlMinutes); void Delete(string objectName, GxFileType fileType); bool Exists(string objectName, GxFileType fileType); string Rename(string objectName, string newName, GxFileType fileType); diff --git a/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs b/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs index 8f3a4add6..215b81f2b 100644 --- a/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs +++ b/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs @@ -251,6 +251,11 @@ public string Get(string objectName, GxFileType fileType, int urlMinutes = 0) return string.Empty; } + public string GetUrl(string objectName, GxFileType fileType, int urlMinutes = 0) + { + return GetUrlImpl(objectName, fileType, urlMinutes); + } + private string GetUrlImpl(string objectName, GxFileType fileType, int urlMinutes = 0) { bool isPrivate = IsPrivateUpload(fileType); @@ -332,17 +337,14 @@ public string Copy(string objectName, GxFileType sourceFileType, string newName, private S3CannedACL GetCannedACL(GxFileType acl) { - if (acl == GxFileType.Default) - { - acl = this.defaultAcl; + if (acl.HasFlag(GxFileType.Private)) { + return S3CannedACL.Private; } - - S3CannedACL accessControl = S3CannedACL.Private; - if (acl == GxFileType.PublicRead) + if (acl.HasFlag(GxFileType.PublicRead)) { - accessControl = S3CannedACL.PublicRead; - } - return accessControl; + return S3CannedACL.PublicRead; + } + return (this.defaultAcl == GxFileType.PublicRead) ? S3CannedACL.PublicRead : S3CannedACL.Private; } public string Upload(string fileName, Stream stream, GxFileType destFileType) diff --git a/dotnet/src/dotnetframework/Providers/Storage/GXAzureStorage/AzureStorageExternalProvider.cs b/dotnet/src/dotnetframework/Providers/Storage/GXAzureStorage/AzureStorageExternalProvider.cs index a9ff43173..4d4397522 100644 --- a/dotnet/src/dotnetframework/Providers/Storage/GXAzureStorage/AzureStorageExternalProvider.cs +++ b/dotnet/src/dotnetframework/Providers/Storage/GXAzureStorage/AzureStorageExternalProvider.cs @@ -104,12 +104,16 @@ private CloudBlockBlob GetCloudBlockBlob(string externalFileName, GxFileType fil } private bool IsPrivateFile(GxFileType fileType) - { - if (fileType == GxFileType.Default) + { + if (fileType.HasFlag(GxFileType.Private)) + { + return true; + } + if (fileType.HasFlag(GxFileType.PublicRead)) { - return this.defaultAcl == GxFileType.Private; + return false; } - return fileType.HasFlag(GxFileType.Private); + return (this.defaultAcl == GxFileType.PublicRead) ? false : true; } public string Get(string objectName, GxFileType fileType, int urlMinutes) @@ -122,6 +126,12 @@ public string Get(string objectName, GxFileType fileType, int urlMinutes) return string.Empty; } + public string GetUrl(string objectName, GxFileType fileType, int urlMinutes = 0) + { + CloudBlockBlob blob = GetCloudBlockBlob(objectName, fileType); + return GetURL(blob, fileType, urlMinutes); + } + private string GetURL(CloudBlockBlob blob, GxFileType fileType, int urlMinutes = 0) { string url = StorageUri + StorageUtils.DELIMITER + blob.Container.Name + StorageUtils.DELIMITER + StorageUtils.EncodeUrl(blob.Name); diff --git a/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs b/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs index 6254ab45f..758bd2f37 100644 --- a/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs +++ b/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs @@ -206,7 +206,7 @@ private UploadObjectOptions GetUploadOptions(GxFileType fileType) { UploadObjectOptions options = new UploadObjectOptions(); - if (fileType.HasFlag(GxFileType.Private) || fileType == GxFileType.Default && this.defaultAcl == GxFileType.Private) + if (fileType.HasFlag(GxFileType.Private) || fileType.HasFlag(GxFileType.Default) && this.defaultAcl == GxFileType.Private) options.PredefinedAcl = PredefinedObjectAcl.ProjectPrivate; else options.PredefinedAcl = PredefinedObjectAcl.PublicRead; @@ -252,9 +252,27 @@ public string Get(string objectName, GxFileType fileType, int urlMinutes) return string.Empty; } - private string GetURL(string objectName, GxFileType fileType, int urlMinutes = 0) + public string GetUrl(string objectName, GxFileType fileType, int urlMinutes) + { + return GetURL(objectName, fileType, urlMinutes); + } + + private bool IsPrivateResource(GxFileType fileType) { if (fileType.HasFlag(GxFileType.Private)) + { + return true; + } + if (fileType.HasFlag(GxFileType.PublicRead)) + { + return false; + } + return (this.defaultAcl == GxFileType.PublicRead) ? false : true; + } + + private string GetURL(string objectName, GxFileType fileType, int urlMinutes = 0) + { + if (IsPrivateResource(fileType)) return Signer.Sign(Bucket, StorageUtils.EncodeUrl(objectName), ResolveExpiration(urlMinutes), HttpMethod.Get); else { @@ -287,7 +305,10 @@ public DateTime GetLastModified(string objectName, GxFileType fileType) public long GetLength(string objectName, GxFileType fileType) { - return (long)Client.GetObject(Bucket, objectName).Size.GetValueOrDefault(); + System.Diagnostics.Debug.Assert(false); + var obj = Client.GetObject(Bucket, objectName); + return (long)obj.Size.GetValueOrDefault(); + } public void CreateDirectory(string directoryName) diff --git a/dotnet/src/dotnetframework/Providers/Storage/GXOpenStack/ExternalProviderOpenStack.cs b/dotnet/src/dotnetframework/Providers/Storage/GXOpenStack/ExternalProviderOpenStack.cs index bd9a94348..d85e22127 100644 --- a/dotnet/src/dotnetframework/Providers/Storage/GXOpenStack/ExternalProviderOpenStack.cs +++ b/dotnet/src/dotnetframework/Providers/Storage/GXOpenStack/ExternalProviderOpenStack.cs @@ -151,6 +151,11 @@ public string Get(string externalFileName, GxFileType fileType, int urlMinutes) return GetURL(externalFileName, fileType); } + public string GetUrl(string externalFileName, GxFileType fileType, int urlMinutes) + { + return GetURL(externalFileName, fileType); + } + public void Delete(string objectName, GxFileType fileType) { openstackFilesProvider.DeleteObject(GetBucket(fileType), objectName); From 71cd95a4644f448e640876d59204f97eb1fcd198 Mon Sep 17 00:00:00 2001 From: Gonzalo Date: Thu, 1 Jul 2021 17:08:53 -0300 Subject: [PATCH 38/42] Try fix Google Cloud Storage Bug. https://github.com/googleapis/google-cloud-dotnet/pull/3677 --- .../DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs b/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs index 1b2862950..c90103116 100644 --- a/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs +++ b/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs @@ -233,6 +233,10 @@ private String TryGet(String objectName, GxFileType acl) String getValue = ""; try { + if (this is ExternalProviderGoogleTest) + { + objectName = objectName.Replace("%2F", "/"); //Google Cloud Storage Bug. https://github.com/googleapis/google-cloud-dotnet/pull/3677 + } getValue = provider.Get(objectName, acl, 5); } catch (Exception) From 03b99c256266489cd8aee523e489d3e66f02f21a Mon Sep 17 00:00:00 2001 From: Gonzalo Date: Fri, 2 Jul 2021 09:53:32 -0300 Subject: [PATCH 39/42] #Fix: ContentType was forced to image/jpeg for every Multimedia and Blob Upload. --- .../Services/Storage/ExternalProviderBase.cs | 32 ++++++++++++++++++- .../Storage/GXAmazonS3/ExternalProviderS3.cs | 19 ++++++++--- .../AzureStorageExternalProvider.cs | 15 ++++++--- .../GXGoogleCloud/ExternalProviderGoogle.cs | 24 ++++++++------ 4 files changed, 71 insertions(+), 19 deletions(-) diff --git a/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderBase.cs b/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderBase.cs index 3cd3ff76f..8d3644a5e 100644 --- a/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderBase.cs +++ b/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderBase.cs @@ -1,6 +1,12 @@ using System; +using System.IO; using GeneXus.Encryption; using log4net; +#if NETCORE +using GeneXus.Mime; +#else +using System.Web; +#endif namespace GeneXus.Services { @@ -16,7 +22,8 @@ public abstract class ExternalProviderBase protected static String DEFAULT_ACL_DEPRECATED = "STORAGE_PROVIDER_DEFAULT_ACL"; protected static String DEFAULT_EXPIRATION_DEPRECATED = "STORAGE_PROVIDER_DEFAULT_EXPIRATION"; protected TimeSpan defaultExpiration = new TimeSpan(24, 0, 0); - + protected static string DEFAULT_TMP_CONTENT_TYPE = "image/jpeg"; + protected static string DEFAULT_CONTENT_TYPE = "application/octet-stream"; protected GxFileType defaultAcl = GxFileType.Private; public string Folder { get; set; } @@ -143,5 +150,28 @@ protected String ResolvePropertyName(String propertyName) return $"STORAGE_{GetName()}_{propertyName}"; } + protected bool TryGetContentType(string fileName, out string mimeType, string defaultValue = null) + { + mimeType = defaultValue; + string extension = Path.GetExtension(fileName); + if (!string.IsNullOrEmpty(extension)) + { + if (fileName.EndsWith(".tmp")) + { + mimeType = DEFAULT_TMP_CONTENT_TYPE; + } + else + { + try + { + mimeType = MimeMapping.GetMimeMapping(fileName); + } + catch (Exception) + { + } + } + } + return mimeType != null; + } } } diff --git a/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs b/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs index 215b81f2b..2458e3b8f 100644 --- a/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs +++ b/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs @@ -356,9 +356,8 @@ public string Upload(string fileName, Stream stream, GxFileType destFileType) InputStream = stream, CannedACL = GetCannedACL(destFileType) }; - if (Path.GetExtension(fileName).Equals(".tmp")) - { - objectRequest.ContentType = "image/jpeg"; + if (TryGetContentType(fileName, out string mimeType)) { + objectRequest.ContentType = mimeType; } PutObjectResponse result = PutObject(objectRequest); return Get(fileName, destFileType); @@ -378,8 +377,15 @@ public string Copy(string url, string newName, string tableName, string fieldNam SourceKey = url, DestinationBucket = Bucket, DestinationKey = resourceKey, - CannedACL = GetCannedACL(destFileType) + CannedACL = GetCannedACL(destFileType), + MetadataDirective = S3MetadataDirective.REPLACE }; + + if (TryGetContentType(newName, out string mimeType, DEFAULT_CONTENT_TYPE)) + { + request.ContentType = mimeType; + } + AddObjectMetadata(request.Metadata, tableName, fieldName, resourceKey); CopyObject(request); @@ -399,6 +405,11 @@ public string Save(Stream fileStream, string fileName, string tableName, string InputStream = fileStream, CannedACL = GetCannedACL(destFileType) }; + if (TryGetContentType(fileName, out string mimeType)) + { + objectRequest.ContentType = mimeType; + } + AddObjectMetadata(objectRequest.Metadata, tableName, fieldName, resourceKey); PutObjectResponse result = PutObject(objectRequest); diff --git a/dotnet/src/dotnetframework/Providers/Storage/GXAzureStorage/AzureStorageExternalProvider.cs b/dotnet/src/dotnetframework/Providers/Storage/GXAzureStorage/AzureStorageExternalProvider.cs index 4d4397522..b7ba7bd2e 100644 --- a/dotnet/src/dotnetframework/Providers/Storage/GXAzureStorage/AzureStorageExternalProvider.cs +++ b/dotnet/src/dotnetframework/Providers/Storage/GXAzureStorage/AzureStorageExternalProvider.cs @@ -175,10 +175,11 @@ public string Copy(string objectName, GxFileType sourceFileType, string newName, public string Upload(string fileName, Stream stream, GxFileType fileType) { CloudBlockBlob blob = GetCloudBlockBlob(fileName, fileType); - if (Path.GetExtension(fileName).Equals(".tmp")) - blob.Properties.ContentType = "image/jpeg"; - else - blob.Properties.ContentType = MimeMapping.GetMimeMapping(fileName); + + if (TryGetContentType(fileName, out string mimeType)) + { + blob.Properties.ContentType = mimeType; + } blob.UploadFromStreamAsync(stream).GetAwaiter().GetResult(); return GetURL(blob, fileType); @@ -194,8 +195,14 @@ public string Copy(string sourceUrl, string newName, string tableName, string fi targetBlob.Metadata["Table"] = tableName; targetBlob.Metadata["Field"] = fieldName; targetBlob.Metadata["KeyValue"] = StorageUtils.EncodeUrl(newName); + + if (TryGetContentType(newName, out string mimeType, DEFAULT_CONTENT_TYPE)) + { + targetBlob.Properties.ContentType = mimeType; + } targetBlob.StartCopyAsync(sourceBlob).GetAwaiter().GetResult(); + targetBlob.SetPropertiesAsync().GetAwaiter().GetResult(); //Required to apply new object metadata return GetURL(targetBlob, fileType); } return string.Empty; diff --git a/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs b/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs index 758bd2f37..043b45821 100644 --- a/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs +++ b/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs @@ -151,11 +151,11 @@ public string Upload(string fileName, Stream stream, GxFileType fileType) obj.Name = fileName; obj.Bucket = Bucket; - if (Path.GetExtension(fileName).Equals(".tmp")) - obj.ContentType = "image/jpeg"; - else - obj.ContentType = MimeMapping.GetMimeMapping(fileName); - + if (TryGetContentType(fileName, out string mimeType, DEFAULT_CONTENT_TYPE)) + { + obj.ContentType = mimeType; + } + Client.UploadObject(obj, stream, GetUploadOptions(fileType)); return GetURL(fileName, fileType); } @@ -197,7 +197,8 @@ public string Upload(string localFile, string objectName, GxFileType fileType) { using (FileStream stream = new FileStream(localFile, FileMode.Open)) { - Google.Apis.Storage.v1.Data.Object obj = Client.UploadObject(Bucket, objectName, "application/octet-stream", stream, GetUploadOptions(fileType)); + TryGetContentType(objectName, out string mimeType, DEFAULT_CONTENT_TYPE); + Google.Apis.Storage.v1.Data.Object obj = Client.UploadObject(Bucket, objectName, mimeType, stream, GetUploadOptions(fileType)); return GetURL(objectName, fileType); } } @@ -254,7 +255,7 @@ public string Get(string objectName, GxFileType fileType, int urlMinutes) public string GetUrl(string objectName, GxFileType fileType, int urlMinutes) { - return GetURL(objectName, fileType, urlMinutes); + return GetURL(objectName, fileType, urlMinutes); } private bool IsPrivateResource(GxFileType fileType) @@ -286,8 +287,13 @@ public string Copy(string url, string newName, string tableName, string fieldNam url = StorageUtils.DecodeUrl(url.Replace(StorageUri, string.Empty)); Google.Apis.Storage.v1.Data.Object obj = Client.CopyObject(Bucket, url, Bucket, newName, GetCopyOptions(fileType)); + obj.Metadata = CreateObjectMetadata(tableName, fieldName, newName); - Client.UpdateObject(obj); + if (TryGetContentType(newName, out string mimeType, DEFAULT_CONTENT_TYPE)) + { + obj.ContentType = mimeType; + } + Client.UpdateObject(obj); return GetURL(newName, fileType, 0); } @@ -305,10 +311,8 @@ public DateTime GetLastModified(string objectName, GxFileType fileType) public long GetLength(string objectName, GxFileType fileType) { - System.Diagnostics.Debug.Assert(false); var obj = Client.GetObject(Bucket, objectName); return (long)obj.Size.GetValueOrDefault(); - } public void CreateDirectory(string directoryName) From 580dfb5a1d14d28e7a976e0ba0c6bc71ed5c3bab Mon Sep 17 00:00:00 2001 From: Gonzalo Date: Fri, 2 Jul 2021 16:06:28 -0300 Subject: [PATCH 40/42] Disable temporally TestDeleteFile Private Test as is failing randomly on Google Cloud Storage --- .../DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs b/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs index c90103116..d51fcdf09 100644 --- a/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs +++ b/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs @@ -187,6 +187,8 @@ public void TestDeleteFile() [SkippableFact] public void TestDeleteFilePrivate() { + Skip.If(this is ExternalProviderGoogleTest, "This test is failing randomly for Google Cloud Storage."); + GxFileType acl = GxFileType.Private; TestUploadPrivateMethod(); String url = TryGet(TEST_SAMPLE_FILE_NAME, acl); From 3387637b1dbc1210db5ab010a3e73447f849660c Mon Sep 17 00:00:00 2001 From: Gonzalo Date: Mon, 5 Jul 2021 12:58:25 -0300 Subject: [PATCH 41/42] More private test cases for Providers S3, Azure, Google --- .../Services/Storage/ExternalProviderBase.cs | 6 +- .../Storage/GXAmazonS3/ExternalProviderS3.cs | 16 ++- .../AzureStorageExternalProvider.cs | 20 +++- .../GXGoogleCloud/ExternalProviderGoogle.cs | 45 +++++--- .../ExternalProviderAzurePrivateTest.cs | 14 +++ .../ExternalProviderGooglePrivateTest.cs | 15 +++ .../ExternalProviderS3PrivateTest.cs | 13 +++ .../ExternalProvider/ExternalProviderTest.cs | 109 ++++++++++++------ 8 files changed, 171 insertions(+), 67 deletions(-) create mode 100644 dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderAzurePrivateTest.cs create mode 100644 dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderGooglePrivateTest.cs create mode 100644 dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderS3PrivateTest.cs diff --git a/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderBase.cs b/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderBase.cs index 8d3644a5e..6db3409fb 100644 --- a/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderBase.cs +++ b/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderBase.cs @@ -56,11 +56,7 @@ private void Initialize() String aclS = GetPropertyValue(DEFAULT_ACL, DEFAULT_ACL_DEPRECATED, ""); if (!String.IsNullOrEmpty(aclS)) { - GxFileType.TryParse(aclS, out this.defaultAcl); - } - if (this.defaultAcl.HasFlag(GxFileType.Default)) - { - this.defaultAcl = GxFileType.PublicRead; + this.defaultAcl = aclS.Equals("Private") ? GxFileType.Private : GxFileType.PublicRead; } String expirationS = GetPropertyValue(DEFAULT_EXPIRATION, DEFAULT_EXPIRATION, defaultExpiration.TotalMinutes.ToString()); diff --git a/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs b/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs index 2458e3b8f..c7a26cd14 100644 --- a/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs +++ b/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs @@ -337,14 +337,22 @@ public string Copy(string objectName, GxFileType sourceFileType, string newName, private S3CannedACL GetCannedACL(GxFileType acl) { - if (acl.HasFlag(GxFileType.Private)) { + if (acl.HasFlag(GxFileType.Private)) + { return S3CannedACL.Private; } - if (acl.HasFlag(GxFileType.PublicRead)) + else if (acl.HasFlag(GxFileType.PublicRead)) { return S3CannedACL.PublicRead; - } - return (this.defaultAcl == GxFileType.PublicRead) ? S3CannedACL.PublicRead : S3CannedACL.Private; + } + else if (this.defaultAcl == GxFileType.Private) + { + return S3CannedACL.Private; + } + else + { + return S3CannedACL.PublicRead; + } } public string Upload(string fileName, Stream stream, GxFileType destFileType) diff --git a/dotnet/src/dotnetframework/Providers/Storage/GXAzureStorage/AzureStorageExternalProvider.cs b/dotnet/src/dotnetframework/Providers/Storage/GXAzureStorage/AzureStorageExternalProvider.cs index b7ba7bd2e..036ec89e0 100644 --- a/dotnet/src/dotnetframework/Providers/Storage/GXAzureStorage/AzureStorageExternalProvider.cs +++ b/dotnet/src/dotnetframework/Providers/Storage/GXAzureStorage/AzureStorageExternalProvider.cs @@ -91,7 +91,8 @@ public string Upload(string localFile, string externalFileName, GxFileType fileT private CloudBlockBlob GetCloudBlockBlob(string externalFileName, GxFileType fileType) { CloudBlockBlob blob; - if (fileType.HasFlag(GxFileType.Private)) + + if (IsPrivateFile(fileType)) { blob = PrivateContainer.GetBlockBlobReference(externalFileName); } @@ -103,17 +104,24 @@ private CloudBlockBlob GetCloudBlockBlob(string externalFileName, GxFileType fil return blob; } - private bool IsPrivateFile(GxFileType fileType) - { - if (fileType.HasFlag(GxFileType.Private)) + private bool IsPrivateFile(GxFileType acl) + { + if (acl.HasFlag(GxFileType.Private)) + { + return true; + } + else if (acl.HasFlag(GxFileType.PublicRead)) + { + return false; + } + else if (this.defaultAcl == GxFileType.Private) { return true; } - if (fileType.HasFlag(GxFileType.PublicRead)) + else { return false; } - return (this.defaultAcl == GxFileType.PublicRead) ? false : true; } public string Get(string objectName, GxFileType fileType, int urlMinutes) diff --git a/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs b/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs index 043b45821..a309a692e 100644 --- a/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs +++ b/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs @@ -175,14 +175,34 @@ public string Copy(string objectName, GxFileType sourceFileType, string newName, return GetURL(objectName, targetFileType); } - private static CopyObjectOptions GetCopyOptions(GxFileType fileType) + + + private PredefinedObjectAcl GetPredefinedAcl(GxFileType acl) { - CopyObjectOptions options = new CopyObjectOptions(); - if (fileType.HasFlag(GxFileType.Private)) - options.DestinationPredefinedAcl = PredefinedObjectAcl.ProjectPrivate; - else - options.DestinationPredefinedAcl = PredefinedObjectAcl.PublicRead; - return options; + PredefinedObjectAcl objectPredefinedObjectAcl = PredefinedObjectAcl.PublicRead; + if (acl.HasFlag(GxFileType.Private)) + { + objectPredefinedObjectAcl = PredefinedObjectAcl.ProjectPrivate; + } + else if (acl.HasFlag(GxFileType.PublicRead)) + { + objectPredefinedObjectAcl = PredefinedObjectAcl.PublicRead; + } + else if (this.defaultAcl == GxFileType.Private) + { + objectPredefinedObjectAcl = PredefinedObjectAcl.Private; + } + return objectPredefinedObjectAcl; + } + + private CopyObjectOptions GetCopyOptions(GxFileType acl) + { + return new CopyObjectOptions() { DestinationPredefinedAcl = GetPredefinedAcl(acl) }; + } + + private UploadObjectOptions GetUploadOptions(GxFileType acl) + { + return new UploadObjectOptions() { PredefinedAcl = GetPredefinedAcl(acl) }; } public void Delete(string objectName, GxFileType fileType) @@ -203,16 +223,7 @@ public string Upload(string localFile, string objectName, GxFileType fileType) } } - private UploadObjectOptions GetUploadOptions(GxFileType fileType) - { - UploadObjectOptions options = new UploadObjectOptions(); - - if (fileType.HasFlag(GxFileType.Private) || fileType.HasFlag(GxFileType.Default) && this.defaultAcl == GxFileType.Private) - options.PredefinedAcl = PredefinedObjectAcl.ProjectPrivate; - else - options.PredefinedAcl = PredefinedObjectAcl.PublicRead; - return options; - } + public void Download(string objectName, string localFile, GxFileType fileType) { diff --git a/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderAzurePrivateTest.cs b/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderAzurePrivateTest.cs new file mode 100644 index 000000000..7738d2b3a --- /dev/null +++ b/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderAzurePrivateTest.cs @@ -0,0 +1,14 @@ +using GeneXus.Storage.GXAmazonS3; +using GeneXus.Storage.GXAzureStorage; +using UnitTesting; + +namespace DotNetUnitTest +{ + public class ExternalProviderAzurePrivateTest : ExternalProviderTest + { + public ExternalProviderAzurePrivateTest(): base(AzureStorageExternalProvider.Name, typeof(AzureStorageExternalProvider), true) + { + } + + } +} diff --git a/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderGooglePrivateTest.cs b/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderGooglePrivateTest.cs new file mode 100644 index 000000000..263869594 --- /dev/null +++ b/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderGooglePrivateTest.cs @@ -0,0 +1,15 @@ +using GeneXus.Storage.GXAmazonS3; +using GeneXus.Storage.GXAzureStorage; +using GeneXus.Storage.GXGoogleCloud; +using UnitTesting; + +namespace DotNetUnitTest +{ + public class ExternalProviderGooglePrivateTest : ExternalProviderTest + { + public ExternalProviderGooglePrivateTest(): base(ExternalProviderGoogle.Name, typeof(ExternalProviderGoogle), true) + { + } + + } +} diff --git a/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderS3PrivateTest.cs b/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderS3PrivateTest.cs new file mode 100644 index 000000000..048cf4add --- /dev/null +++ b/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderS3PrivateTest.cs @@ -0,0 +1,13 @@ +using GeneXus.Storage.GXAmazonS3; +using UnitTesting; + +namespace DotNetUnitTest +{ + public class ExternalProviderS3PrivateTest : ExternalProviderTest + { + public ExternalProviderS3PrivateTest(): base(ExternalProviderS3.Name, typeof(ExternalProviderS3), true) + { + } + + } +} diff --git a/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs b/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs index d51fcdf09..bba36e918 100644 --- a/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs +++ b/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs @@ -3,6 +3,7 @@ using System.IO; using System.Net; using DotNetUnitTest; +using GeneXus.Services; using GeneXus.Storage; using Xunit; @@ -12,16 +13,17 @@ namespace UnitTesting { [Collection("Sequential")] public abstract class ExternalProviderTest - { + { private GeneXus.Services.ExternalProvider provider; - private static String TEST_SAMPLE_FILE_NAME = "text.txt"; - private static String TEST_SAMPLE_FILE_PATH = Path.Combine("resources", TEST_SAMPLE_FILE_NAME).ToString(CultureInfo.InvariantCulture); + private static String TEST_SAMPLE_FILE_NAME = $"text{new Random().Next(1, 100)}.txt"; + private static String TEST_SAMPLE_FILE_PATH = Path.Combine("resources", "text.txt").ToString(CultureInfo.InvariantCulture); + private bool defaultAclPrivate; - - public ExternalProviderTest(string providerName, Type externalProviderType) - { + public ExternalProviderTest(string providerName, Type externalProviderType, bool isPrivate) + { + defaultAclPrivate = isPrivate; + Environment.SetEnvironmentVariable($"STORAGE_{providerName}_DEFAULT_ACL", defaultAclPrivate ? GxFileType.Private.ToString() : GxFileType.PublicRead.ToString()); - bool testEnabled = Environment.GetEnvironmentVariable(providerName + "_TEST_ENABLED") == "true"; Skip.IfNot(testEnabled, "Environment variables not set"); @@ -31,13 +33,25 @@ public ExternalProviderTest(string providerName, Type externalProviderType) Assert.NotNull(provider); } + public ExternalProviderTest(string providerName, Type externalProviderType) : this(providerName, externalProviderType, false) + { + + } + [SkippableFact] public void TestUploadPublicMethod() - { + { String upload = provider.Upload(TEST_SAMPLE_FILE_PATH, TEST_SAMPLE_FILE_NAME, GxFileType.PublicRead); EnsureUrl(upload, GxFileType.PublicRead); } + [SkippableFact] + public void TestUploadPrivateSubfolderMethod() + { + String upload = provider.Upload(TEST_SAMPLE_FILE_PATH, $"folder/folder2/folder3/{TEST_SAMPLE_FILE_NAME}", GxFileType.Private); + EnsureUrl(upload, GxFileType.Private); + } + [SkippableFact] public void TestUploadDefaultMethod() { @@ -45,6 +59,13 @@ public void TestUploadDefaultMethod() EnsureUrl(upload, GxFileType.Default); } + [SkippableFact] + public void TestUploadDefaultAttributeMethod() + { + String upload = provider.Upload(TEST_SAMPLE_FILE_PATH, TEST_SAMPLE_FILE_NAME, GxFileType.DefaultAttribute); + EnsureUrl(upload, GxFileType.DefaultAttribute); + } + [SkippableFact] public void TestUploadAndCopyDefault() { @@ -83,17 +104,18 @@ public void TestUploadAndCopyPublicToPrivate() public void TestUploadAndCopyByAcl(GxFileType aclUpload, GxFileType aclCopy) { - String copyFileName = "test-upload-and-copy.txt"; + string copySourceName = $"test-source-upload-and-copy_{new Random().Next()}.txt"; + String copyTargetName = $"test-upload-and-copy_{new Random().Next()}.txt"; DeleteSafe(TEST_SAMPLE_FILE_PATH); - DeleteSafe(copyFileName); - String upload = provider.Upload(TEST_SAMPLE_FILE_PATH, TEST_SAMPLE_FILE_NAME, aclUpload); - Assert.True(UrlExists(upload), "Not found URL: " + upload); + DeleteSafe(copyTargetName); + String upload = provider.Upload(TEST_SAMPLE_FILE_PATH, copySourceName, aclUpload); + EnsureUrl(upload, aclUpload); - String copyUrl = TryGet(copyFileName, aclCopy); + String copyUrl = TryGet(copyTargetName, aclCopy); Assert.False(UrlExists(copyUrl), "URL cannot exist: " + copyUrl); - provider.Copy(TEST_SAMPLE_FILE_NAME, aclUpload, copyFileName, aclCopy); - upload = provider.Get(copyFileName, aclCopy, 100); + provider.Copy(copySourceName, aclUpload, copyTargetName, aclCopy); + upload = provider.Get(copyTargetName, aclCopy, 100); EnsureUrl(upload, aclCopy); } @@ -109,20 +131,21 @@ public void TestCopyPrivateMethod() { String copyFileName = "copy-text-private.txt"; Copy(copyFileName, GxFileType.Private); - } + } [SkippableFact] public void TestMultimediaUpload() { + string sourceFile = $"folder1/folder2/folder3{TEST_SAMPLE_FILE_NAME}"; String copyFileName = "copy-text-private.txt"; GxFileType acl = GxFileType.Private; - provider.Upload(TEST_SAMPLE_FILE_PATH, TEST_SAMPLE_FILE_NAME, acl); - String upload = provider.Get(TEST_SAMPLE_FILE_NAME, acl, 100); + provider.Upload(TEST_SAMPLE_FILE_PATH, sourceFile, acl); + String upload = provider.Get(sourceFile, acl, 100); EnsureUrl(upload, acl); DeleteSafe(copyFileName); - upload = provider.Copy(TEST_SAMPLE_FILE_NAME, copyFileName, "Table", "Field", acl); + upload = provider.Copy(sourceFile, copyFileName, "Table", "Field", acl); copyFileName = StorageFactory.GetProviderObjectAbsoluteUriSafe(provider, upload); if (!copyFileName.StartsWith("http", StringComparison.OrdinalIgnoreCase)) @@ -136,7 +159,7 @@ public void TestMultimediaUpload() public void TestGetMethod() { TestUploadPublicMethod(); - String url = provider.Get("text.txt", GxFileType.PublicRead, 10); + String url = provider.Get(TEST_SAMPLE_FILE_NAME, GxFileType.PublicRead, 10); EnsureUrl(url, GxFileType.PublicRead); } @@ -148,12 +171,12 @@ public void TestGetObjectName() Assert.True(UrlExists(url)); string objectName; provider.TryGetObjectNameFromURL(url, out objectName); - Assert.Equal("text.txt", objectName); + Assert.Equal(TEST_SAMPLE_FILE_NAME, objectName); } [SkippableFact] public void TestDownloadMethod() - { + { TestUploadPublicMethod(); String downloadPath = Path.Combine("resources", "test", TEST_SAMPLE_FILE_NAME); @@ -190,12 +213,9 @@ public void TestDeleteFilePrivate() Skip.If(this is ExternalProviderGoogleTest, "This test is failing randomly for Google Cloud Storage."); GxFileType acl = GxFileType.Private; - TestUploadPrivateMethod(); - String url = TryGet(TEST_SAMPLE_FILE_NAME, acl); - EnsureUrl(url, acl); + provider.Upload(TEST_SAMPLE_FILE_PATH, TEST_SAMPLE_FILE_NAME, acl); provider.Delete(TEST_SAMPLE_FILE_NAME, acl); - - url = TryGet(TEST_SAMPLE_FILE_NAME, acl); + String url = TryGet(TEST_SAMPLE_FILE_NAME, acl); Assert.False(UrlExists(url)); } @@ -225,7 +245,7 @@ private void Copy(String copyFileName, GxFileType acl) String urlCopy = TryGet(copyFileName, GxFileType.PublicRead); Assert.False(UrlExists(urlCopy), "URL cannot exist: " + urlCopy); - provider.Copy("text.txt", acl, copyFileName, acl); + provider.Copy(TEST_SAMPLE_FILE_NAME, acl, copyFileName, acl); upload = provider.Get(copyFileName, acl, 100); EnsureUrl(upload, acl); } @@ -235,7 +255,7 @@ private String TryGet(String objectName, GxFileType acl) String getValue = ""; try { - if (this is ExternalProviderGoogleTest) + if (((ExternalProviderBase)provider).GetName() == GeneXus.Storage.GXGoogleCloud.ExternalProviderGoogle.Name) { objectName = objectName.Replace("%2F", "/"); //Google Cloud Storage Bug. https://github.com/googleapis/google-cloud-dotnet/pull/3677 } @@ -282,18 +302,37 @@ private bool DeleteSafeImpl(String objectName, GxFileType acl) } private static void Wait(int milliseconds) - { + { System.Threading.Thread.Sleep(milliseconds); } - private static void EnsureUrl(String signedOrUnsignedUrl, GxFileType acl) + private void EnsureUrl(String signedOrUnsignedUrl, GxFileType acl) { - Assert.True(UrlExists(signedOrUnsignedUrl), "Resource not found: " + signedOrUnsignedUrl); - if (acl == GxFileType.Private) + Assert.True(UrlExists(signedOrUnsignedUrl), "URL not found: " + signedOrUnsignedUrl); + if (IsPrivateFile(acl)) { - String noSignedUrl = signedOrUnsignedUrl.Substring(0, signedOrUnsignedUrl.IndexOf('?') + 1); - Assert.False(UrlExists(noSignedUrl), "Resource must be private: " + noSignedUrl); + if (!(this is ExternalProviderMinioTest)) //Minio local installation not supported + { + String noSignedUrl = signedOrUnsignedUrl.Substring(0, signedOrUnsignedUrl.IndexOf('?') + 1); + Assert.False(UrlExists(noSignedUrl), "URL must be private: " + noSignedUrl); + } + } + else + { + Assert.False(signedOrUnsignedUrl.Contains("?"), "URL cannot be signed"); + } + } + private bool IsPrivateFile(GxFileType acl) + { + if (acl.HasFlag(GxFileType.Private)) + { + return true; + } + else if (acl.HasFlag(GxFileType.PublicRead)) + { + return false; } + return defaultAclPrivate; } #pragma warning disable CA1054 // Uri parameters should not be strings From d64487c920c5f17594f392fda107f20c9d232c59 Mon Sep 17 00:00:00 2001 From: Gonzalo Date: Tue, 6 Jul 2021 10:02:09 -0300 Subject: [PATCH 42/42] Add Oracle UnitTest and include WA for Google Cloud Storage Bug (https://github.com/googleapis/google-cloud-dotnet/pull/3677) --- .../GXGoogleCloud/ExternalProviderGoogle.cs | 10 ++-- .../ExternalProviderOracleTest.cs | 13 +++++ .../ExternalProvider/ExternalProviderTest.cs | 52 +++++++++++++++---- 3 files changed, 61 insertions(+), 14 deletions(-) create mode 100644 dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderOracleTest.cs diff --git a/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs b/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs index a309a692e..e0ab68018 100644 --- a/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs +++ b/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs @@ -202,15 +202,12 @@ private CopyObjectOptions GetCopyOptions(GxFileType acl) private UploadObjectOptions GetUploadOptions(GxFileType acl) { - return new UploadObjectOptions() { PredefinedAcl = GetPredefinedAcl(acl) }; + return new UploadObjectOptions() { PredefinedAcl = GetPredefinedAcl(acl) }; } public void Delete(string objectName, GxFileType fileType) { - Google.Apis.Storage.v1.Data.Object obj = new Google.Apis.Storage.v1.Data.Object(); - obj.Name = objectName; - obj.Bucket = Bucket; - Client.DeleteObject(obj); + Client.DeleteObject(Bucket, objectName); } public string Upload(string localFile, string objectName, GxFileType fileType) @@ -257,6 +254,7 @@ public bool Exists(string objectName, GxFileType fileType) public string Get(string objectName, GxFileType fileType, int urlMinutes) { + objectName = objectName.Replace("%2F", "/"); //Google Cloud Storage Bug. https://github.com/googleapis/google-cloud-dotnet/pull/3677 if (Exists(objectName, fileType)) { return GetURL(objectName, fileType, urlMinutes); @@ -482,12 +480,14 @@ public bool TryGetObjectNameFromURL(string url, out string objectName) if (url.StartsWith(baseUrl)) { objectName = url.Replace(baseUrl, string.Empty); + objectName = objectName.Replace("%2F", "/"); //Google Cloud Storage Bug. https://github.com/googleapis/google-cloud-dotnet/pull/3677 return true; } baseUrl = $"https://storage.googleapis.com/{Bucket}/"; if (url.StartsWith(baseUrl)) { objectName = url.Replace(baseUrl, string.Empty); + objectName = objectName.Replace("%2F", "/"); //Google Cloud Storage Bug. https://github.com/googleapis/google-cloud-dotnet/pull/3677 return true; } objectName = null; diff --git a/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderOracleTest.cs b/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderOracleTest.cs new file mode 100644 index 000000000..41e6ee3f0 --- /dev/null +++ b/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderOracleTest.cs @@ -0,0 +1,13 @@ +using GeneXus.Storage.GXAmazonS3; +using UnitTesting; + +namespace DotNetUnitTest +{ + public class ExternalProviderOracleTest : ExternalProviderTest + { + public ExternalProviderOracleTest(): base("ORACLE", typeof(ExternalProviderS3)) + { + } + + } +} diff --git a/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs b/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs index bba36e918..1dd2b991b 100644 --- a/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs +++ b/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Globalization; using System.IO; using System.Net; @@ -14,23 +15,26 @@ namespace UnitTesting [Collection("Sequential")] public abstract class ExternalProviderTest { + private GeneXus.Services.ExternalProvider provider; - private static String TEST_SAMPLE_FILE_NAME = $"text{new Random().Next(1, 100)}.txt"; - private static String TEST_SAMPLE_FILE_PATH = Path.Combine("resources", "text.txt").ToString(CultureInfo.InvariantCulture); + private String TEST_SAMPLE_FILE_NAME = $"text{new Random().Next(1, 10000)}.txt"; + private String TEST_SAMPLE_FILE_PATH; private bool defaultAclPrivate; public ExternalProviderTest(string providerName, Type externalProviderType, bool isPrivate) { defaultAclPrivate = isPrivate; Environment.SetEnvironmentVariable($"STORAGE_{providerName}_DEFAULT_ACL", defaultAclPrivate ? GxFileType.Private.ToString() : GxFileType.PublicRead.ToString()); - bool testEnabled = Environment.GetEnvironmentVariable(providerName + "_TEST_ENABLED") == "true"; + Skip.IfNot(testEnabled, "Environment variables not set"); provider = (GeneXus.Services.ExternalProvider)Activator.CreateInstance(externalProviderType); - - Skip.IfNot(testEnabled); + Assert.NotNull(provider); + + TEST_SAMPLE_FILE_PATH = Path.Combine("resources", TEST_SAMPLE_FILE_NAME).ToString(CultureInfo.InvariantCulture); + File.WriteAllText(TEST_SAMPLE_FILE_PATH, "This is a Sample Test from External Storage GeneXus .NET Generator Unit Tests"); } public ExternalProviderTest(string providerName, Type externalProviderType) : this(providerName, externalProviderType, false) @@ -207,11 +211,10 @@ public void TestDeleteFile() Assert.False(UrlExists(url)); } + [SkippableFact] public void TestDeleteFilePrivate() - { - Skip.If(this is ExternalProviderGoogleTest, "This test is failing randomly for Google Cloud Storage."); - + { GxFileType acl = GxFileType.Private; provider.Upload(TEST_SAMPLE_FILE_PATH, TEST_SAMPLE_FILE_NAME, acl); provider.Delete(TEST_SAMPLE_FILE_NAME, acl); @@ -233,6 +236,34 @@ public void TestUploadPrivateMethod() } + + [SkippableFact] + public void TestEmptyFolder() + { + GxFileType acl = GxFileType.PublicRead; + string folderName = $"folderTemp{new Random().Next(1, 100)}"; + + provider.DeleteDirectory(folderName); + + List urls = new List(); + + urls.Add(provider.Upload(TEST_SAMPLE_FILE_PATH, $"{folderName}/test1.png", acl)); + urls.Add(provider.Upload(TEST_SAMPLE_FILE_PATH, $"{folderName}/text2.txt", acl)); + urls.Add(provider.Upload(TEST_SAMPLE_FILE_PATH, $"{folderName}/text3.txt", acl)); + urls.Add(provider.Upload(TEST_SAMPLE_FILE_PATH, $"{folderName}/text4.txt", acl)); + urls.Add(provider.Upload(TEST_SAMPLE_FILE_PATH, $"{folderName}/test1.png", acl)); + + + var files = provider.GetFiles(folderName); + Assert.Equal(4, files.Count); + + provider.DeleteDirectory(folderName); + + files = provider.GetFiles(folderName); + Assert.Empty(files); + + } + private void Copy(String copyFileName, GxFileType acl) { provider.Upload(TEST_SAMPLE_FILE_PATH, TEST_SAMPLE_FILE_NAME, acl); @@ -240,7 +271,7 @@ private void Copy(String copyFileName, GxFileType acl) EnsureUrl(upload, acl); DeleteSafe(copyFileName); - Wait(1000); //Google CDN replication seems to be delayed. + Wait(500); //Google CDN replication seems to be delayed. String urlCopy = TryGet(copyFileName, GxFileType.PublicRead); Assert.False(UrlExists(urlCopy), "URL cannot exist: " + urlCopy); @@ -250,6 +281,8 @@ private void Copy(String copyFileName, GxFileType acl) EnsureUrl(upload, acl); } + + private String TryGet(String objectName, GxFileType acl) { String getValue = ""; @@ -313,6 +346,7 @@ private void EnsureUrl(String signedOrUnsignedUrl, GxFileType acl) { if (!(this is ExternalProviderMinioTest)) //Minio local installation not supported { + Skip.If(this is ExternalProviderMinioTest); String noSignedUrl = signedOrUnsignedUrl.Substring(0, signedOrUnsignedUrl.IndexOf('?') + 1); Assert.False(UrlExists(noSignedUrl), "URL must be private: " + noSignedUrl); }