From e948ad2c20910653e5810d4cb14b447ec65a45b6 Mon Sep 17 00:00:00 2001 From: Gonzalo Date: Thu, 18 Jun 2020 15:14:32 -0300 Subject: [PATCH 01/33] Remove IBM Bluemix as has been deprecated by IBM (cherry picked from commit 7b337d82b7fac94df59f2a5f144f9f58d2e94734) --- .../GXBluemix/ExternalProviderBluemix.cs | 1434 ----------------- .../Storage/GXBluemix/GXBluemix.csproj | 28 - .../Storage/GXBluemix/GlobalSuppressions.cs | 18 - .../GXBluemix/Properties/AssemblyInfo.cs | 27 - .../Providers/Storage/GXBluemix/app.config | 15 - 5 files changed, 1522 deletions(-) delete mode 100644 dotnet/src/dotnetframework/Providers/Storage/GXBluemix/ExternalProviderBluemix.cs delete mode 100644 dotnet/src/dotnetframework/Providers/Storage/GXBluemix/GXBluemix.csproj delete mode 100644 dotnet/src/dotnetframework/Providers/Storage/GXBluemix/GlobalSuppressions.cs delete mode 100644 dotnet/src/dotnetframework/Providers/Storage/GXBluemix/Properties/AssemblyInfo.cs delete mode 100644 dotnet/src/dotnetframework/Providers/Storage/GXBluemix/app.config diff --git a/dotnet/src/dotnetframework/Providers/Storage/GXBluemix/ExternalProviderBluemix.cs b/dotnet/src/dotnetframework/Providers/Storage/GXBluemix/ExternalProviderBluemix.cs deleted file mode 100644 index 058182ba4..000000000 --- a/dotnet/src/dotnetframework/Providers/Storage/GXBluemix/ExternalProviderBluemix.cs +++ /dev/null @@ -1,1434 +0,0 @@ -using GeneXus.Configuration; -using GeneXus.Services; -using JSIStudios.SimpleRESTServices.Client; -using log4net; -using net.openstack.Core; -using net.openstack.Core.Caching; -using net.openstack.Core.Domain; -using net.openstack.Core.Domain.Mapping; -using net.openstack.Core.Exceptions; -using net.openstack.Core.Exceptions.Response; -using net.openstack.Core.Providers; -using net.openstack.Core.Validators; -using net.openstack.Providers.Rackspace; -using net.openstack.Providers.Rackspace.Exceptions; -using net.openstack.Providers.Rackspace.Objects; -using net.openstack.Providers.Rackspace.Validators; -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; -using OpenStack.Authentication; -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Net; -using System.Security.Cryptography; -using System.Text; -using Thread = System.Threading.Thread; -using GeneXus.Utils; -using GeneXus.Encryption; - -namespace GeneXus.Storage.GXBluemix -{ - public class ExternalProviderBluemix : ExternalProvider - { - const string USER = "STORAGE_PROVIDER_USER"; - const string PASSWORD = "STORAGE_PROVIDER_PASSWORD"; - const string SERVER_URL = "SERVER_URL"; - const string PROJECT_ID = "PROJECT_ID"; - const string REGION = "STORAGE_PROVIDER_REGION"; - const string PRIVATE_BUCKET = "PRIVATE_BUCKET_NAME"; - const string PUBLIC_BUCKET = "PUBLIC_BUCKET_NAME"; - const string FOLDER = "FOLDER_NAME"; - - BluemixFilesProvider Client { get; set; } - string PublicBucket { get; set; } - string PrivateBucket { get; set; } - string Folder { get; set; } - string AuthToken { get; set; } - public string StorageUri { get; set; } - string PrivateTempKeyUrl { get; set; } - - public ExternalProviderBluemix() - : this(ServiceFactory.GetGXServices().Get(GXServices.STORAGE_SERVICE)) - { - } - - public ExternalProviderBluemix(GXService providerService) - { - var identityEndpoint = new Uri(providerService.Properties.Get(SERVER_URL)); - CloudIdentityWithProject identity = new CloudIdentityWithProject - { - Username = CryptoImpl.Decrypt(providerService.Properties.Get(USER)), - Password = CryptoImpl.Decrypt(providerService.Properties.Get(PASSWORD)), - ProjectName = providerService.Properties.Get(PROJECT_ID), - }; - - BlueMixIdentityProvider identityProvider = new BlueMixIdentityProvider(identityEndpoint, identity); - - GetStorageEndpoint(identityProvider, identity); - - Client = new BluemixFilesProvider(null, providerService.Properties.Get(REGION), identityProvider, null); - PublicBucket = CryptoImpl.Decrypt(providerService.Properties.Get(PUBLIC_BUCKET)); - PrivateBucket = CryptoImpl.Decrypt(providerService.Properties.Get(PRIVATE_BUCKET)); - Folder = providerService.Properties.Get(FOLDER); - AuthToken = identityProvider.AuthToken; - - CreateBuckets(); - CreateFolder(Folder); - } - - public ExternalProviderBluemix(string proj, string url, string user, string pass, string bucket, string region) - { - var identityEndpoint = new Uri(url); - CloudIdentityWithProject identity = new CloudIdentityWithProject - { - Username = user, - Password = pass, - ProjectName = proj, - }; - - BlueMixIdentityProvider identityProvider = new BlueMixIdentityProvider(identityEndpoint, identity); - - GetStorageEndpoint(identityProvider, identity); - - Client = new BluemixFilesProvider(null, region, identityProvider, null); - PublicBucket = bucket; - AuthToken = identityProvider.AuthToken; - - CreateBuckets(); - } - - private void GetStorageEndpoint(BlueMixIdentityProvider identityProvider, CloudIdentityWithProject identity) - { - UserAccessV3 user = identityProvider.GetUserAccess(identity); - var catalog = user.Catalog; - Endpoint objectStorageEndpoint = null; - foreach (Catalog service in catalog) - if (service.Type == "object-store") - if (service.Endpoints.Where(e => e.Region_id == "dallas").Any()) - objectStorageEndpoint = service.Endpoints.Where(e => e.Region_id == "dallas").First(); - - StorageUri = objectStorageEndpoint.Url; - - if (String.IsNullOrEmpty(StorageUri)) - throw new Exception("Can't find the object storage endpoint, please check the credentials in the storage configuration."); - } - - private Dictionary CreateObjectMetadata(string tableName, string fieldName, string key) - { - Dictionary metadata = new Dictionary(); - metadata.Add("Table", tableName); - metadata.Add("Field", fieldName); - metadata.Add("KeyValue", key); - return metadata; - } - - private void CreateBuckets() - { - Dictionary headers = new Dictionary(); - headers.Add("X-Auth-Token", AuthToken); - PrivateTempKeyUrl = Guid.NewGuid().ToString(); - headers.Add("X-Container-Meta-Temp-URL-Key", PrivateTempKeyUrl); - Client.CreateContainer(PrivateBucket, headers); - - headers = new Dictionary(); - headers.Add("X-Auth-Token", AuthToken); - headers.Add("X-Container-Read", ".r:*"); - Client.CreateContainer(PublicBucket, headers); - } - - private void CreateFolder(string folder, string table = null, string field = null) - { - string name = StorageUtils.NormalizeDirectoryName(folder); - if (table != null) - name += table + StorageUtils.DELIMITER; - if (field != null) - name += field + StorageUtils.DELIMITER; - using (var stream = new MemoryStream()) - { - Client.CreateObject(PublicBucket, stream, name, "application/directory"); - } - } - - public void Download(string externalFileName, string localFile, GxFileType fileType) - { - string bucket = GetBucket(fileType); - string localDirectory = Path.GetDirectoryName(localFile); - string localFileName = Path.GetFileName(localFile); - Client.GetObjectSaveToFile(bucket, localDirectory, externalFileName, localFileName); - } - - public string Upload(string localFile, string externalFileName, GxFileType fileType) - { - string bucket = GetBucket(fileType); - Client.CreateObjectFromFile(bucket, localFile, externalFileName); - return StorageUri + StorageUtils.DELIMITER + bucket + StorageUtils.DELIMITER + StorageUtils.EncodeUrl(externalFileName); - } - - private string GetBucket(GxFileType fileType) - { - return (fileType.HasFlag(GxFileType.Private)) ? PrivateBucket : PublicBucket; - } - - public string Get(string externalFileName, GxFileType fileType, int urlMinutes) - { - string bucket = GetBucket(fileType); - if (Exists(externalFileName, fileType)) - { - if (fileType.HasFlag(GxFileType.Private)) - return Client.CreateTemporaryPublicUri(HttpMethod.GET, bucket, externalFileName, PrivateTempKeyUrl, DateTimeOffset.Now.AddMinutes(urlMinutes)).ToString(); - return StorageUri + StorageUtils.DELIMITER + bucket + StorageUtils.DELIMITER + StorageUtils.EncodeUrl(externalFileName); - } - return string.Empty; - } - - public void Delete(string objectName, GxFileType fileType) - { - Client.DeleteObject(GetBucket(fileType), objectName); - } - - public bool Exists(string objectName, GxFileType fileType) - { - try - { - Client.GetObjectMetaData(GetBucket(fileType), objectName); - return true; - } - catch (ItemNotFoundException) - { - return false; - } - } - - public string Rename(string objectName, string newName, GxFileType fileType) - { - string bucket = GetBucket(fileType); - Copy(objectName, fileType, newName, fileType); - Delete(objectName, fileType); - return StorageUri + StorageUtils.DELIMITER + bucket + StorageUtils.DELIMITER + StorageUtils.EncodeUrl(newName); - } - - public string Copy(string objectName, GxFileType sourceFileType, string newName, GxFileType destFileType) - { - string sourceBucket = GetBucket(sourceFileType); - string destBucket = GetBucket(destFileType); - Client.CopyObject(sourceBucket, objectName, destBucket, newName); - return Get(objectName, destFileType, 0); - } - - public string Upload(string fileName, Stream stream, GxFileType fileType) - { - string bucket = GetBucket(fileType); - if (Path.GetExtension(fileName).Equals(".tmp")) - Client.CreateObject(bucket, stream, fileName, "image / jpeg"); - else - Client.CreateObject(bucket, stream, fileName); - - return Get(fileName, fileType, 0); - } - - public string Copy(string url, string newName, string tableName, string fieldName, GxFileType fileType) - { - string bucket = GetBucket(fileType); - string resourceKey = Folder + StorageUtils.DELIMITER + tableName + StorageUtils.DELIMITER + fieldName + StorageUtils.DELIMITER + newName; - CreateFolder(Folder, tableName, fieldName); - url = StorageUtils.DecodeUrl(url.Replace(StorageUri + StorageUtils.DELIMITER + bucket + StorageUtils.DELIMITER, "")); - - Copy(url, fileType, resourceKey, fileType); - Client.UpdateObjectMetadata(bucket, resourceKey, CreateObjectMetadata(tableName, fieldName, resourceKey)); - - return Get(resourceKey, fileType, 0); - } - - public Stream GetStream(string objectName, GxFileType fileType) - { - MemoryStream stream = new MemoryStream(); - Client.GetObject(GetBucket(fileType), objectName, stream); - return stream; - } - - public string GetDirectory(string directoryName) - { - directoryName = StorageUtils.NormalizeDirectoryName(directoryName); - if (ExistsDirectory(directoryName)) - return PublicBucket + StorageUtils.DELIMITER + directoryName; - else - return ""; - } - - public long GetLength(string objectName, GxFileType fileType) - { - foreach (ContainerObject obj in Client.ListObjects(GetBucket(fileType))) - if (obj.Name.Equals(objectName)) - return obj.Bytes; - return 0; - } - - public DateTime GetLastModified(string objectName, GxFileType fileType) - { - foreach (ContainerObject obj in Client.ListObjects(GetBucket(fileType))) - if (obj.Name.Equals(objectName)) - return obj.LastModified.UtcDateTime; - return new DateTime(); - } - - public void CreateDirectory(string directoryName) - { - CreateFolder(directoryName); - } - - public void DeleteDirectory(string directoryName) - { - List objs = new List(); - foreach (ContainerObject obj in Client.ListObjects(PublicBucket, prefix: StorageUtils.NormalizeDirectoryName(directoryName))) - { - objs.Add(obj.Name); - } - objs.Add(directoryName); - Client.DeleteObjects(PublicBucket, objs); - } - - public bool ExistsDirectory(string directoryName) - { - List directories = GetDirectories(); - return directories.Contains(directoryName) || directories.Contains(StorageUtils.NormalizeDirectoryName(directoryName)); - } - - public void RenameDirectory(string directoryName, string newDirectoryName) - { - directoryName = StorageUtils.NormalizeDirectoryName(directoryName); - newDirectoryName = StorageUtils.NormalizeDirectoryName(newDirectoryName); - - foreach (ContainerObject obj in Client.ListObjects(PublicBucket, prefix: directoryName)) - { - Client.CopyObject(PublicBucket, obj.Name, PublicBucket, obj.Name.Replace(directoryName, newDirectoryName)); - } - DeleteDirectory(directoryName); - } - - public List GetSubDirectories(string directoryName) - { - return GetDirectories(StorageUtils.NormalizeDirectoryName(directoryName)); - } - - private List GetDirectories(string directoryName = null) - { - List subdir = new List(); - foreach (ContainerObject obj in Client.ListObjects(PublicBucket, prefix: directoryName)) - { - if (directoryName == null) - { - string dir = ""; - string[] parts = obj.Name.Split(Convert.ToChar(StorageUtils.DELIMITER)); - for (int i = 0; i < parts.Length - 1; i++) - { - dir += parts[i] + StorageUtils.DELIMITER; - if (!subdir.Contains(dir)) - subdir.Add(dir); - } - } - else - { - string name = obj.Name.Replace(directoryName, ""); - int i = name.IndexOf(StorageUtils.DELIMITER); - if (i != -1) - { - name = name.Substring(0, i); - string dir = StorageUtils.NormalizeDirectoryName(directoryName + name); - if (!subdir.Contains(dir)) - subdir.Add(dir); - } - } - - } - if (directoryName != null) - { - subdir.Remove(directoryName); - } - return subdir; - } - - public List GetFiles(string directoryName, string filter = "") - { - directoryName = StorageUtils.NormalizeDirectoryName(directoryName); - List files = new List(); - foreach (ContainerObject obj in Client.ListObjects(PublicBucket, prefix: directoryName)) - { - if (IsFile(obj, directoryName) && (String.IsNullOrEmpty(filter) || obj.Name.Contains(filter))) - files.Add(obj.Name); - } - return files; - } - - private bool IsFile(ContainerObject obj, string directory = null) - { - char delimiter = Convert.ToChar(StorageUtils.DELIMITER); - if (directory == null) - return obj.Name.Split(delimiter).Length == 1; - else - return obj.Name.Replace(StorageUtils.NormalizeDirectoryName(directory), "").Split(delimiter).Length == 1 && !String.IsNullOrEmpty(obj.Name.Replace(StorageUtils.NormalizeDirectoryName(directory), "").Split(delimiter)[0]); - } - - public bool GetMessageFromException(Exception ex, SdtMessages_Message msg) - { - try - { - ResponseException rex = (ResponseException)ex; - msg.gxTpr_Id = rex.Response.Status; - return true; - } - catch (Exception) - { - return false; - } - } - - public string Save(Stream fileStream, string fileName, string tableName, string fieldName, GxFileType fileType) - { - CreateFolder(Folder, tableName, fieldName); - string resourceKey = Folder + StorageUtils.DELIMITER + tableName + StorageUtils.DELIMITER + fieldName + StorageUtils.DELIMITER + fileName; - return Upload(resourceKey, fileStream, fileType); - } - - public bool GetObjectNameFromURL(string url, out string objectName) - { - string baseUrl = StorageUri + StorageUtils.DELIMITER + PublicBucket + StorageUtils.DELIMITER; - if (url.StartsWith(baseUrl)) - { - objectName = url.Replace(baseUrl, string.Empty); - return true; - } - objectName = null; - return false; - } - public string GetBaseURL() - { - return StorageUri + StorageUtils.DELIMITER + PublicBucket + StorageUtils.DELIMITER + Folder + StorageUtils.DELIMITER; - } - - #region Extension of openstack.net (issue: https://github.com/openstacknetsdk/openstack.net/issues/503) to enable openstack Identity API v3: http://developer.openstack.org/api-ref-identity-v3.html - - public class IdentityTokenV3 : IdentityToken - { - public new string Id { get; set; } - } - - public class BlueMixIdentityProvider : CloudIdentityProvider - { - public string AuthToken { get; set; } - new ICache TokenCache { get; } - - public BlueMixIdentityProvider(Uri urlBase, CloudIdentity defaultIdentity) - : this(urlBase, defaultIdentity, null, null) - { - AuthToken = null; - } - - public BlueMixIdentityProvider(Uri urlBase, CloudIdentity defaultIdentity, IRestService restService, ICache tokenCache) - : base(defaultIdentity, restService, tokenCache, urlBase) - { - AuthToken = null; - if (urlBase == null) - throw new ArgumentNullException(nameof(urlBase)); - } - - public new IdentityToken GetToken(CloudIdentity identity, bool forceCacheRefresh = false) - { - CheckIdentity(identity); - - GetUserAccess(identity, forceCacheRefresh); - - IdentityTokenV3 token = new IdentityTokenV3(); - token.Id = AuthToken; - return token; - } - public new UserAccessV3 GetUserAccess(CloudIdentity identity, bool forceCacheRefresh = false) - { - identity = identity ?? DefaultIdentity; - - CloudIdentityWithProject identityWithProject = identity as CloudIdentityWithProject; - - if (string.IsNullOrEmpty(identityWithProject.Password)) - throw new NotSupportedException(string.Format("The {0} identity must specify a password.", typeof(CloudIdentityWithProject))); - if (!string.IsNullOrEmpty(identityWithProject.APIKey)) - throw new NotSupportedException(string.Format("The {0} identity does not support API key authentication.", typeof(CloudIdentityWithProject))); - - var auth = Authorization.CreateCredentials(identityWithProject); - var projectId = identityWithProject.ProjectId != null ? JToken.FromObject(identityWithProject.ProjectId) : string.Empty; - - var response = ExecuteRESTRequest(identity, new Uri(UrlBase, "/v3/auth/tokens"), HttpMethod.POST, auth, isTokenRequest: true); - if (response == null || response.Data == null) - return null; - - AuthToken = response.Headers[0].Value; - JToken userAccessObject = response.Data["token"]; - if (userAccessObject == null) - return null; - - UserAccessV3 access = userAccessObject.ToObject(); - if (access == null) - return null; - - string key = string.Format("{0}:{1}/{2}", UrlBase, identityWithProject.ProjectId, identityWithProject.Username); - - return access; - } - - protected override string LookupServiceTypeKey(IServiceType serviceType) - { - return serviceType.Type; - } - - } - - public class Authorization - { - //Generates the JSON with the credentials to authenticate to the Identity API v3 - public static JObject CreateCredentials(CloudIdentityWithProject identityWithProject) - { - return new JObject( - new JProperty("auth", new JObject( - new JProperty("identity", new JObject( - new JProperty("methods", new JArray("password")), - new JProperty("password", new JObject( - new JProperty("user", new JObject( - new JProperty("id", JValue.CreateString(identityWithProject.Username)), - new JProperty("password", JValue.CreateString(identityWithProject.Password)))))))), - new JProperty("scope", new JObject( - new JProperty("project", new JObject( - new JProperty("id", JValue.CreateString(identityWithProject.ProjectName))))))))); - } - } - - internal class EncodeDecodeProviderAux : IEncodeDecodeProvider - { - - private static readonly EncodeDecodeProviderAux _default = new EncodeDecodeProviderAux(); - - public static EncodeDecodeProviderAux Default - { - get - { - return _default; - } - } - - public string UrlEncode(string stringToEncode) - { - if (stringToEncode == null) - return null; - - return UriUtility.UriEncode(stringToEncode, UriPart.AnyUrl); - } - - public string UrlDecode(string stringToDecode) - { - if (stringToDecode == null) - return null; - - return UriUtility.UriDecode(stringToDecode); - } - } - - public class BluemixFilesProvider : CloudFilesProvider - { - private readonly IObjectStorageValidator _cloudFilesValidator; - private readonly IEncodeDecodeProvider _encodeDecodeProvider; - private readonly IObjectStorageMetadataProcessor _cloudFilesMetadataProcessor; - private readonly IStatusParser _statusParser; - private readonly IObjectMapper _bulkDeletionResultMapper; - public BluemixFilesProvider(CloudIdentity defaultIdentity, string defaultRegion, IIdentityProvider identityProvider, IRestService restService) - : base(defaultIdentity, defaultRegion, identityProvider, restService) - { - _cloudFilesValidator = CloudFilesValidator.Default; - _encodeDecodeProvider = EncodeDecodeProviderAux.Default; - _cloudFilesMetadataProcessor = CloudFilesMetadataProcessor.Default; - _statusParser = HttpStatusCodeParser.Default; - _bulkDeletionResultMapper = new BulkDeletionResultMapper(_statusParser); - } - - public new string GetServiceEndpointCloudFiles(CloudIdentity identity, string region = null, bool useInternalUrl = false) - { - string serviceType = "object-store"; - - if (serviceType == null) - throw new ArgumentNullException("serviceType"); - if (string.IsNullOrEmpty(serviceType)) - throw new ArgumentException("serviceType cannot be empty"); - CheckIdentity(identity); - - identity = GetDefaultIdentity(identity); - - var userAccess = ((BlueMixIdentityProvider)IdentityProvider).GetUserAccess(identity); - - if (userAccess == null || userAccess.Catalog == null) - throw new UserAuthenticationException("Unable to authenticate user and retrieve authorized service endpoints."); - - IEnumerable services = userAccess.Catalog.Where(sc => string.Equals(sc.Type, serviceType, StringComparison.OrdinalIgnoreCase)); - - IEnumerable> endpoints = - services.SelectMany(service => service.Endpoints.Select(endpoint => Tuple.Create(service, endpoint))); - - string effectiveRegion = region; - if (string.IsNullOrEmpty(effectiveRegion)) - { - if (!string.IsNullOrEmpty(DefaultRegion)) - effectiveRegion = DefaultRegion; - else if (!string.IsNullOrEmpty(userAccess.User.DefaultRegion)) - effectiveRegion = userAccess.User.DefaultRegion; - } - - IEnumerable> regionEndpoints = - endpoints.Where(i => string.Equals(i.Item2.Region ?? string.Empty, effectiveRegion ?? string.Empty, StringComparison.OrdinalIgnoreCase)); - - if (regionEndpoints.Any()) - endpoints = regionEndpoints; - else - endpoints = endpoints.Where(i => string.IsNullOrEmpty(i.Item2.Region)); - - if (effectiveRegion == null && !endpoints.Any()) - throw new NoDefaultRegionSetException("No region was provided, the service does not provide a region-independent endpoint, and there is no default region set for the user's account."); - - Tuple serviceEndpoint = endpoints.Where(e => e.Item2.Url.Contains("softlayer")).FirstOrDefault(); - if (serviceEndpoint == null) - throw new UserAuthorizationException("The user does not have access to the requested service or region."); - - return serviceEndpoint.Item2.Url; - } - public new ObjectStore CreateContainer(string container, Dictionary headers = null, string region = null, bool useInternalUrl = false, CloudIdentity identity = null) - { - if (container == null) - throw new ArgumentNullException(nameof(container)); - if (string.IsNullOrEmpty(container)) - throw new ArgumentException("container cannot be empty"); - CheckIdentity(identity); - - _cloudFilesValidator.ValidateContainerName(container); - var urlPath = new Uri(string.Format("{0}/{1}", GetServiceEndpointCloudFiles(identity, region, useInternalUrl), _encodeDecodeProvider.UrlEncode(container))); - - var response = ExecuteRESTRequest(identity, urlPath, HttpMethod.PUT, headers: headers); - - switch (response.StatusCode) - { - case HttpStatusCode.Created: - return ObjectStore.ContainerCreated; - - case HttpStatusCode.Accepted: - return ObjectStore.ContainerExists; - - default: - throw new ResponseException(string.Format("Unexpected status {0} returned by Create Container.", response.StatusCode), response); - } - } - - public new void CreateObject(string container, Stream stream, string objectName, string contentType = null, int chunkSize = 4096, Dictionary headers = null, string region = null, Action progressUpdated = null, bool useInternalUrl = false, CloudIdentity identity = null) - { - if (container == null) - throw new ArgumentNullException(nameof(container)); - if (stream == null) - throw new ArgumentNullException(nameof(stream)); - if (objectName == null) - throw new ArgumentNullException(nameof(objectName)); - if (string.IsNullOrEmpty(container)) - throw new ArgumentException("container cannot be empty"); - if (string.IsNullOrEmpty(objectName)) - throw new ArgumentException("objectName cannot be empty"); - if (chunkSize < 0) - throw new ArgumentOutOfRangeException(nameof(chunkSize)); - CheckIdentity(identity); - - _cloudFilesValidator.ValidateContainerName(container); - _cloudFilesValidator.ValidateObjectName(objectName); - - if (stream.Length > LargeFileBatchThreshold) - { - throw new ArgumentException("objectName is too big"); - } - var urlPath = new Uri(string.Format("{0}/{1}/{2}", GetServiceEndpointCloudFiles(identity, region, useInternalUrl), _encodeDecodeProvider.UrlEncode(container), _encodeDecodeProvider.UrlEncode(objectName))); - - RequestSettings settings = BuildDefaultRequestSettings(); - settings.ChunkRequest = true; - settings.ContentType = contentType; - - StreamRESTRequest(identity, urlPath, HttpMethod.PUT, stream, chunkSize, headers: headers, progressUpdated: progressUpdated, requestSettings: settings); - } - - public new void CreateObjectFromFile(string container, string filePath, string objectName = null, string contentType = null, int chunkSize = 4096, Dictionary headers = null, string region = null, Action progressUpdated = null, bool useInternalUrl = false, CloudIdentity identity = null) - { - if (container == null) - throw new ArgumentNullException(nameof(container)); - if (filePath == null) - throw new ArgumentNullException(nameof(filePath)); - if (string.IsNullOrEmpty(container)) - throw new ArgumentException("container cannot be empty"); - if (string.IsNullOrEmpty(filePath)) - throw new ArgumentException("filePath cannot be empty"); - if (chunkSize < 0) - throw new ArgumentOutOfRangeException(nameof(chunkSize)); - CheckIdentity(identity); - - if (string.IsNullOrEmpty(objectName)) - objectName = Path.GetFileName(filePath); - - using (var stream = File.OpenRead(filePath)) - { - CreateObject(container, stream, objectName, contentType, chunkSize, headers, region, progressUpdated, useInternalUrl, identity); - } - } - public new void UpdateObjectMetadata(string container, string objectName, Dictionary metadata, string region = null, bool useInternalUrl = false, CloudIdentity identity = null) - { - if (container == null) - throw new ArgumentNullException(nameof(container)); - if (objectName == null) - throw new ArgumentNullException(nameof(objectName)); - if (metadata == null) - throw new ArgumentNullException(nameof(metadata)); - if (string.IsNullOrEmpty(container)) - throw new ArgumentException("container cannot be empty"); - if (string.IsNullOrEmpty(objectName)) - throw new ArgumentException("objectName cannot be empty"); - CheckIdentity(identity); - - _cloudFilesValidator.ValidateContainerName(container); - _cloudFilesValidator.ValidateObjectName(objectName); - - var headers = new Dictionary(StringComparer.OrdinalIgnoreCase); - foreach (KeyValuePair m in metadata) - { - if (string.IsNullOrEmpty(m.Key)) - throw new ArgumentException("metadata cannot contain any empty keys"); - - headers.Add(ObjectMetaDataPrefix + m.Key, EncodeUnicodeValue(m.Value)); - } - - var urlPath = new Uri(string.Format("{0}/{1}/{2}", GetServiceEndpointCloudFiles(identity, region, useInternalUrl), _encodeDecodeProvider.UrlEncode(container), _encodeDecodeProvider.UrlEncode(objectName))); - - RequestSettings settings = BuildDefaultRequestSettings(); - // make sure the content type is not changed by the metadata operation - settings.ContentType = null; - - ExecuteRESTRequest(identity, urlPath, HttpMethod.POST, headers: headers, settings: settings); - } - - public new Dictionary GetObjectMetaData(string container, string objectName, string region = null, bool useInternalUrl = false, CloudIdentity identity = null) - { - if (container == null) - throw new ArgumentNullException(nameof(container)); - if (objectName == null) - throw new ArgumentNullException(nameof(objectName)); - if (string.IsNullOrEmpty(container)) - throw new ArgumentException("container cannot be empty"); - if (string.IsNullOrEmpty(objectName)) - throw new ArgumentException("objectName cannot be empty"); - CheckIdentity(identity); - - _cloudFilesValidator.ValidateContainerName(container); - _cloudFilesValidator.ValidateObjectName(objectName); - var urlPath = new Uri(string.Format("{0}/{1}/{2}", GetServiceEndpointCloudFiles(identity, region, useInternalUrl), _encodeDecodeProvider.UrlEncode(container), _encodeDecodeProvider.UrlEncode(objectName))); - - var response = ExecuteRESTRequest(identity, urlPath, HttpMethod.HEAD); - - var processedHeaders = _cloudFilesMetadataProcessor.ProcessMetadata(response.Headers); - - return processedHeaders[ProcessedHeadersMetadataKey]; - } - - public new void GetObject(string container, string objectName, Stream outputStream, int chunkSize = 4096, Dictionary headers = null, string region = null, bool verifyEtag = false, Action progressUpdated = null, bool useInternalUrl = false, CloudIdentity identity = null) - { - if (container == null) - throw new ArgumentNullException(nameof(container)); - if (objectName == null) - throw new ArgumentNullException(nameof(objectName)); - if (outputStream == null) - throw new ArgumentNullException(nameof(outputStream)); - if (string.IsNullOrEmpty(container)) - throw new ArgumentException("container cannot be empty"); - if (string.IsNullOrEmpty(objectName)) - throw new ArgumentException("objectName cannot be empty"); - if (chunkSize < 0) - throw new ArgumentOutOfRangeException(nameof(chunkSize)); - CheckIdentity(identity); - - _cloudFilesValidator.ValidateContainerName(container); - _cloudFilesValidator.ValidateObjectName(objectName); - - var urlPath = new Uri(string.Format("{0}/{1}/{2}", GetServiceEndpointCloudFiles(identity, region, useInternalUrl), _encodeDecodeProvider.UrlEncode(container), _encodeDecodeProvider.UrlEncode(objectName))); - - long? initialPosition; - try - { - initialPosition = outputStream.Position; - } - catch (NotSupportedException) - { - if (verifyEtag) - throw; - - initialPosition = null; - } - - // This flag indicates whether the outputStream needs to be set prior to copying data. - // See: https://github.com/openstacknetsdk/openstack.net/issues/297 - bool requiresPositionReset = false; - - var response = ExecuteRESTRequest(identity, urlPath, HttpMethod.GET, (resp, isError) => - { - if (resp == null) - return new Response(0, null, null); - - string body; - - if (!isError) - { - using (var respStream = resp.GetResponseStream()) - { - // The second condition will throw a proper NotSupportedException if the position - // cannot be checked. - if (requiresPositionReset && outputStream.Position != initialPosition) - outputStream.Position = initialPosition.Value; - - requiresPositionReset = true; - CopyStream(respStream, outputStream, chunkSize, progressUpdated); - } - - body = "[Binary]"; - } - else - { - using (StreamReader reader = new StreamReader(resp.GetResponseStream())) - { - body = reader.ReadToEnd(); - } - } - - var respHeaders = resp.Headers.AllKeys.Select(key => new HttpHeader(key, resp.GetResponseHeader(key))).ToList(); - - return new Response(resp.StatusCode, respHeaders, body); - }, headers: headers); - - } - - public new void GetObjectSaveToFile(string container, string saveDirectory, string objectName, string fileName = null, int chunkSize = 65536, Dictionary headers = null, string region = null, bool verifyEtag = false, Action progressUpdated = null, bool useInternalUrl = false, CloudIdentity identity = null) - { - if (container == null) - throw new ArgumentNullException(nameof(container)); - if (saveDirectory == null) - throw new ArgumentNullException(nameof(saveDirectory)); - if (objectName == null) - throw new ArgumentNullException(nameof(objectName)); - if (string.IsNullOrEmpty(container)) - throw new ArgumentException("container cannot be empty"); - if (string.IsNullOrEmpty(saveDirectory)) - throw new ArgumentException("saveDirectory cannot be empty"); - if (string.IsNullOrEmpty(objectName)) - throw new ArgumentException("objectName cannot be empty"); - if (chunkSize < 0) - throw new ArgumentOutOfRangeException(nameof(chunkSize)); - CheckIdentity(identity); - - if (string.IsNullOrEmpty(fileName)) - fileName = objectName; - - var filePath = Path.Combine(saveDirectory, string.IsNullOrEmpty(fileName) ? objectName : fileName); - - try - { - using (var fileStream = File.Open(filePath, FileMode.Create, FileAccess.ReadWrite)) - { - GetObject(container, objectName, fileStream, chunkSize, headers, region, verifyEtag, progressUpdated, useInternalUrl, identity); - } - } - catch (InvalidETagException) - { - File.Delete(filePath); - throw; - } - } - - private static string EncodeUnicodeValue(string value) - { - if (value == null) - return null; - - return Encoding.GetEncoding("ISO-8859-1").GetString(Encoding.UTF8.GetBytes(value)); - } - internal void CheckResponse(Response response) - { - if (response == null) - throw new ArgumentNullException(nameof(response)); - - ResponseCodeValidator.Validate(response); - } - protected new Response ExecuteRESTRequest(CloudIdentity identity, Uri absoluteUri, HttpMethod method, object body = null, Dictionary queryStringParameter = null, Dictionary headers = null, bool isRetry = false, bool isTokenRequest = false, RequestSettings settings = null) - { - if (absoluteUri == null) - throw new ArgumentNullException(nameof(absoluteUri)); - CheckIdentity(identity); - - return ExecuteRESTRequest>(identity, absoluteUri, method, body, queryStringParameter, headers, isRetry, isTokenRequest, settings, RestService.Execute); - } - protected new Response ExecuteRESTRequest(CloudIdentity identity, Uri absoluteUri, HttpMethod method, object body = null, Dictionary queryStringParameter = null, Dictionary headers = null, bool isRetry = false, bool isTokenRequest = false, RequestSettings settings = null) - { - if (absoluteUri == null) - throw new ArgumentNullException(nameof(absoluteUri)); - CheckIdentity(identity); - - return ExecuteRESTRequest(identity, absoluteUri, method, body, queryStringParameter, headers, isRetry, isTokenRequest, settings, RestService.Execute); - } - - protected new Response ExecuteRESTRequest(CloudIdentity identity, Uri absoluteUri, HttpMethod method, Func buildResponseCallback, object body = null, Dictionary queryStringParameter = null, Dictionary headers = null, bool isRetry = false, bool isTokenRequest = false, RequestSettings settings = null) - { - if (absoluteUri == null) - throw new ArgumentNullException(nameof(absoluteUri)); - CheckIdentity(identity); - - return ExecuteRESTRequest(identity, absoluteUri, method, body, queryStringParameter, headers, isRetry, isTokenRequest, settings, - (uri, requestMethod, requestBody, requestHeaders, requestQueryParams, requestSettings) => RestService.Execute(uri, requestMethod, buildResponseCallback, requestBody, requestHeaders, requestQueryParams, requestSettings)); - } - - private T ExecuteRESTRequest(CloudIdentity identity, Uri absoluteUri, HttpMethod method, object body, Dictionary queryStringParameter, Dictionary headers, bool isRetry, bool isTokenRequest, RequestSettings requestSettings, - Func, Dictionary, RequestSettings, T> callback) where T : Response - { - if (absoluteUri == null) - throw new ArgumentNullException(nameof(absoluteUri)); - CheckIdentity(identity); - - identity = GetDefaultIdentity(identity); - - if (requestSettings == null) - requestSettings = BuildDefaultRequestSettings(); - - if (headers == null) - headers = new Dictionary(); - - if (!isTokenRequest) - headers["X-Auth-Token"] = ((IdentityTokenV3)((BlueMixIdentityProvider)IdentityProvider).GetToken(identity, isRetry)).Id; - - string bodyStr = null; - if (body != null) - { - if (body is JObject) - bodyStr = body.ToString(); - else if (body is string) - bodyStr = body as string; - else - bodyStr = JsonConvert.SerializeObject(body, new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore }); - } - - if (string.IsNullOrEmpty(requestSettings.UserAgent)) - requestSettings.UserAgent = DefaultUserAgent; - - var response = callback(absoluteUri, method, bodyStr, headers, queryStringParameter, requestSettings); - - // on errors try again 1 time. - if (response.StatusCode == HttpStatusCode.Unauthorized && !isRetry && !isTokenRequest) - { - return ExecuteRESTRequest(identity, absoluteUri, method, body, queryStringParameter, headers, true, isTokenRequest, requestSettings, callback); - } - - CheckResponse(response); - - return response; - } - - protected new Response StreamRESTRequest(CloudIdentity identity, Uri absoluteUri, HttpMethod method, Stream stream, int chunkSize, long maxReadLength = 0, Dictionary queryStringParameter = null, Dictionary headers = null, bool isRetry = false, RequestSettings requestSettings = null, Action progressUpdated = null) - { - if (absoluteUri == null) - throw new ArgumentNullException(nameof(absoluteUri)); - if (stream == null) - throw new ArgumentNullException(nameof(stream)); - if (chunkSize <= 0) - throw new ArgumentOutOfRangeException(nameof(chunkSize)); - if (maxReadLength < 0) - throw new ArgumentOutOfRangeException(nameof(maxReadLength)); - CheckIdentity(identity); - - identity = GetDefaultIdentity(identity); - - if (requestSettings == null) - requestSettings = BuildDefaultRequestSettings(); - - requestSettings.Timeout = TimeSpan.FromMilliseconds(14400000); // Need to pass this in. - - if (headers == null) - headers = new Dictionary(); - - headers["X-Auth-Token"] = ((IdentityTokenV3)((BlueMixIdentityProvider)IdentityProvider).GetToken(identity, isRetry)).Id; ; - - if (string.IsNullOrEmpty(requestSettings.UserAgent)) - requestSettings.UserAgent = DefaultUserAgent; - - long? initialPosition; - try - { - initialPosition = stream.Position; - } - catch (NotSupportedException) - { - initialPosition = null; - } - - Response response; - try - { - response = RestService.Stream(absoluteUri, method, stream, chunkSize, maxReadLength, headers, queryStringParameter, requestSettings, progressUpdated); - } - catch (ProtocolViolationException) - { - ServicePoint servicePoint = ServicePointManager.FindServicePoint(absoluteUri); - if (servicePoint.ProtocolVersion < HttpVersion.Version11) - { - // this is a workaround for issue #333 - // https://github.com/openstacknetsdk/openstack.net/issues/333 - // http://stackoverflow.com/a/22976809/138304 - int maxIdleTime = servicePoint.MaxIdleTime; - servicePoint.MaxIdleTime = 0; - Thread.Sleep(1); - servicePoint.MaxIdleTime = maxIdleTime; - } - - response = RestService.Stream(absoluteUri, method, stream, chunkSize, maxReadLength, headers, queryStringParameter, requestSettings, progressUpdated); - } - - // on errors try again 1 time. - if (response.StatusCode == HttpStatusCode.Unauthorized && !isRetry && initialPosition != null) - { - bool canRetry; - - try - { - if (stream.Position != initialPosition.Value) - stream.Position = initialPosition.Value; - - canRetry = true; - } - catch (NotSupportedException) - { - // unable to retry the operation - canRetry = false; - } - - if (canRetry) - { - return StreamRESTRequest(identity, absoluteUri, method, stream, chunkSize, maxReadLength, queryStringParameter, headers, true, requestSettings, progressUpdated); - } - } - - CheckResponse(response); - - return response; - } - - public new void DeleteObject(string container, string objectName, Dictionary headers = null, bool deleteSegments = true, string region = null, bool useInternalUrl = false, CloudIdentity identity = null) - { - if (container == null) - throw new ArgumentNullException(nameof(container)); - if (objectName == null) - throw new ArgumentNullException(nameof(objectName)); - if (string.IsNullOrEmpty(container)) - throw new ArgumentException("container cannot be empty"); - if (string.IsNullOrEmpty(objectName)) - throw new ArgumentException("objectName cannot be empty"); - CheckIdentity(identity); - - _cloudFilesValidator.ValidateContainerName(container); - _cloudFilesValidator.ValidateObjectName(objectName); - - Dictionary objectHeader = null; - if (deleteSegments) - objectHeader = GetObjectHeaders(container, objectName, region, useInternalUrl, identity); - - if (deleteSegments && objectHeader != null && objectHeader.Any(h => h.Key.Equals(ObjectManifest, StringComparison.OrdinalIgnoreCase))) - { - var objects = ListObjects(container, region: region, useInternalUrl: useInternalUrl, - identity: identity); - - if (objects != null && objects.Any()) - { - var segments = objects.Where(f => f.Name.StartsWith(string.Format("{0}.seg", objectName))); - var delObjects = new List { objectName }; - if (segments.Any()) - delObjects.AddRange(segments.Select(s => s.Name)); - - DeleteObjects(container, delObjects, headers, region, useInternalUrl, identity); - } - else - throw new Exception(string.Format("Object \"{0}\" in container \"{1}\" does not exist.", objectName, container)); - } - else - { - var urlPath = new Uri(string.Format("{0}/{1}/{2}", GetServiceEndpointCloudFiles(identity, region, useInternalUrl), _encodeDecodeProvider.UrlEncode(container), _encodeDecodeProvider.UrlEncode(objectName))); - - ExecuteRESTRequest(identity, urlPath, HttpMethod.DELETE, headers: headers); - } - } - - public new Dictionary GetObjectHeaders(string container, string objectName, string region = null, bool useInternalUrl = false, CloudIdentity identity = null) - { - if (container == null) - throw new ArgumentNullException(nameof(container)); - if (objectName == null) - throw new ArgumentNullException(nameof(objectName)); - if (string.IsNullOrEmpty(container)) - throw new ArgumentException("container cannot be empty"); - if (string.IsNullOrEmpty(objectName)) - throw new ArgumentException("objectName cannot be empty"); - CheckIdentity(identity); - - _cloudFilesValidator.ValidateContainerName(container); - _cloudFilesValidator.ValidateObjectName(objectName); - var urlPath = new Uri(string.Format("{0}/{1}/{2}", GetServiceEndpointCloudFiles(identity, region, useInternalUrl), _encodeDecodeProvider.UrlEncode(container), _encodeDecodeProvider.UrlEncode(objectName))); - - var response = ExecuteRESTRequest(identity, urlPath, HttpMethod.HEAD); - - var processedHeaders = _cloudFilesMetadataProcessor.ProcessMetadata(response.Headers); - - return processedHeaders[ProcessedHeadersHeaderKey]; - } - - public new IEnumerable ListObjects(string container, int? limit = null, string marker = null, string markerEnd = null, string prefix = null, string region = null, bool useInternalUrl = false, CloudIdentity identity = null) - { - if (container == null) - throw new ArgumentNullException(nameof(container)); - if (string.IsNullOrEmpty(container)) - throw new ArgumentException("container cannot be empty"); - if (limit < 0) - throw new ArgumentOutOfRangeException(nameof(limit)); - CheckIdentity(identity); - - _cloudFilesValidator.ValidateContainerName(container); - var urlPath = new Uri(string.Format("{0}/{1}", GetServiceEndpointCloudFiles(identity, region, useInternalUrl), _encodeDecodeProvider.UrlEncode(container))); - - var queryStringParameter = new Dictionary(); - - if (limit != null) - queryStringParameter.Add("limit", limit.ToString()); - - if (!string.IsNullOrEmpty(marker)) - queryStringParameter.Add("marker", marker); - - if (!string.IsNullOrEmpty(markerEnd)) - queryStringParameter.Add("end_marker", markerEnd); - - if (!string.IsNullOrEmpty(prefix)) - queryStringParameter.Add("prefix", prefix); - - var response = ExecuteRESTRequest(identity, urlPath, HttpMethod.GET, null, queryStringParameter); - - if (response == null || response.Data == null) - return null; - - return response.Data; - } - - public new void CopyObject(string sourceContainer, string sourceObjectName, string destinationContainer, string destinationObjectName, string destinationContentType = null, Dictionary headers = null, string region = null, bool useInternalUrl = false, CloudIdentity identity = null) - { - if (sourceContainer == null) - throw new ArgumentNullException(nameof(sourceContainer)); - if (sourceObjectName == null) - throw new ArgumentNullException(nameof(sourceObjectName)); - if (destinationContainer == null) - throw new ArgumentNullException(nameof(destinationContainer)); - if (destinationObjectName == null) - throw new ArgumentNullException(nameof(destinationObjectName)); - if (string.IsNullOrEmpty(sourceContainer)) - throw new ArgumentException("sourceContainer cannot be empty"); - if (string.IsNullOrEmpty(sourceObjectName)) - throw new ArgumentException("sourceObjectName cannot be empty"); - if (string.IsNullOrEmpty(destinationContainer)) - throw new ArgumentException("destinationContainer cannot be empty"); - if (string.IsNullOrEmpty(destinationObjectName)) - throw new ArgumentException("destinationObjectName cannot be empty"); - CheckIdentity(identity); - - _cloudFilesValidator.ValidateContainerName(sourceContainer); - _cloudFilesValidator.ValidateObjectName(sourceObjectName); - - _cloudFilesValidator.ValidateContainerName(destinationContainer); - _cloudFilesValidator.ValidateObjectName(destinationObjectName); - - var urlPath = new Uri(string.Format("{0}/{1}/{2}", GetServiceEndpointCloudFiles(identity, region, useInternalUrl), _encodeDecodeProvider.UrlEncode(sourceContainer), _encodeDecodeProvider.UrlEncode(sourceObjectName))); - - if (headers == null) - headers = new Dictionary(StringComparer.OrdinalIgnoreCase); - - headers.Add(Destination, string.Format("{0}/{1}", UriUtility.UriEncode(destinationContainer, UriPart.AnyUrl, Encoding.UTF8), UriUtility.UriEncode(destinationObjectName, UriPart.AnyUrl, Encoding.UTF8))); - - RequestSettings settings = BuildDefaultRequestSettings(); - if (destinationContentType != null) - { - settings.ContentType = destinationContentType; - } - else - { - // make sure to preserve the content type during the copy operation - settings.ContentType = null; - } - - ExecuteRESTRequest(identity, urlPath, HttpMethod.COPY, headers: headers, settings: settings); - } - - public new void DeleteObjects(string container, IEnumerable objects, Dictionary headers = null, string region = null, bool useInternalUrl = false, CloudIdentity identity = null) - { - if (container == null) - throw new ArgumentNullException(nameof(container)); - if (objects == null) - throw new ArgumentNullException(nameof(objects)); - if (string.IsNullOrEmpty(container)) - throw new ArgumentException("container cannot be empty"); - - BulkDelete(objects.Select(o => new KeyValuePair(container, o)), headers, region, useInternalUrl, identity); - } - - public new void BulkDelete(IEnumerable> items, Dictionary headers = null, string region = null, bool useInternalUrl = false, CloudIdentity identity = null) - { - var urlPath = new Uri(string.Format("{0}/?bulk-delete", GetServiceEndpointCloudFiles(identity, region, useInternalUrl))); - - var encoded = items.Select( - pair => - { - if (string.IsNullOrEmpty(pair.Key)) - throw new ArgumentException("items", "items cannot contain any entries with a null or empty key (container name)"); - if (string.IsNullOrEmpty(pair.Value)) - throw new ArgumentException("items", "items cannot contain any entries with a null or empty value (object name)"); - _cloudFilesValidator.ValidateContainerName(pair.Key); - _cloudFilesValidator.ValidateObjectName(pair.Value); - - return string.Format("/{0}/{1}", _encodeDecodeProvider.UrlEncode(pair.Key), _encodeDecodeProvider.UrlEncode(pair.Value)); - }); - var body = string.Join("\n", encoded.ToArray()); - - var response = ExecuteRESTRequest(identity, urlPath, HttpMethod.POST, body: body, headers: headers, settings: new JSIStudios.SimpleRESTServices.Client.Json.JsonRequestSettings { ContentType = "text/plain" }); - - Status status; - if (_statusParser.TryParse(response.Data.Status, out status)) - { - if (status.Code != 200 && !response.Data.Errors.Any()) - { - response.Data.AllItems = encoded; - throw new BulkDeletionException(response.Data.Status, _bulkDeletionResultMapper.Map(response.Data)); - } - } - } - - public new Uri CreateTemporaryPublicUri(HttpMethod method, string container, string objectName, string key, DateTimeOffset expiration, string region = null, bool useInternalUrl = false, CloudIdentity identity = null) - { - if (container == null) - throw new ArgumentNullException(nameof(container)); - if (objectName == null) - throw new ArgumentNullException(nameof(objectName)); - if (key == null) - throw new ArgumentNullException(nameof(key)); - if (string.IsNullOrEmpty(container)) - throw new ArgumentException("container cannot be empty"); - if (string.IsNullOrEmpty(objectName)) - throw new ArgumentException("objectName cannot be empty"); - if (string.IsNullOrEmpty(key)) - throw new ArgumentException("key cannot be empty"); - CheckIdentity(identity); - - _cloudFilesValidator.ValidateContainerName(container); - _cloudFilesValidator.ValidateObjectName(objectName); - - Uri baseAddress = new Uri(GetServiceEndpointCloudFiles(identity, region, useInternalUrl), UriKind.Absolute); - - StringBuilder body = new StringBuilder(); - body.Append(method.ToString().ToUpperInvariant()).Append('\n'); - body.Append(ToTimestamp(expiration) / 1000).Append('\n'); - body.Append(baseAddress.PathAndQuery).Append('/').Append(container).Append('/').Append(objectName); - - using (HMAC hmac = HMAC.Create()) - { - hmac.HashName = "SHA1"; - hmac.Key = Encoding.UTF8.GetBytes(key); - byte[] hash = hmac.ComputeHash(Encoding.UTF8.GetBytes(body.ToString())); - string sig = string.Join(string.Empty, Array.ConvertAll(hash, i => i.ToString("x2"))); - string expires = (ToTimestamp(expiration) / 1000).ToString(); - - return new Uri(String.Format("{0}/{1}/{2}?temp_url_sig={3}&temp_url_expires={4}", baseAddress, container, objectName, sig, expires)); - } - } - private long ToTimestamp(DateTimeOffset dateTimeOffset) - { - DateTimeOffset Epoch = new DateTimeOffset(new DateTime(1970, 1, 1), TimeSpan.Zero); - return (long)(dateTimeOffset - Epoch).TotalMilliseconds; - } - - } - internal class BulkDeletionResultMapper : IObjectMapper - { - private readonly IStatusParser _statusParser; - - public BulkDeletionResultMapper(IStatusParser statusParser) - { - _statusParser = statusParser; - } - - public BulkDeletionResults Map(BulkDeleteResponse from) - { - var successfulObjects = from.AllItems.Where(i => !from.IsItemError(i)); - var failedObjects = from.Errors.Select(e => - { - var eParts = e.ToArray(); - Status errorStatus; - string errorItem; - - if (eParts.Length != 2) - { - errorStatus = new Status(0, "Unknown"); - errorItem = string.Format("The error array has an unexpected length. Array: {0}", string.Join("||", eParts)); - } - else - { - errorItem = eParts[1]; - if (!_statusParser.TryParse(eParts[0], out errorStatus)) - { - errorItem = eParts[0]; - if (!_statusParser.TryParse(eParts[1], out errorStatus)) - { - errorStatus = new Status(0, "Unknown"); - errorItem = string.Format("The error array is in an unknown format. Array: {0}", string.Join("||", eParts)); - } - } - } - - return new BulkDeletionFailedObject(errorItem, errorStatus); - }); - - return new BulkDeletionResults(successfulObjects, failedObjects); - } - - public BulkDeleteResponse Map(BulkDeletionResults to) - { - throw new NotImplementedException(); - } - } - internal class CloudFilesMetadataProcessor : IObjectStorageMetadataProcessor - { - - private static readonly CloudFilesMetadataProcessor _default = new CloudFilesMetadataProcessor(); - - public static CloudFilesMetadataProcessor Default - { - get - { - return _default; - } - } - - public virtual Dictionary> ProcessMetadata(IList httpHeaders) - { - if (httpHeaders == null) - throw new ArgumentNullException(nameof(httpHeaders)); - - var pheaders = new Dictionary(StringComparer.OrdinalIgnoreCase); - var metadata = new Dictionary(StringComparer.OrdinalIgnoreCase); - foreach (var header in httpHeaders) - { - if (header == null) - throw new ArgumentException("httpHeaders cannot contain any null values"); - if (string.IsNullOrEmpty(header.Key)) - throw new ArgumentException("httpHeaders cannot contain any values with a null or empty key"); - - if (header.Key.StartsWith(CloudFilesProvider.AccountMetaDataPrefix, StringComparison.OrdinalIgnoreCase)) - { - metadata.Add(header.Key.Substring(CloudFilesProvider.AccountMetaDataPrefix.Length), DecodeUnicodeValue(header.Value)); - } - else if (header.Key.StartsWith(CloudFilesProvider.ContainerMetaDataPrefix, StringComparison.OrdinalIgnoreCase)) - { - metadata.Add(header.Key.Substring(CloudFilesProvider.ContainerMetaDataPrefix.Length), DecodeUnicodeValue(header.Value)); - } - else if (header.Key.StartsWith(CloudFilesProvider.ObjectMetaDataPrefix, StringComparison.OrdinalIgnoreCase)) - { - metadata.Add(header.Key.Substring(CloudFilesProvider.ObjectMetaDataPrefix.Length), DecodeUnicodeValue(header.Value)); - } - else - { - pheaders.Add(header.Key, header.Value); - } - } - - var processedHeaders = new Dictionary>(StringComparer.OrdinalIgnoreCase) - { - {CloudFilesProvider.ProcessedHeadersHeaderKey, pheaders}, - {CloudFilesProvider.ProcessedHeadersMetadataKey, metadata} - }; - - return processedHeaders; - } - - private string DecodeUnicodeValue(string value) - { - return Encoding.UTF8.GetString(Encoding.GetEncoding("ISO-8859-1").GetBytes(value)); - } - } - - [JsonObject(MemberSerialization.OptIn)] - internal class BulkDeleteResponse - { - [JsonProperty("Number Not Found")] - public int NumberNotFound { get; set; } - - [JsonProperty("Response Status")] - public string Status { get; set; } - - [JsonProperty("Errors")] - public IEnumerable> Errors { get; set; } - - [JsonProperty("Number Deleted")] - public int NumberDeleted { get; set; } - - [JsonProperty("Response Body")] - public string ResponseBody { get; set; } - - public IEnumerable AllItems { get; set; } - - public bool IsItemError(string s) - { - return Errors.Any(e => e.Any(e2 => e2.Equals(s))); - } - } - #region Json Classes - [JsonObject(MemberSerialization.OptIn)] - public class Endpoint - { - [JsonProperty("region_id")] - public string Region_id { get; set; } - [JsonProperty("url")] - public string Url { get; set; } - [JsonProperty("region")] - public string Region { get; set; } - [JsonProperty("interface")] - public string @Interface { get; set; } - [JsonProperty("id")] - public string Id { get; set; } - } - [JsonObject(MemberSerialization.OptIn)] - public class Catalog - { - [JsonProperty("endpoints")] - public Endpoint[] Endpoints { get; set; } - [JsonProperty("type")] - public string Type { get; set; } - [JsonProperty("id")] - public string Id { get; set; } - [JsonProperty("name")] - public string Name { get; set; } - } - - [JsonObject(MemberSerialization.OptIn)] - public class UserAccessV3 : UserAccess - { - [JsonProperty("catalog")] - public Catalog[] Catalog { get; set; } - - [JsonExtensionData] - public IDictionary _additionalData; - } - #endregion - #endregion - - } -} - diff --git a/dotnet/src/dotnetframework/Providers/Storage/GXBluemix/GXBluemix.csproj b/dotnet/src/dotnetframework/Providers/Storage/GXBluemix/GXBluemix.csproj deleted file mode 100644 index 37f002ed7..000000000 --- a/dotnet/src/dotnetframework/Providers/Storage/GXBluemix/GXBluemix.csproj +++ /dev/null @@ -1,28 +0,0 @@ - - - net462 - GXBluemix - GXBluemix - CA1501;CA1812 - Bluemix - GeneXus.Bluemix - - - - - - - - - - 0.2.5 - - - - - - - - - - \ No newline at end of file diff --git a/dotnet/src/dotnetframework/Providers/Storage/GXBluemix/GlobalSuppressions.cs b/dotnet/src/dotnetframework/Providers/Storage/GXBluemix/GlobalSuppressions.cs deleted file mode 100644 index 6ca697c8d..000000000 --- a/dotnet/src/dotnetframework/Providers/Storage/GXBluemix/GlobalSuppressions.cs +++ /dev/null @@ -1,18 +0,0 @@ -// This file is used by Code Analysis to maintain SuppressMessage -// attributes that are applied to this project. -// Project-level suppressions either have no target or are given -// a specific target and scoped to a namespace, type, member, etc. - -[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Design", "CA1034:Nested types should not be visible", Justification = "", Scope = "type", Target = "~T:GeneXus.Storage.GXBluemix.ExternalProviderBluemix.Authorization")] -[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Design", "CA1034:Nested types should not be visible", Justification = "", Scope = "type", Target = "~T:GeneXus.Storage.GXBluemix.ExternalProviderBluemix.BluemixFilesProvider")] -[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Design", "CA1034:Nested types should not be visible", Justification = "", Scope = "type", Target = "~T:GeneXus.Storage.GXBluemix.ExternalProviderBluemix.BlueMixIdentityProvider")] -[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Design", "CA1034:Nested types should not be visible", Justification = "", Scope = "type", Target = "~T:GeneXus.Storage.GXBluemix.ExternalProviderBluemix.Catalog")] -[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Design", "CA1034:Nested types should not be visible", Justification = "", Scope = "type", Target = "~T:GeneXus.Storage.GXBluemix.ExternalProviderBluemix.Endpoint")] -[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Design", "CA1034:Nested types should not be visible", Justification = "", Scope = "type", Target = "~T:GeneXus.Storage.GXBluemix.ExternalProviderBluemix.IdentityTokenV3")] -[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Design", "CA1034:Nested types should not be visible", Justification = "", Scope = "type", Target = "~T:GeneXus.Storage.GXBluemix.ExternalProviderBluemix.UserAccessV3")] -[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Naming", "CA1707:Identifiers should not contain underscores", Justification = "", Scope = "member", Target = "~P:GeneXus.Storage.GXBluemix.ExternalProviderBluemix.Endpoint.Region_id")] - -[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Performance", "CA1819:Properties should not return arrays", Justification = "", Scope = "member", Target = "~P:GeneXus.Storage.GXBluemix.ExternalProviderBluemix.Catalog.Endpoints")] -[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Performance", "CA1819:Properties should not return arrays", Justification = "", Scope = "member", Target = "~P:GeneXus.Storage.GXBluemix.ExternalProviderBluemix.UserAccessV3.Catalog")] -[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Usage", "CA2208:Instantiate argument exceptions correctly", Justification = "", Scope = "member", Target = "~M:GeneXus.Storage.GXBluemix.ExternalProviderBluemix.BluemixFilesProvider.BulkDelete(System.Collections.Generic.IEnumerable{System.Collections.Generic.KeyValuePair{System.String,System.String}},System.Collections.Generic.Dictionary{System.String,System.String},System.String,System.Boolean,net.openstack.Core.Domain.CloudIdentity)")] -[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Usage", "CA2208:Instantiate argument exceptions correctly", Justification = "", Scope = "member", Target = "~M:GeneXus.Storage.GXBluemix.ExternalProviderBluemix.BluemixFilesProvider.GetServiceEndpointCloudFiles(net.openstack.Core.Domain.CloudIdentity,System.String,System.Boolean)~System.String")] \ No newline at end of file diff --git a/dotnet/src/dotnetframework/Providers/Storage/GXBluemix/Properties/AssemblyInfo.cs b/dotnet/src/dotnetframework/Providers/Storage/GXBluemix/Properties/AssemblyInfo.cs deleted file mode 100644 index bb7c4dab9..000000000 --- a/dotnet/src/dotnetframework/Providers/Storage/GXBluemix/Properties/AssemblyInfo.cs +++ /dev/null @@ -1,27 +0,0 @@ -using System.Reflection; -using System.Resources; -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; - -// General Information about an assembly is controlled through the following -// set of attributes. Change these attribute values to modify the information -// associated with an assembly. -[assembly: AssemblyTitle("GXBluemix")] - - -// Setting ComVisible to false makes the types in this assembly not visible -// to COM components. If you need to access a type in this assembly from -// COM, set the ComVisible attribute to true on that type. - -// The following GUID is for the ID of the typelib if this project is exposed to COM -// Version information for an assembly consists of the following four values: -// -// Major Version -// Minor Version -// Build Number -// Revision -// -// You can specify all the values or you can default the Build and Revision Numbers -// by using the '*' as shown below: -// [assembly: AssemblyVersion("1.0.*")] -[assembly: NeutralResourcesLanguageAttribute("en")] diff --git a/dotnet/src/dotnetframework/Providers/Storage/GXBluemix/app.config b/dotnet/src/dotnetframework/Providers/Storage/GXBluemix/app.config deleted file mode 100644 index 1cd55066a..000000000 --- a/dotnet/src/dotnetframework/Providers/Storage/GXBluemix/app.config +++ /dev/null @@ -1,15 +0,0 @@ - - - - - - - - - - - - - - - From 2810d1b044a15a1084dcb8ec480b16101cf66170 Mon Sep 17 00:00:00 2001 From: Gonzalo Date: Tue, 1 Jun 2021 19:52:20 -0300 Subject: [PATCH 02/33] External Storage support --- dotnet/DotNetStandardClasses.sln | 7 -- .../src/dotnetcore/GxClasses/GxClasses.csproj | 1 + .../GxClasses/Core/GXApplication.cs | 2 +- .../GxClasses/Core/GXUtilsCommon.cs | 2 +- .../GxClasses/Data/GXDataNTierADO.cs | 18 +-- .../GxClasses/Domain/GXFileIO.cs | 12 +- .../Services/Storage/ExternalProviderBase.cs | 111 ++++++++++++++++++ 7 files changed, 129 insertions(+), 24 deletions(-) create mode 100644 dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderBase.cs diff --git a/dotnet/DotNetStandardClasses.sln b/dotnet/DotNetStandardClasses.sln index cade4fecd..7aca0805e 100644 --- a/dotnet/DotNetStandardClasses.sln +++ b/dotnet/DotNetStandardClasses.sln @@ -65,8 +65,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "GXAmazonS3", "src\dotnetfra EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "GXAzureStorage", "src\dotnetframework\Providers\Storage\GXAzureStorage\GXAzureStorage.csproj", "{F6372249-AF37-4455-B572-5BDF8DE2ACC8}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "GXBluemix", "src\dotnetframework\Providers\Storage\GXBluemix\GXBluemix.csproj", "{FCA5536D-542E-480E-92B0-316E0290553B}" -EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "GXGoogleCloud", "src\dotnetframework\Providers\Storage\GXGoogleCloud\GXGoogleCloud.csproj", "{E9072D95-D116-4D4B-B981-46146BCDE052}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "GXOpenStack", "src\dotnetframework\Providers\Storage\GXOpenStack\GXOpenStack.csproj", "{64E958D8-CE7B-482D-B339-3B52E26CA4AC}" @@ -249,10 +247,6 @@ Global {F6372249-AF37-4455-B572-5BDF8DE2ACC8}.Debug|Any CPU.Build.0 = Debug|Any CPU {F6372249-AF37-4455-B572-5BDF8DE2ACC8}.Release|Any CPU.ActiveCfg = Release|Any CPU {F6372249-AF37-4455-B572-5BDF8DE2ACC8}.Release|Any CPU.Build.0 = Release|Any CPU - {FCA5536D-542E-480E-92B0-316E0290553B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {FCA5536D-542E-480E-92B0-316E0290553B}.Debug|Any CPU.Build.0 = Debug|Any CPU - {FCA5536D-542E-480E-92B0-316E0290553B}.Release|Any CPU.ActiveCfg = Release|Any CPU - {FCA5536D-542E-480E-92B0-316E0290553B}.Release|Any CPU.Build.0 = Release|Any CPU {E9072D95-D116-4D4B-B981-46146BCDE052}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {E9072D95-D116-4D4B-B981-46146BCDE052}.Debug|Any CPU.Build.0 = Debug|Any CPU {E9072D95-D116-4D4B-B981-46146BCDE052}.Release|Any CPU.ActiveCfg = Release|Any CPU @@ -398,7 +392,6 @@ Global {F82842DA-F15E-49C5-993E-4C269818FF1F} = {F900A4AD-7249-41B4-B918-CB9E8C73747C} {3701565F-1A95-4592-B90D-291CCDE5505D} = {F82842DA-F15E-49C5-993E-4C269818FF1F} {F6372249-AF37-4455-B572-5BDF8DE2ACC8} = {F82842DA-F15E-49C5-993E-4C269818FF1F} - {FCA5536D-542E-480E-92B0-316E0290553B} = {F82842DA-F15E-49C5-993E-4C269818FF1F} {E9072D95-D116-4D4B-B981-46146BCDE052} = {F82842DA-F15E-49C5-993E-4C269818FF1F} {64E958D8-CE7B-482D-B339-3B52E26CA4AC} = {F82842DA-F15E-49C5-993E-4C269818FF1F} {A14C2C2C-ACE3-4712-A527-E4E5F02729FA} = {F900A4AD-7249-41B4-B918-CB9E8C73747C} diff --git a/dotnet/src/dotnetcore/GxClasses/GxClasses.csproj b/dotnet/src/dotnetcore/GxClasses/GxClasses.csproj index 4271eb3fa..3cda96e94 100644 --- a/dotnet/src/dotnetcore/GxClasses/GxClasses.csproj +++ b/dotnet/src/dotnetcore/GxClasses/GxClasses.csproj @@ -78,6 +78,7 @@ + diff --git a/dotnet/src/dotnetframework/GxClasses/Core/GXApplication.cs b/dotnet/src/dotnetframework/GxClasses/Core/GXApplication.cs index a31c42067..1d4b532ea 100644 --- a/dotnet/src/dotnetframework/GxClasses/Core/GXApplication.cs +++ b/dotnet/src/dotnetframework/GxClasses/Core/GXApplication.cs @@ -690,7 +690,7 @@ public static bool GetHttpRequestPostedFile(IGxContext gxContext, string varName ext = ext.TrimStart('.'); string filePath = FileUtil.getTempFileName(tempDir); GXLogging.Debug(log, "cgiGet(" + varName + "), fileName:" + filePath); - GxFile file = new GxFile(tempDir, filePath, GxFileType.PrivateAttribute); + GxFile file = new GxFile(tempDir, filePath, GxFileType.Private); filePath = file.Create(pf.InputStream); string fileGuid = GxUploadHelper.GetUploadFileGuid(); fileToken = GxUploadHelper.GetUploadFileId(fileGuid); diff --git a/dotnet/src/dotnetframework/GxClasses/Core/GXUtilsCommon.cs b/dotnet/src/dotnetframework/GxClasses/Core/GXUtilsCommon.cs index 2dc30e5fa..98ca76aa8 100644 --- a/dotnet/src/dotnetframework/GxClasses/Core/GXUtilsCommon.cs +++ b/dotnet/src/dotnetframework/GxClasses/Core/GXUtilsCommon.cs @@ -5264,7 +5264,7 @@ public static string ResolveUri(string uriString, bool absUrl, IGxContext contex string basePath = Path.Combine(Path.Combine(Preferences.getBLOB_PATH(), MultimediaDirectory)); try { - GxFile file = new GxFile(string.Empty, PathUtil.SafeCombine(basePath, fileName), GxFileType.PublicAttribute); + GxFile file = new GxFile(string.Empty, PathUtil.SafeCombine(basePath, fileName), GxFileType.Public); return PathToUrl(file.GetURI(), absUrl, context); } catch (ArgumentException ex) diff --git a/dotnet/src/dotnetframework/GxClasses/Data/GXDataNTierADO.cs b/dotnet/src/dotnetframework/GxClasses/Data/GXDataNTierADO.cs index 0219ffb9d..e1904d8bd 100644 --- a/dotnet/src/dotnetframework/GxClasses/Data/GXDataNTierADO.cs +++ b/dotnet/src/dotnetframework/GxClasses/Data/GXDataNTierADO.cs @@ -105,7 +105,7 @@ public string getBLOBFile(int id) public string getBLOBFile(int id, string extension, string name) { - string fileName = FileUtil.getTempFileName(_gxDbCommand.Conn.BlobPath, name, extension, GxFileType.PrivateAttribute); + string fileName = FileUtil.getTempFileName(_gxDbCommand.Conn.BlobPath, name, extension, GxFileType.Private); return getBLOBFile(id, extension, name, fileName, true); } @@ -127,7 +127,7 @@ private string getBLOBFile(int id, string extension, string name, string fileNam } } if (temporary) - GXFileWatcher.Instance.AddTemporaryFile(new GxFile(_gxDbCommand.Conn.BlobPath, new GxFileInfo(fileName, _gxDbCommand.Conn.BlobPath), GxFileType.PrivateAttribute), _gxDbCommand.Conn.DataStore.Context); + GXFileWatcher.Instance.AddTemporaryFile(new GxFile(_gxDbCommand.Conn.BlobPath, new GxFileInfo(fileName, _gxDbCommand.Conn.BlobPath), GxFileType.Private), _gxDbCommand.Conn.DataStore.Context); fileName = new FileInfo(fileName).FullName; } catch (IOException e) @@ -149,7 +149,7 @@ public string getMultimediaFile(int id, string gxdbFileUri) string filePath = PathUtil.SafeCombine(_gxDbCommand.Conn.MultimediaPath, fileName); try { - GxFile file = new GxFile(string.Empty, filePath, GxFileType.PublicAttribute); + GxFile file = new GxFile(string.Empty, filePath, GxFileType.Public); if (file.Exists()) { return filePath; @@ -331,13 +331,13 @@ public string getBLOBFile(int id) public string getBLOBFile(int id, string extension, string name) { - string fileName = FileUtil.getTempFileName(_gxDbCommand.Conn.BlobPath, name, extension, GxFileType.PrivateAttribute); + string fileName = FileUtil.getTempFileName(_gxDbCommand.Conn.BlobPath, name, extension, GxFileType.Private); String value = getBLOBFile(id, extension, name, fileName, true); TraceRow("getBLOBFile - index : ", id.ToString(), " value:", (value!=null ? value.ToString() : string.Empty)); return value; } - private string getBLOBFile(int id, string extension, string name, string fileName, bool temporary, GxFileType fileType = GxFileType.PrivateAttribute) + private string getBLOBFile(int id, string extension, string name, string fileName, bool temporary, GxFileType fileType = GxFileType.Private) { GxFile file = null; Stream fs = null; @@ -433,7 +433,7 @@ public string getMultimediaFile(int id, string gxdbFileUri) try { - GxFile file = new GxFile(string.Empty, filePath, GxFileType.PublicAttribute); + GxFile file = new GxFile(string.Empty, filePath, GxFileType.Public); if (file.Exists()) { @@ -441,7 +441,7 @@ public string getMultimediaFile(int id, string gxdbFileUri) } else { - return getBLOBFile(id, FileUtil.GetFileType(gxdbFileUri), FileUtil.GetFileName(gxdbFileUri), filePath, false, GxFileType.PublicAttribute); + return getBLOBFile(id, FileUtil.GetFileType(gxdbFileUri), FileUtil.GetFileName(gxdbFileUri), filePath, false, GxFileType.Public); } } catch (ArgumentException) @@ -617,7 +617,7 @@ public void SetParameterMultimedia(int id, string image_gxi, string image, strin { try { - multimediaUri = ServiceFactory.GetExternalProvider().Copy(image_gxi, GXDbFile.GenerateUri(image_gxi, !GXDbFile.HasToken(image_gxi), false), tableName, fieldName, GxFileType.PublicAttribute); + multimediaUri = ServiceFactory.GetExternalProvider().Copy(image_gxi, GXDbFile.GenerateUri(image_gxi, !GXDbFile.HasToken(image_gxi), false), tableName, fieldName, GxFileType.Public); GXLogging.Debug(log, "Copy file already in ExternalProvider:", multimediaUri); } catch (Exception ex) @@ -726,7 +726,7 @@ private static string PushToExternalProvider(Stream fileStream, string externalF string multimediaUri; using (fileStream) { - multimediaUri = ServiceFactory.GetExternalProvider().Save(fileStream, externalFileName, tableName, fieldName, GxFileType.PublicAttribute); + multimediaUri = ServiceFactory.GetExternalProvider().Save(fileStream, externalFileName, tableName, fieldName, GxFileType.Public); GXLogging.Debug(log, "Upload file to ExternalProvider:", multimediaUri); } diff --git a/dotnet/src/dotnetframework/GxClasses/Domain/GXFileIO.cs b/dotnet/src/dotnetframework/GxClasses/Domain/GXFileIO.cs index 8fe6a1f61..bfb0243e8 100644 --- a/dotnet/src/dotnetframework/GxClasses/Domain/GXFileIO.cs +++ b/dotnet/src/dotnetframework/GxClasses/Domain/GXFileIO.cs @@ -432,12 +432,14 @@ public GxExternalFileInfo(string storageObjectFullname, ExternalProvider provide _url = storageObjectFullname; } else { - if (fileType.HasFlag(GxFileType.Attribute)) //Attributes multimedia consider Storage Provider Folder + if (fileType.HasFlag(GxFileType.Private)) //Attributes multimedia consider Storage Provider Folder { _url = provider.GetBaseURL() + storageObjectFullname; _name = _url.Replace(provider.StorageUri, string.Empty); if (_name.StartsWith("/")) + { _name = _name.Substring(1, _name.Length - 1); + } } } _fileTypeAtt = fileType; @@ -640,11 +642,9 @@ public Stream GetStream() [Flags] public enum GxFileType { - Public = 0, - Private = 1, - Attribute = 2, - PublicAttribute = Attribute | Public, - PrivateAttribute = Attribute | Private, + Default = 0, + Public = 1, + Private = 2 } public class GxFile diff --git a/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderBase.cs b/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderBase.cs new file mode 100644 index 000000000..1bef72a61 --- /dev/null +++ b/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderBase.cs @@ -0,0 +1,111 @@ +using System; +using GeneXus.Encryption; +using log4net; + +namespace GeneXus.Services +{ + public abstract class ExternalProviderBase + { + static readonly ILog logger = log4net.LogManager.GetLogger(typeof(ExternalProviderBase)); + + private GXService service; + + protected static String DEFAULT_ACL = "DEFAULT_ACL"; + protected static String DEFAULT_EXPIRATION = "DEFAULT_EXPIRATION"; + protected static String FOLDER = "FOLDER_NAME"; + protected static String DEFAULT_ACL_DEPRECATED = "STORAGE_PROVIDER_DEFAULT_ACL"; + protected static String DEFAULT_EXPIRATION_DEPRECATED = "STORAGE_PROVIDER_DEFAULT_EXPIRATION"; + protected const int DefaultExpirationMinutes = 24 * 60; + + + protected GxFileType defaultAcl = GxFileType.Private; + + public ExternalProviderBase() + { + Initialize(); + } + + public ExternalProviderBase(GXService s) + { + this.service = s; + Initialize(); + } + + public abstract String GetName(); + + private void Initialize() + { + String aclS = GetPropertyValue(DEFAULT_ACL, DEFAULT_ACL_DEPRECATED, ""); + if (!String.IsNullOrEmpty(aclS)) + { + GxFileType.TryParse(aclS, out this.defaultAcl); + } + } + + protected String GetEncryptedPropertyValue(String propertyName, String alternativePropertyName) + { + String value = GetEncryptedPropertyValue(propertyName, alternativePropertyName, null); + if (value == null) + { + String errorMessage = String.Format($"Service configuration error - Property name {ResolvePropertyName(propertyName)} must be defined"); + logger.Fatal(errorMessage); + throw new Exception(errorMessage); + } + return value; + } + + protected String GetEncryptedPropertyValue(String propertyName, String alternativePropertyName, String defaultValue) + { + String value = GetPropertyValue(propertyName, alternativePropertyName, defaultValue); + if (!String.IsNullOrEmpty(value)) + { + try + { + value = CryptoImpl.Decrypt(value); + } + catch (Exception) + { + logger.Warn($"Could not decrypt property name: {ResolvePropertyName(propertyName)}"); + } + } + return value; + } + + protected String GetPropertyValue(String propertyName, String alternativePropertyName) + { + String value = GetPropertyValue(propertyName, alternativePropertyName, null); + if (value == null) + { + String errorMessage = String.Format($"Service configuration error - Property name {ResolvePropertyName(propertyName)} must be defined"); + logger.Fatal(errorMessage); + throw new Exception(errorMessage); + } + return value; + } + + protected String GetPropertyValue(String propertyName, String alternativePropertyName, String defaultValue) + { + propertyName = ResolvePropertyName(propertyName); + String value = Environment.GetEnvironmentVariable(propertyName); + if (String.IsNullOrEmpty(value)) + { + value = Environment.GetEnvironmentVariable(alternativePropertyName); + } + if (this.service != null) + { + value = this.service.Properties.Get(propertyName); + if (String.IsNullOrEmpty(value)) + { + value = this.service.Properties.Get(alternativePropertyName); + } + } + return !String.IsNullOrEmpty(value) ? value : defaultValue; + } + + protected String ResolvePropertyName(String propertyName) + { + return String.Format("STORAGE_%s_%s", GetName(), propertyName); + } + + } +} From bb1422d708a2983bc81adf994c2d7ba3439696dc Mon Sep 17 00:00:00 2001 From: Gonzalo Date: Thu, 10 Jun 2021 18:38:42 -0300 Subject: [PATCH 03/33] Improvements --- .../Configuration/ExternalStorage.cs | 40 +-- .../dotnetframework/GxClasses/Core/GXUtils.cs | 6 +- .../GxClasses/Core/GXUtilsCommon.cs | 135 ++++--- .../GxClasses/Data/GXDataNTierADO.cs | 16 +- .../GxClasses/Domain/GXFileIO.cs | 12 +- .../Services/Storage/ExternalProviderBase.cs | 33 +- .../src/dotnetframework/GxExcel/GxExcelI.cs | 2 +- .../Storage/GXAmazonS3/ExternalProviderS3.cs | 126 ++++--- .../AzureStorageExternalProvider.cs | 37 +- .../GXGoogleCloud/ExternalProviderGoogle.cs | 72 ++-- .../test/DotNetUnitTest/DotNetUnitTest.csproj | 19 +- .../ExternalProviderAzureTest.cs | 14 + .../ExternalProviderGoogleTest.cs | 15 + .../ExternalProviderS3Test.cs | 13 + .../ExternalProvider/ExternalProviderTest.cs | 339 ++++++++++++++++++ .../Properties/launchSettings.json | 11 + dotnet/test/DotNetUnitTest/resources/text.txt | 1 + 17 files changed, 669 insertions(+), 222 deletions(-) create mode 100644 dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderAzureTest.cs create mode 100644 dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderGoogleTest.cs create mode 100644 dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderS3Test.cs create mode 100644 dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs create mode 100644 dotnet/test/DotNetUnitTest/Properties/launchSettings.json create mode 100644 dotnet/test/DotNetUnitTest/resources/text.txt diff --git a/dotnet/src/dotnetframework/GxClasses/Configuration/ExternalStorage.cs b/dotnet/src/dotnetframework/GxClasses/Configuration/ExternalStorage.cs index 37b103595..d37cfcfbb 100644 --- a/dotnet/src/dotnetframework/GxClasses/Configuration/ExternalStorage.cs +++ b/dotnet/src/dotnetframework/GxClasses/Configuration/ExternalStorage.cs @@ -95,7 +95,7 @@ public bool Connect(string profileName, GXProperties properties, ref GxStoragePr private void preprocess(String name, GXProperties properties) { - string className = null; + string className; switch (name) { @@ -104,37 +104,27 @@ private void preprocess(String name, GXProperties properties) className = "GeneXus.Storage.GXAmazonS3.ExternalProviderS3"; SetDefaultProperty(properties, "STORAGE_PROVIDER_REGION", "us-east-1"); SetDefaultProperty(properties, "STORAGE_ENDPOINT", "s3.amazonaws.com"); - SetEncryptProperty(properties, "STORAGE_PROVIDER_ACCESSKEYID"); - SetEncryptProperty(properties, "STORAGE_PROVIDER_SECRETACCESSKEY"); - SetEncryptProperty(properties, "BUCKET_NAME"); + SetEncryptedProperty(properties, "STORAGE_PROVIDER_ACCESSKEYID"); + SetEncryptedProperty(properties, "STORAGE_PROVIDER_SECRETACCESSKEY"); + SetEncryptedProperty(properties, "BUCKET_NAME"); break; case "AZURESTORAGE": className = "GeneXus.Storage.GXAzureStorage.AzureStorageExternalProvider"; - SetEncryptProperty(properties, "PUBLIC_CONTAINER_NAME"); - SetEncryptProperty(properties, "PRIVATE_CONTAINER_NAME"); - SetEncryptProperty(properties, "ACCOUNT_NAME"); - SetEncryptProperty(properties, "ACCESS_KEY"); + SetEncryptedProperty(properties, "PUBLIC_CONTAINER_NAME"); + SetEncryptedProperty(properties, "PRIVATE_CONTAINER_NAME"); + SetEncryptedProperty(properties, "ACCOUNT_NAME"); + SetEncryptedProperty(properties, "ACCESS_KEY"); break; - - case "BLUEMIXSTORAGE": - className = "GeneXus.Storage.GXBluemix.ExternalProviderBluemix"; - SetDefaultProperty(properties, "SERVER_URL", "https://identity.open.softlayer.com"); - SetDefaultProperty(properties, "STORAGE_PROVIDER_REGION", "dallas"); - SetEncryptProperty(properties, "PUBLIC_BUCKET_NAME"); - SetEncryptProperty(properties, "PRIVATE_BUCKET_NAME"); - SetEncryptProperty(properties, "STORAGE_PROVIDER_USER"); - SetEncryptProperty(properties, "STORAGE_PROVIDER_PASSWORD"); - break; - + //case "BOX": // className = "{class}"; // break; case "GOOGLE": className = "GeneXus.Storage.GXGoogleCloud.ExternalProviderGoogle"; - SetEncryptProperty(properties, "KEY"); - SetEncryptProperty(properties, "BUCKET_NAME"); + SetEncryptedProperty(properties, "KEY"); + SetEncryptedProperty(properties, "BUCKET_NAME"); break; //case "IBMCOS": @@ -146,9 +136,9 @@ private void preprocess(String name, GXProperties properties) case "OPENSTACKSTORAGE": className = "GeneXus.Storage.GXOpenStack.ExternalProviderOpenStack"; - SetEncryptProperty(properties, "BUCKET_NAME"); - SetEncryptProperty(properties, "STORAGE_PROVIDER_USER"); - SetEncryptProperty(properties, "STORAGE_PROVIDER_PASSWORD"); + SetEncryptedProperty(properties, "BUCKET_NAME"); + SetEncryptedProperty(properties, "STORAGE_PROVIDER_USER"); + SetEncryptedProperty(properties, "STORAGE_PROVIDER_PASSWORD"); break; default: @@ -171,7 +161,7 @@ private void SetDefaultProperty(GXProperties properties, String prop, String val properties.Set(prop, value); } - private void SetEncryptProperty(GXProperties properties, String prop) + private void SetEncryptedProperty(GXProperties properties, String prop) { String value = properties.Get(prop); if (string.IsNullOrEmpty(value)) diff --git a/dotnet/src/dotnetframework/GxClasses/Core/GXUtils.cs b/dotnet/src/dotnetframework/GxClasses/Core/GXUtils.cs index 20e1ce69d..e4354f935 100644 --- a/dotnet/src/dotnetframework/GxClasses/Core/GXUtils.cs +++ b/dotnet/src/dotnetframework/GxClasses/Core/GXUtils.cs @@ -1120,7 +1120,7 @@ public bool Upload(string filefullpath, string storageobjectfullname, GxFile upl { storageobjectfullname = Path.GetFileName(filefullpath); } - string url = provider.Upload(filefullpath, storageobjectfullname, GxFileType.Public); + string url = provider.Upload(filefullpath, storageobjectfullname, GxFileType.PublicRead); uploadedFile.FileInfo = new GxExternalFileInfo(storageobjectfullname, url, provider); return true; } @@ -1160,7 +1160,7 @@ public bool Download(string storageobjectfullname, GxFile localFile, GXBaseColle destFileName = localFile.GetAbsoluteName(); else destFileName = Path.Combine(GxContext.StaticPhysicalPath(), localFile.Source); - provider.Download(storageobjectfullname, destFileName, GxFileType.Public); + provider.Download(storageobjectfullname, destFileName, GxFileType.PublicRead); return true; } catch (Exception ex) @@ -1197,7 +1197,7 @@ public bool Get(string storageobjectfullname, GxFile externalFile, GXBaseCollect try { ValidProvider(); - string url = provider.Get(storageobjectfullname, GxFileType.Public, 0); + string url = provider.Get(storageobjectfullname, GxFileType.PublicRead, 0); if (String.IsNullOrEmpty(url)) { GXUtil.ErrorToMessages("Get Error", "File doesn't exists", messages); diff --git a/dotnet/src/dotnetframework/GxClasses/Core/GXUtilsCommon.cs b/dotnet/src/dotnetframework/GxClasses/Core/GXUtilsCommon.cs index 98ca76aa8..a3dc44cb7 100644 --- a/dotnet/src/dotnetframework/GxClasses/Core/GXUtilsCommon.cs +++ b/dotnet/src/dotnetframework/GxClasses/Core/GXUtilsCommon.cs @@ -2630,19 +2630,19 @@ static public string CMonth(DateTime dt, string lang) public string Time() { return DateTime.Now.ToString(TimeFormatFromSize(8, -1, ":"), cultureInfo); - } + } - static public DateTime AddMth(DateTime dt, int cantMonths) - { - if (dt == nullDate && cantMonths < 0) - return nullDate; - return dt.AddMonths(cantMonths); - } - static public DateTime AddYr(DateTime dt, int cantYears) - { - if (dt == nullDate && cantYears < 0) - return nullDate; - return dt.AddYears(cantYears); + static public DateTime AddMth(DateTime dt, int cantMonths) + { + if (dt == nullDate) + return nullDate; + return dt.AddMonths(cantMonths); + } + static public DateTime AddYr(DateTime dt, int cantYears) + { + if (dt == nullDate) + return nullDate; + return dt.AddYears(cantYears); } static public DateTime DateEndOfMonth(DateTime dt) { @@ -2691,29 +2691,30 @@ static public long TDiff(DateTime dtMinu, DateTime dtSust) } static public int DDiff(DateTime dtMinu, DateTime dtSust) { - return Convert.ToInt32((dtMinu - dtSust).TotalDays); - } - static public DateTime TAdd(DateTime dt, int seconds) - { - if (dt == nullDate && seconds < 0) - return nullDate; - return dt.AddSeconds(seconds); - } - static public DateTime TAddMs(DateTime dt, double seconds) - { - if (dt == nullDate && seconds < 0) - return nullDate; - if (seconds % 1 == 0) - return dt.AddSeconds((int)seconds); - else - return dt.AddMilliseconds(seconds * 1000); - } - static public DateTime DAdd(DateTime dt, int days) - { - if (dt == nullDate && days < 0) - return nullDate; - return dt.AddDays(days); - } + return Convert.ToInt32((dtMinu - dtSust).TotalDays); + } + static public DateTime TAdd(DateTime dt, int seconds) + { + if (dt == nullDate) + return nullDate; + return dt.AddSeconds(seconds); + } + static public DateTime TAddMs(DateTime dt, double seconds) + { + + if (dt == nullDate) + return nullDate; + if (seconds % 1 == 0) + return dt.AddSeconds((int)seconds); + else + return dt.AddMilliseconds(seconds * 1000); + } + static public DateTime DAdd(DateTime dt, int days) + { + if (dt == nullDate) + return nullDate; + return dt.AddDays(days); + } public DateTime CToT(string strDate, int picFmt, int ampmFmt) { if (isNullDateTime(strDate, picFmt, ampmFmt)) @@ -3271,6 +3272,8 @@ static public DateTime FromTimeZone(DateTime dt, String sTZ, IGxContext context) public class FileUtil { static readonly ILog log = log4net.LogManager.GetLogger(typeof(GeneXus.Utils.FileUtil)); + + private static object syncObj = new object(); public static byte DeleteFile(string fileName) { try @@ -3363,10 +3366,30 @@ public static string UriToPath(string uriString) return uriString; } } - public static string getTempFileName(string baseDir, string name="", string extension="tmp", GxFileType fileType = GxFileType.Public) + public static string getTempFileName(string baseDir, string name, string extension, GxFileType fileType = GxFileType.PublicRead) { - name = FixFileName(FileUtil.FileNamePrettify(name), string.Empty); - return tempFileName(baseDir, name, extension); + String fileName; + name = FileUtil.FileNamePrettify(name); + + lock (syncObj) + { + + name = FixFileName(name, string.Empty); + if (string.IsNullOrEmpty(name)) + { + fileName = tempFileName(baseDir, name, extension); + } + else + { + fileName = PathUtil.SafeCombine(baseDir, name + "." + extension); + } + GxFile file = new GxFile(baseDir, fileName, fileType); + while (file.FileInfo.Exist(fileName)) + { + fileName = tempFileName(baseDir, name, extension); + } + } + return fileName.Trim(); } private static string tempFileName(string baseDir, string name, string extension) @@ -3405,12 +3428,6 @@ public static string GetFileType(string FileName) { if (FileName.Trim().Length == 0) return string.Empty; - - if (GxUploadHelper.IsUpload(FileName)) - { - return new GxFile(string.Empty, FileName, GxFileType.Private).GetExtension(); - } - string extension = string.Empty; try { @@ -3433,28 +3450,16 @@ public static string GetFileType(string FileName) public static string FileNamePrettify(string s) { - if (!string.IsNullOrEmpty(s)) - { - string str = GXUtil.RemoveDiacritics(s.Trim().ToLower()); //remove accents - str = Regex.Replace(str, @"\s+", " ").Trim(); // convert multiple spaces into one space - str = Regex.Replace(str, @"\s", "-"); // //Replace spaces by dashes - return str; - } - else - { - return s; - } + string str = GXUtil.RemoveDiacritics(s.Trim().ToLower()); //remove accents + str = Regex.Replace(str, @"\s+", " ").Trim(); // convert multiple spaces into one space + str = Regex.Replace(str, @"\s", "-"); // //Replace spaces by dashes + return str; } public static string GetFileName(string FileName) { if (FileName.Trim().Length == 0) return ""; - - if (GxUploadHelper.IsUpload(FileName)) - { - FileName = new GxFile(string.Empty, FileName, GxFileType.Private).GetName(); - } try { return Path.GetFileNameWithoutExtension(FileName);//FileNames with URI or local (exist) @@ -3673,10 +3678,6 @@ public static string GetValidFileName(string path, string replaceStr) if (!IsValidFilePath(path)) validPath = GetValidPath(path, "_"); - Uri result; - if (Uri.TryCreate(validPath, UriKind.Absolute, out result)) - validPath = result.LocalPath; - string fileName = Path.GetFileName(validPath); if (!IsValidFileName(fileName)) { @@ -5264,7 +5265,7 @@ public static string ResolveUri(string uriString, bool absUrl, IGxContext contex string basePath = Path.Combine(Path.Combine(Preferences.getBLOB_PATH(), MultimediaDirectory)); try { - GxFile file = new GxFile(string.Empty, PathUtil.SafeCombine(basePath, fileName), GxFileType.Public); + GxFile file = new GxFile(string.Empty, PathUtil.SafeCombine(basePath, fileName), GxFileType.PublicRead); return PathToUrl(file.GetURI(), absUrl, context); } catch (ArgumentException ex) @@ -5315,10 +5316,6 @@ public static string GetUriFromFile(string name, string type, string path) { if (String.IsNullOrEmpty(name) && String.IsNullOrEmpty(type) && !String.IsNullOrEmpty(path)) { - if (GxUploadHelper.IsUpload(path)) - { - return new GxFile(string.Empty, path, GxFileType.Private).GetName(); - } string fromPathType = Path.GetExtension(path); if (!String.IsNullOrEmpty(fromPathType) && fromPathType != "tmp") { diff --git a/dotnet/src/dotnetframework/GxClasses/Data/GXDataNTierADO.cs b/dotnet/src/dotnetframework/GxClasses/Data/GXDataNTierADO.cs index e1904d8bd..10befd758 100644 --- a/dotnet/src/dotnetframework/GxClasses/Data/GXDataNTierADO.cs +++ b/dotnet/src/dotnetframework/GxClasses/Data/GXDataNTierADO.cs @@ -149,7 +149,7 @@ public string getMultimediaFile(int id, string gxdbFileUri) string filePath = PathUtil.SafeCombine(_gxDbCommand.Conn.MultimediaPath, fileName); try { - GxFile file = new GxFile(string.Empty, filePath, GxFileType.Public); + GxFile file = new GxFile(string.Empty, filePath, GxFileType.PublicRead); if (file.Exists()) { return filePath; @@ -433,7 +433,7 @@ public string getMultimediaFile(int id, string gxdbFileUri) try { - GxFile file = new GxFile(string.Empty, filePath, GxFileType.Public); + GxFile file = new GxFile(string.Empty, filePath, GxFileType.PublicRead); if (file.Exists()) { @@ -441,7 +441,7 @@ public string getMultimediaFile(int id, string gxdbFileUri) } else { - return getBLOBFile(id, FileUtil.GetFileType(gxdbFileUri), FileUtil.GetFileName(gxdbFileUri), filePath, false, GxFileType.Public); + return getBLOBFile(id, FileUtil.GetFileType(gxdbFileUri), FileUtil.GetFileName(gxdbFileUri), filePath, false, GxFileType.PublicRead); } } catch (ArgumentException) @@ -617,7 +617,7 @@ public void SetParameterMultimedia(int id, string image_gxi, string image, strin { try { - multimediaUri = ServiceFactory.GetExternalProvider().Copy(image_gxi, GXDbFile.GenerateUri(image_gxi, !GXDbFile.HasToken(image_gxi), false), tableName, fieldName, GxFileType.Public); + multimediaUri = ServiceFactory.GetExternalProvider().Copy(image_gxi, GXDbFile.GenerateUri(image_gxi, !GXDbFile.HasToken(image_gxi), false), tableName, fieldName, GxFileType.PublicRead); GXLogging.Debug(log, "Copy file already in ExternalProvider:", multimediaUri); } catch (Exception ex) @@ -635,7 +635,7 @@ public void SetParameterMultimedia(int id, string image_gxi, string image, strin using (var fileStream = new MemoryStream(new WebClient().DownloadData(image_gxi))) { //Cannot pass Http Stream directly, because some Providers (AWS S3) does not support Http Stream. - multimediaUri = ServiceFactory.GetExternalProvider().Save(fileStream, GXDbFile.GenerateUri(image_gxi, !GXDbFile.HasToken(image_gxi), false), tableName, fieldName, GxFileType.Public); + multimediaUri = ServiceFactory.GetExternalProvider().Save(fileStream, GXDbFile.GenerateUri(image_gxi, !GXDbFile.HasToken(image_gxi), false), tableName, fieldName, GxFileType.PublicRead); GXLogging.Debug(log, "Upload external file to ExternalProvider:", multimediaUri); } #pragma warning disable SYSLIB0014 // WebClient @@ -657,7 +657,7 @@ public void SetParameterMultimedia(int id, string image_gxi, string image, strin String fileName = PathUtil.GetValidFileName(fileFullName, "_"); using (fileStream) { - multimediaUri = ServiceFactory.GetExternalProvider().Save(fileStream, GXDbFile.GenerateUri(fileName, !GXDbFile.HasToken(fileName), false), tableName, fieldName, GxFileType.Public); + multimediaUri = ServiceFactory.GetExternalProvider().Save(fileStream, GXDbFile.GenerateUri(fileName, !GXDbFile.HasToken(fileName), false), tableName, fieldName, GxFileType.PublicRead); GXLogging.Debug(log, "Upload file (_gxi) to ExternalProvider:", multimediaUri); } } @@ -665,7 +665,7 @@ public void SetParameterMultimedia(int id, string image_gxi, string image, strin { try { - multimediaUri = ServiceFactory.GetExternalProvider().Copy(image, GXDbFile.GenerateUri(image_gxi, !GXDbFile.HasToken(image_gxi), false), tableName, fieldName, GxFileType.Public); + multimediaUri = ServiceFactory.GetExternalProvider().Copy(image, GXDbFile.GenerateUri(image_gxi, !GXDbFile.HasToken(image_gxi), false), tableName, fieldName, GxFileType.PublicRead); GXLogging.Debug(log, "Copy external file in ExternalProvider:", multimediaUri); } catch(Exception e) @@ -726,7 +726,7 @@ private static string PushToExternalProvider(Stream fileStream, string externalF string multimediaUri; using (fileStream) { - multimediaUri = ServiceFactory.GetExternalProvider().Save(fileStream, externalFileName, tableName, fieldName, GxFileType.Public); + multimediaUri = ServiceFactory.GetExternalProvider().Save(fileStream, externalFileName, tableName, fieldName, GxFileType.PublicRead); GXLogging.Debug(log, "Upload file to ExternalProvider:", multimediaUri); } diff --git a/dotnet/src/dotnetframework/GxClasses/Domain/GXFileIO.cs b/dotnet/src/dotnetframework/GxClasses/Domain/GXFileIO.cs index bfb0243e8..963328abf 100644 --- a/dotnet/src/dotnetframework/GxClasses/Domain/GXFileIO.cs +++ b/dotnet/src/dotnetframework/GxClasses/Domain/GXFileIO.cs @@ -209,7 +209,7 @@ public IGxFileInfo[] GetFiles(string searchPattern) searchPattern = ""; List files = _provider.GetFiles(_name, searchPattern); - GxExternalFileInfo[] externalFiles = files.Select(elem => new GxExternalFileInfo(elem, _provider, GxFileType.Public)).ToArray(); + GxExternalFileInfo[] externalFiles = files.Select(elem => new GxExternalFileInfo(elem, _provider, GxFileType.PublicRead)).ToArray(); return externalFiles; } @@ -412,7 +412,7 @@ public class GxExternalFileInfo : IGxFileInfo private string _name; private ExternalProvider _provider; private string _url; - private GxFileType _fileTypeAtt = GxFileType.Public; + private GxFileType _fileTypeAtt = GxFileType.PublicRead; public GxExternalFileInfo(ExternalProvider provider) { @@ -445,7 +445,7 @@ public GxExternalFileInfo(string storageObjectFullname, ExternalProvider provide _fileTypeAtt = fileType; } - public GxExternalFileInfo(string storageObjectFullname, string url, ExternalProvider provider, GxFileType fileType = GxFileType.Public) + public GxExternalFileInfo(string storageObjectFullname, string url, ExternalProvider provider, GxFileType fileType = GxFileType.PublicRead) { _name = storageObjectFullname; _provider = provider; @@ -643,7 +643,7 @@ public Stream GetStream() public enum GxFileType { Default = 0, - Public = 1, + PublicRead = 1, Private = 2 } @@ -670,13 +670,13 @@ public GxFile(string baseDirectory) _baseDirectory = _baseDirectory.Substring(0, _baseDirectory.Length - 1); } - public GxFile(string baseDirectory, IGxFileInfo file, GxFileType fileType = GxFileType.Public) + public GxFile(string baseDirectory, IGxFileInfo file, GxFileType fileType = GxFileType.PublicRead) : this(baseDirectory) { _file = file; } - public GxFile(string baseDirectory, string fileName, GxFileType fileType = GxFileType.Public) + public GxFile(string baseDirectory, string fileName, GxFileType fileType = GxFileType.PublicRead) : this(baseDirectory) { if (GxUploadHelper.IsUpload(fileName)) diff --git a/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderBase.cs b/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderBase.cs index 1bef72a61..a495f88a3 100644 --- a/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderBase.cs +++ b/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderBase.cs @@ -42,7 +42,7 @@ private void Initialize() } } - protected String GetEncryptedPropertyValue(String propertyName, String alternativePropertyName) + protected String GetEncryptedPropertyValue(String propertyName, String alternativePropertyName = null) { String value = GetEncryptedPropertyValue(propertyName, alternativePropertyName, null); if (value == null) @@ -71,7 +71,7 @@ protected String GetEncryptedPropertyValue(String propertyName, String alternati return value; } - protected String GetPropertyValue(String propertyName, String alternativePropertyName) + protected String GetPropertyValue(String propertyName, String alternativePropertyName = null) { String value = GetPropertyValue(propertyName, alternativePropertyName, null); if (value == null) @@ -85,26 +85,31 @@ protected String GetPropertyValue(String propertyName, String alternativePropert protected String GetPropertyValue(String propertyName, String alternativePropertyName, String defaultValue) { - propertyName = ResolvePropertyName(propertyName); - String value = Environment.GetEnvironmentVariable(propertyName); - if (String.IsNullOrEmpty(value)) - { - value = Environment.GetEnvironmentVariable(alternativePropertyName); - } - if (this.service != null) + String value = String.Empty; + value = string.IsNullOrEmpty(value) ? GetPropertyValueImpl(ResolvePropertyName(propertyName)) : value; + value = string.IsNullOrEmpty(value) ? GetPropertyValueImpl(propertyName) : value; + value = string.IsNullOrEmpty(value) ? GetPropertyValueImpl(alternativePropertyName) : value; + value = string.IsNullOrEmpty(value) ? defaultValue : value; + return value; + } + + private string GetPropertyValueImpl(string propertyName) + { + String value = String.Empty; + if (!string.IsNullOrEmpty(propertyName)) { - value = this.service.Properties.Get(propertyName); - if (String.IsNullOrEmpty(value)) + value = Environment.GetEnvironmentVariable(propertyName); + if (this.service != null) { - value = this.service.Properties.Get(alternativePropertyName); + value = this.service.Properties.Get(propertyName); } } - return !String.IsNullOrEmpty(value) ? value : defaultValue; + return value; } protected String ResolvePropertyName(String propertyName) { - return String.Format("STORAGE_%s_%s", GetName(), propertyName); + return $"STORAGE_{GetName()}_{propertyName}"; } } diff --git a/dotnet/src/dotnetframework/GxExcel/GxExcelI.cs b/dotnet/src/dotnetframework/GxExcel/GxExcelI.cs index 8317f9bc5..79168b9f3 100644 --- a/dotnet/src/dotnetframework/GxExcel/GxExcelI.cs +++ b/dotnet/src/dotnetframework/GxExcel/GxExcelI.cs @@ -144,7 +144,7 @@ public String Template string localTemplate = value; if (!Path.IsPathRooted(localTemplate)) localTemplate = Path.Combine(GxContext.StaticPhysicalPath(), localTemplate); - ServiceFactory.GetExternalProvider().Upload(localTemplate, template, GxFileType.Public); + ServiceFactory.GetExternalProvider().Upload(localTemplate, template, GxFileType.PublicRead); } } else if (!Path.IsPathRooted(value)) diff --git a/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs b/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs index ba2967ae3..54accf387 100644 --- a/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs +++ b/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs @@ -10,23 +10,40 @@ using System.Collections.Generic; using System.IO; using System.Threading; -using System.Threading.Tasks; + namespace GeneXus.Storage.GXAmazonS3 { - public class ExternalProviderS3 : ExternalProvider + public class ExternalProviderS3 : ExternalProviderBase, ExternalProvider { - const int PRIVATE_URL_MINUTES_EXPIRATION = 24 * 60; // 24 hours - const string ACCESS_KEY_ID = "STORAGE_PROVIDER_ACCESSKEYID"; - const string SECRET_ACCESS_KEY = "STORAGE_PROVIDER_SECRETACCESSKEY"; - const string REGION = "STORAGE_PROVIDER_REGION"; - const string ENDPOINT = "STORAGE_ENDPOINT"; - const string STORAGE_CUSTOM_ENDPOINT = "STORAGE_CUSTOM_ENDPOINT"; - const string STORAGE_CUSTOM_ENDPOINT_VALUE = "custom"; + public const string Name = "AWSS3"; + + const string ACCESS_KEY = "ACCESS_KEY"; + const string SECRET_ACCESS_KEY = "SECRET_KEY"; + const string STORAGE_CUSTOM_ENDPOINT = "CUSTOM_ENDPOINT"; + const string STORAGE_ENDPOINT = "ENDPOINT"; const string BUCKET = "BUCKET_NAME"; - const string FOLDER = "FOLDER_NAME"; + const string REGION = "REGION"; + const string STORAGE_CUSTOM_ENDPOINT_VALUE = "custom"; + const string DEFAULT_REGION = "us-east-1"; + [Obsolete("Use Property ACCESS_KEY instead", false)] + const string ACCESS_KEY_ID_DEPRECATED = "STORAGE_PROVIDER_ACCESSKEYID"; + [Obsolete("Use Property SECRET_ACCESS_KEY instead", false)] + const string SECRET_ACCESS_KEY_DEPRECATED = "STORAGE_PROVIDER_SECRETACCESSKEY"; + [Obsolete("Use Property REGION instead", false)] + const string REGION_DEPRECATED = "STORAGE_PROVIDER_REGION"; + [Obsolete("Use Property STORAGE_ENDPOINT instead", false)] + const string ENDPOINT_DEPRECATED = "STORAGE_ENDPOINT"; + [Obsolete("Use Property STORAGE_CUSTOM_ENDPOINT instead", false)] + const string STORAGE_CUSTOM_ENDPOINT_DEPRECATED = "STORAGE_CUSTOM_ENDPOINT"; + + [Obsolete("Use Property BUCKET instead", false)] + const string BUCKET_DEPRECATED = "BUCKET_NAME"; + [Obsolete("Use Property FOLDER instead", false)] + const string FOLDER_DEPRECATED = "FOLDER_NAME"; + string _storageUri; IAmazonS3 Client { get; set; } @@ -35,13 +52,12 @@ public class ExternalProviderS3 : ExternalProvider string Endpoint { get; set; } string Region { get; set; } - bool UseCustomEndpoint = false; bool ForcePathStyle = false; public string StorageUri { get { - return _storageUri; + return (ForcePathStyle)? $"{Endpoint}/": $"https://{Bucket}.{Endpoint}/"; } } @@ -50,41 +66,40 @@ public string GetBaseURL() return StorageUri + Folder + StorageUtils.DELIMITER; } - public ExternalProviderS3() - : this(ServiceFactory.GetGXServices().Get(GXServices.STORAGE_SERVICE)) - { + public ExternalProviderS3(): this(null) + { } - public ExternalProviderS3(GXService providerService) + public ExternalProviderS3(GXService providerService): base(providerService) { - string keyId = CryptoImpl.Decrypt(providerService.Properties.Get(ACCESS_KEY_ID)); - string keySecret = CryptoImpl.Decrypt(providerService.Properties.Get(SECRET_ACCESS_KEY)); + Initialize(); + } + + private void Initialize() { + string keyId = GetEncryptedPropertyValue(ACCESS_KEY, ACCESS_KEY_ID_DEPRECATED); + string keySecret = GetEncryptedPropertyValue(SECRET_ACCESS_KEY, SECRET_ACCESS_KEY_DEPRECATED); AWSCredentials credentials = null; if (!string.IsNullOrEmpty(keyId) && !string.IsNullOrEmpty(keySecret)) { credentials = new BasicAWSCredentials(keyId, keySecret); } - var region = Amazon.RegionEndpoint.GetBySystemName(providerService.Properties.Get(REGION)); - - string _endPoint = providerService.Properties.Get(ENDPOINT); - UseCustomEndpoint = !String.IsNullOrEmpty(_endPoint) && _endPoint.Equals(STORAGE_CUSTOM_ENDPOINT_VALUE, StringComparison.OrdinalIgnoreCase); - - - Endpoint = (UseCustomEndpoint)? providerService.Properties.Get(STORAGE_CUSTOM_ENDPOINT): _endPoint; - ForcePathStyle = UseCustomEndpoint; + var region = Amazon.RegionEndpoint.GetBySystemName(GetPropertyValue(REGION, REGION_DEPRECATED)); AmazonS3Config config = new AmazonS3Config() { - RegionEndpoint = region, - ForcePathStyle = ForcePathStyle + RegionEndpoint = region }; - - if (!string.IsNullOrEmpty(Endpoint) && !Endpoint.EndsWith(".amazonaws.com")) + + Endpoint = GetPropertyValue( STORAGE_ENDPOINT , ENDPOINT_DEPRECATED); + if (Endpoint == STORAGE_CUSTOM_ENDPOINT_VALUE) { + Endpoint = GetPropertyValue(STORAGE_CUSTOM_ENDPOINT, STORAGE_CUSTOM_ENDPOINT_DEPRECATED); + ForcePathStyle = true; + config.ForcePathStyle = ForcePathStyle; config.ServiceURL = Endpoint; } - + #if NETCORE if (credentials != null) { @@ -105,8 +120,8 @@ public ExternalProviderS3(GXService providerService) } #endif - Bucket = CryptoImpl.Decrypt(providerService.Properties.Get(BUCKET)); - Folder = providerService.Properties.Get(FOLDER); + Bucket = GetEncryptedPropertyValue(BUCKET, BUCKET_DEPRECATED); + Folder = GetPropertyValue(FOLDER, FOLDER_DEPRECATED); Region = region.SystemName; SetURI(); @@ -116,17 +131,13 @@ public ExternalProviderS3(GXService providerService) private void SetURI() { - if (UseCustomEndpoint) - { - _storageUri = $"{Endpoint}/{(ForcePathStyle ? Bucket + "/" : String.Empty)}"; - } - else if (Region == DEFAULT_REGION) + if (Region == DEFAULT_REGION) { _storageUri = $"https://{Bucket}.{Endpoint}/"; } else { - _storageUri = $"https://{Bucket}.{Endpoint.Replace("s3.amazonaws.com", String.Format("s3.{0}.amazonaws.com", Region.ToLower()))}/"; + _storageUri = $"https://{Bucket}.{Endpoint.Replace("s3.amazonaws.com", $"s3.{Region.ToLower()}.amazonaws.com")}/"; } } @@ -200,11 +211,11 @@ private void CreateBucket() PutBucketRequest request = new PutBucketRequest { BucketName = Bucket, - UseClientRegion = true, - // Every bucket is public - CannedACL = S3CannedACL.PublicRead + UseClientRegion = true }; - + if (defaultAcl == GxFileType.PublicRead) { + request.CannedACL = S3CannedACL.PublicRead; + } PutBucket(request); } } @@ -222,12 +233,12 @@ public string Upload(string localFile, string objectName, GxFileType fileType) return Get(objectName, fileType); } - private static bool IsPrivateUpload(GxFileType fileType) + private bool IsPrivateUpload(GxFileType fileType) { - return fileType.HasFlag(GxFileType.Private); + return GetCannedACL(fileType) != S3CannedACL.PublicRead; } - public string Get(string objectName, GxFileType fileType, int urlMinutes = PRIVATE_URL_MINUTES_EXPIRATION) + public string Get(string objectName, GxFileType fileType, int urlMinutes = DefaultExpirationMinutes) { bool isPrivate = IsPrivateUpload(fileType); if (Exists(objectName, fileType)) @@ -274,7 +285,7 @@ public void Delete(string objectName, GxFileType fileType) //https://github.com/aws/aws-sdk-net/blob/master/sdk/src/Services/S3/Custom/_bcl/IO/S3FileInfo.cs public bool Exists(string objectName, GxFileType fileType) { - bool exists = true; + bool exists; try { exists = new S3FileInfo(Client, Bucket, objectName).Exists; @@ -307,9 +318,19 @@ public string Copy(string objectName, GxFileType sourceFileType, string newName, return StorageUri + StorageUtils.EncodeUrl(newName); } - private static S3CannedACL GetCannedACL(GxFileType destFileType) + private S3CannedACL GetCannedACL(GxFileType acl) { - return (destFileType.HasFlag(GxFileType.Private)) ? S3CannedACL.Private : S3CannedACL.PublicRead; + if (acl == GxFileType.Default) + { + acl = this.defaultAcl; + } + + S3CannedACL accessControl = S3CannedACL.Private; + if (acl == GxFileType.PublicRead) + { + accessControl = S3CannedACL.PublicRead; + } + return accessControl; } public string Upload(string fileName, Stream stream, GxFileType destFileType) @@ -418,7 +439,7 @@ public void RenameDirectory(string directoryName, string newDirectoryName) if (file.Type == FileSystemType.Directory) RenameDirectory(directoryName + "\\" + file.Name, newDirectoryName + "\\" + file.Name); else - Rename(directoryName.Replace("\\", StorageUtils.DELIMITER) + StorageUtils.DELIMITER + file.Name, newDirectoryName.Replace("\\", StorageUtils.DELIMITER) + StorageUtils.DELIMITER + file.Name, GxFileType.Public); + Rename(directoryName.Replace("\\", StorageUtils.DELIMITER) + StorageUtils.DELIMITER + file.Name, newDirectoryName.Replace("\\", StorageUtils.DELIMITER) + StorageUtils.DELIMITER + file.Name, GxFileType.PublicRead); } s3DirectoryInfo.Delete(); } @@ -496,5 +517,10 @@ public bool GetObjectNameFromURL(string url, out string objectName) objectName = null; return false; } + + public override string GetName() + { + return Name; + } } } diff --git a/dotnet/src/dotnetframework/Providers/Storage/GXAzureStorage/AzureStorageExternalProvider.cs b/dotnet/src/dotnetframework/Providers/Storage/GXAzureStorage/AzureStorageExternalProvider.cs index 811b079db..1b46ada6c 100644 --- a/dotnet/src/dotnetframework/Providers/Storage/GXAzureStorage/AzureStorageExternalProvider.cs +++ b/dotnet/src/dotnetframework/Providers/Storage/GXAzureStorage/AzureStorageExternalProvider.cs @@ -14,8 +14,10 @@ namespace GeneXus.Storage.GXAzureStorage { - public class AzureStorageExternalProvider : ExternalProvider + public class AzureStorageExternalProvider : ExternalProviderBase, ExternalProvider { + public static String Name = "AZUREBS"; //Azure Blob Storage + const string ACCOUNT_NAME = "ACCOUNT_NAME"; const string ACCESS_KEY = "ACCESS_KEY"; const string PUBLIC_CONTAINER = "PUBLIC_CONTAINER_NAME"; @@ -26,22 +28,35 @@ public class AzureStorageExternalProvider : ExternalProvider CloudBlobContainer PublicContainer { get; set; } CloudBlobContainer PrivateContainer { get; set; } CloudBlobClient Client { get; set; } + public string StorageUri { get { return $"https://{Account}.blob.core.windows.net"; } } - public AzureStorageExternalProvider() - : this(ServiceFactory.GetGXServices().Get(GXServices.STORAGE_SERVICE)) { } + public override string GetName() + { + return Name; + } + + public AzureStorageExternalProvider() : this(null) + { + } + + public AzureStorageExternalProvider(GXService providerService) : base(providerService) + { + Initialize(); + } - public AzureStorageExternalProvider(GXService providerService) + private void Initialize() { - Account = CryptoImpl.Decrypt(providerService.Properties.Get(ACCOUNT_NAME)); - Key = CryptoImpl.Decrypt(providerService.Properties.Get(ACCESS_KEY)); + Account = GetEncryptedPropertyValue(ACCOUNT_NAME); + Key = GetEncryptedPropertyValue(ACCESS_KEY); - string publicContainer = CryptoImpl.Decrypt(providerService.Properties.Get(PUBLIC_CONTAINER)); - string privateContainer = CryptoImpl.Decrypt(providerService.Properties.Get(PRIVATE_CONTAINER)); + string publicContainer = GetEncryptedPropertyValue(PUBLIC_CONTAINER); + string privateContainer = GetEncryptedPropertyValue(PRIVATE_CONTAINER); + StorageCredentials credentials = new StorageCredentials(Account, Key); CloudStorageAccount storageAccount = new CloudStorageAccount(credentials, true); @@ -254,15 +269,15 @@ public void RenameDirectory(string directoryName, string newDirectoryName) { CloudBlockBlob blob = (CloudBlockBlob)item; fileName = Path.GetFileName(blob.Name); - Copy(blob.Name, GxFileType.Public, newDirectoryName + fileName, GxFileType.Public); - Delete(blob.Name, GxFileType.Public); + Copy(blob.Name, GxFileType.PublicRead, newDirectoryName + fileName, GxFileType.PublicRead); + Delete(blob.Name, GxFileType.PublicRead); } else if (item.GetType() == typeof(CloudPageBlob)) { CloudPageBlob pageBlob = (CloudPageBlob)item; fileName = Path.GetFileName(pageBlob.Name); - Copy(directoryName + fileName, GxFileType.Public, newDirectoryName + fileName, GxFileType.Public); + Copy(directoryName + fileName, GxFileType.PublicRead, newDirectoryName + fileName, GxFileType.PublicRead); } else if (item.GetType() == typeof(CloudBlobDirectory)) diff --git a/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs b/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs index 7e72b04b7..c53ab783f 100644 --- a/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs +++ b/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs @@ -15,15 +15,16 @@ namespace GeneXus.Storage.GXGoogleCloud { - public class ExternalProviderGoogle : ExternalProvider + public class ExternalProviderGoogle : ExternalProviderBase, ExternalProvider { - const int BUCKET_EXISTS = 409; + public static String Name = "GOOGLECS"; //Google Cloud Storage + const int BUCKET_EXISTS = 409; const int OBJECT_NOT_FOUND = 404; const string APPLICATION_NAME = "APPLICATION_NAME"; const string PROJECT_ID = "PROJECT_ID"; const string KEY = "KEY"; const string BUCKET = "BUCKET_NAME"; - const string FOLDER = "FOLDER_NAME"; + StorageClient Client { get; set; } StorageService Service { get; set; } @@ -36,35 +37,48 @@ public string StorageUri get { return $"https://{Bucket}.storage.googleapis.com/"; } } - public ExternalProviderGoogle() - : this(ServiceFactory.GetGXServices().Get(GXServices.STORAGE_SERVICE)) - { - } + public override string GetName() + { + return Name; - public ExternalProviderGoogle(GXService providerService) - { + } + + public ExternalProviderGoogle() : this(null) + { + } + + public ExternalProviderGoogle(GXService providerService) : base(providerService) + { + Initialize(); + } + + + private void Initialize() + { GoogleCredential credentials; - using (Stream stream = KeyStream(CryptoImpl.Decrypt(providerService.Properties.Get(KEY)))) - { - credentials = GoogleCredential.FromStream(stream).CreateScoped(StorageService.Scope.CloudPlatform); - } + string key = GetEncryptedPropertyValue(KEY); - using (Stream stream = KeyStream(CryptoImpl.Decrypt(providerService.Properties.Get(KEY)))) + using (Stream stream = KeyStream(key)) { - Signer = UrlSigner.FromServiceAccountData(stream); - } + credentials = GoogleCredential.FromStream(stream).CreateScoped(StorageService.Scope.CloudPlatform); + } + + using (Stream stream = KeyStream(key)) + { + Signer = UrlSigner.FromServiceAccountData(stream); + } - Client = StorageClient.Create(credentials); + Client = StorageClient.Create(credentials); Service = new StorageService(new BaseClientService.Initializer() { HttpClientInitializer = credentials, - ApplicationName = providerService.Properties.Get(APPLICATION_NAME) + ApplicationName = GetPropertyValue(APPLICATION_NAME) }); - Project = providerService.Properties.Get(PROJECT_ID); - Bucket = CryptoImpl.Decrypt(providerService.Properties.Get(BUCKET)); - Folder = providerService.Properties.Get(FOLDER); + Bucket = GetEncryptedPropertyValue(BUCKET); + Project = GetPropertyValue(PROJECT_ID); + Folder = GetPropertyValue(FOLDER); CreateBucket(); CreateFolder(Folder); @@ -176,7 +190,7 @@ public string Upload(string localFile, string objectName, GxFileType fileType) using (FileStream stream = new FileStream(localFile, FileMode.Open)) { Google.Apis.Storage.v1.Data.Object obj = Client.UploadObject(Bucket, objectName, "application/octet-stream", stream, GetUploadOptions(fileType)); - return obj.MediaLink; + return GetURL(objectName, fileType, DefaultExpirationMinutes); } } @@ -279,12 +293,12 @@ public void DeleteDirectory(string directoryName) foreach (Google.Apis.Storage.v1.Data.Object item in Client.ListObjects(Bucket, directoryName)) { if (!item.Name.EndsWith(StorageUtils.DELIMITER)) - Delete(item.Name, GxFileType.Public); + Delete(item.Name, GxFileType.PublicRead); } foreach (string subdir in GetSubDirectories(directoryName)) DeleteDirectory(subdir); - if (Exists(directoryName, GxFileType.Public)) - Delete(directoryName, GxFileType.Public); + if (Exists(directoryName, GxFileType.PublicRead)) + Delete(directoryName, GxFileType.PublicRead); } public bool ExistsDirectory(string directoryName) @@ -377,13 +391,13 @@ public void RenameDirectory(string directoryName, string newDirectoryName) { if (IsFile(item.Name)) { - Copy(item.Name, GxFileType.Public, item.Name.Replace(directoryName, newDirectoryName), GxFileType.Public); - Delete(item.Name, GxFileType.Public); + Copy(item.Name, GxFileType.PublicRead, item.Name.Replace(directoryName, newDirectoryName), GxFileType.PublicRead); + Delete(item.Name, GxFileType.PublicRead); } } CreateDirectory(newDirectoryName); - if (Exists(directoryName, GxFileType.Public)) - Delete(directoryName, GxFileType.Public); + if (Exists(directoryName, GxFileType.PublicRead)) + Delete(directoryName, GxFileType.PublicRead); request.PageToken = response.NextPageToken; } while (response.NextPageToken != null); foreach (string subdir in GetSubDirectories(directoryName)) diff --git a/dotnet/test/DotNetUnitTest/DotNetUnitTest.csproj b/dotnet/test/DotNetUnitTest/DotNetUnitTest.csproj index c883b01b2..be0c33de9 100644 --- a/dotnet/test/DotNetUnitTest/DotNetUnitTest.csproj +++ b/dotnet/test/DotNetUnitTest/DotNetUnitTest.csproj @@ -1,11 +1,13 @@ - + net462 DotNetUnitTest DotNetUnitTest + + @@ -26,6 +28,7 @@ + all runtime; build; native; contentfiles; analyzers; buildtransitive @@ -33,18 +36,22 @@ - + + + - - PreserveNewest - + + Always + - + + Always + diff --git a/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderAzureTest.cs b/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderAzureTest.cs new file mode 100644 index 000000000..c78b22364 --- /dev/null +++ b/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderAzureTest.cs @@ -0,0 +1,14 @@ +using GeneXus.Storage.GXAmazonS3; +using GeneXus.Storage.GXAzureStorage; +using UnitTesting; + +namespace DotNetUnitTest +{ + public class ExternalProviderAzureTest : ExternalProviderTest + { + public ExternalProviderAzureTest(): base(AzureStorageExternalProvider.Name, typeof(AzureStorageExternalProvider)) + { + } + + } +} diff --git a/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderGoogleTest.cs b/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderGoogleTest.cs new file mode 100644 index 000000000..c58dbb0a1 --- /dev/null +++ b/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderGoogleTest.cs @@ -0,0 +1,15 @@ +using GeneXus.Storage.GXAmazonS3; +using GeneXus.Storage.GXAzureStorage; +using GeneXus.Storage.GXGoogleCloud; +using UnitTesting; + +namespace DotNetUnitTest +{ + public class ExternalProviderGoogleTest : ExternalProviderTest + { + public ExternalProviderGoogleTest(): base(ExternalProviderGoogle.Name, typeof(ExternalProviderGoogle)) + { + } + + } +} diff --git a/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderS3Test.cs b/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderS3Test.cs new file mode 100644 index 000000000..e53655069 --- /dev/null +++ b/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderS3Test.cs @@ -0,0 +1,13 @@ +using GeneXus.Storage.GXAmazonS3; +using UnitTesting; + +namespace DotNetUnitTest +{ + public class ExternalProviderS3Test : ExternalProviderTest + { + public ExternalProviderS3Test(): base(ExternalProviderS3.Name, typeof(ExternalProviderS3)) + { + } + + } +} diff --git a/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs b/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs new file mode 100644 index 000000000..2469d0e70 --- /dev/null +++ b/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs @@ -0,0 +1,339 @@ +using System; +using System.Globalization; +using System.IO; +using System.Net; +using GeneXus.Storage.GXAmazonS3; +using GeneXus.Storage.GXAzureStorage; +using GeneXus.Storage.GXGoogleCloud; +using Xunit; + +#pragma warning disable CA1031 // Do not catch general exception types +namespace UnitTesting +{ + public abstract class ExternalProviderTest + { + private GeneXus.Services.ExternalProvider provider; + private static String TEST_SAMPLE_FILE_NAME = "text.txt"; + private static String TEST_SAMPLE_FILE_PATH = Path.Combine("resources", TEST_SAMPLE_FILE_NAME).ToString(CultureInfo.InvariantCulture); + + + public ExternalProviderTest(string providerName, Type externalProviderType) + { + if (externalProviderType == typeof(ExternalProviderS3)) + { + Environment.SetEnvironmentVariable(providerName + "_TEST_ENABLED", "true"); + Environment.SetEnvironmentVariable("STORAGE_AWSS3_ACCESS_KEY", "AKIAJMQ6SF3Y4IULKD5A"); + Environment.SetEnvironmentVariable("STORAGE_AWSS3_SECRET_KEY", "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"); + Environment.SetEnvironmentVariable("STORAGE_AWSS3_BUCKET_NAME", "genexuss3test"); + Environment.SetEnvironmentVariable("STORAGE_AWSS3_FOLDER_NAME", "gxclasses"); + Environment.SetEnvironmentVariable("STORAGE_AWSS3_REGION", "us-east-1"); + Environment.SetEnvironmentVariable("STORAGE_AWSS3_ENDPOINT", "s3.amazonaws.com"); + } + if (externalProviderType == typeof(AzureStorageExternalProvider)) + { + Environment.SetEnvironmentVariable(providerName + "_TEST_ENABLED", "true"); + Environment.SetEnvironmentVariable($"STORAGE_{providerName}_ACCESS_KEY", "DNiutn2Evl+MNs0TsGUqgg0IDzYWMoMAhLM7Oju/PLi4BEsIsrSY917M6li3Ml7sxj8W+KFD/LZU49OGI40Slg=="); + Environment.SetEnvironmentVariable($"STORAGE_{providerName}_ACCOUNT_NAME", "luistest1"); + Environment.SetEnvironmentVariable($"STORAGE_{providerName}_FOLDER_NAME", "luistest1"); + Environment.SetEnvironmentVariable($"STORAGE_{providerName}_PUBLIC_CONTAINER_NAME", "contluispublic"); + Environment.SetEnvironmentVariable($"STORAGE_{providerName}_PRIVATE_CONTAINER_NAME", "contluisprivate"); + } + + if (externalProviderType == typeof(ExternalProviderGoogle)) + { + Environment.SetEnvironmentVariable(providerName + "_TEST_ENABLED", "true"); + Environment.SetEnvironmentVariable($"STORAGE_{providerName}_KEY", "{\"type\": \"service_account\",\"project_id\": \"gxjavacloudstorageunittests\",\"private_key_id\": \"37e8c0566cd56c7bb07542fdddd9f46ac64a5ba0\",\"private_key\": \"-----BEGIN PRIVATE KEY-----\nMIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDQFJ0NHKkmZNCP\n824SBcXHa0DeoDy1i7zToD1DYiJzQXZAZg9tm91SKEbbCZWMgslBdUP10c7UsnuU\nkvnZTUpZ2YQ680ySiykjAWildp28e/nLBDVZy0f8N7H94484kOS4fxQs2VoW8MH6\nVzsxxm+3yhHjQfQOD+Obes3YBMXUxb7rs/blp7fYWbXNzyGGcpM0kEm4mRNslTbi\nZmVnuwpkaaYfQEPPYLfxWcwfgfpbXEPIJB+bNw+EN5JOUvYvCdpwdPgNyA6MvQFj\nGLTAbsXCOtFh6TtRjE9LkjTZrLTAT0HJ0kODxCyM4pBgag87iZAJ4nsq/355DIMJ\nJtEy3ZtTAgMBAAECggEAYmS89wxMeBlH/inwLJmKMohm/l7rFjXjrnahQZHQFIwp\n7L3WIdCIUWc2SjE4BF9753YaEs2Jbk6P3Wu6taS0udP/kRinZsxjQWhTIZr7b7t4\nHSX6TGGxwnRbuGC4wtjRLuT4l1SYIyzprQU+uoTJIzFsT/hJ/bRJvqXNXI61Na0J\n/ahzB640y75xe8H5Yw06yWXisqD+eiAxX8TU2SRdcZgcIVWaWLDiKhERk7sBctol\nWYgxm3qzpkA+dcvIoykrhLMZGaX9yDu7V9ueXeksqehZrQjSVx0CYSRi5ONy9bcJ\nnQIO4B0lo5oCh6EzKsYlvzJlAPCLJbs4K9Fxviu7QQKBgQDuRBiqAG9J7CR06TqW\nohdCGG6byZZ6x9wmWz3QVLsijFpiQTfu6XHDgh2A4NDau1hOzoJ21rGN5H3XNC9/\ni+8k4Je1nM14aiomlsgNFEp+gUhR7uR54da31q7Vf1DqcQ8ykqOCZitb6/AF0zhU\nlm8syJF1QAi3H7LMN0Jk3LnQsQKBgQDfkV14rRp0nUIdEp0O49bmD+qphKXFAsYn\nWCt37jq3mgZ2IVOfX94+m6bLUAjb+Ipwn9dT+3PrDlZqHRPMKTGvWdm9+41EyBK3\nvjKiNSyqnw/G/gapJn3aN/Nhp8qzOC+xNUTT9xXRf4cEuw9f19zXSm9niDgN5LS3\nE163+ZMNQwKBgB+J7gXaxuBvHKhJExNLY27BUyrV9VBNUkvVegownRDGqVQmM+Qx\nDHkHqSYdHChH8jmERmq6oogYvbuV0c+9Uyt7ezl0BxKwYuH2xYZNsEqsjEkkKSQl\nC8oL5dqm3qwZyRw1ouUo5wZk5cGvot43h4HTDsYJct3imUVE70nwmbwRAoGAQjh4\ni0oaz/fUoW/l/YcXHEYSp+uWfmh38Sd4mKmD0uZYi50Le+WVms3X9djbBuzzdLCj\nw0hz6WfxyLScLJj3Eo12pYNhMMJiaPJ5ZPqDJHbA4ZxUtL2mAYEZIg/lRniaB89T\nd8V0PP2dLJWL1EPIMizmGrCKifL4ZFHkeHIAUKkCgYAhJsPSmUc852/arUFqzhzM\nmtI/bWUUFeULygoARTSN4SVz/RjDj2MJYSV2N9MAskUKFXmmwNMq/NpJ2KkxlHrV\ned1O6cGAg8VYPpQvf6GOmpnHauePQXWXR3xVB++omRH4lD+J4gMT73dOU5o70V5e\nwY7LJ21G5lim4X1G3zRPOQ==\n-----END PRIVATE KEY-----\n\",\"client_email\": \"github-access@gxjavacloudstorageunittests.iam.gserviceaccount.com\",\"client_id\": \"110489372256464678127\",\"auth_uri\": \"https://accounts.google.com/o/oauth2/auth\",\"token_uri\": \"https://oauth2.googleapis.com/token\",\"auth_provider_x509_cert_url\": \"https://www.googleapis.com/oauth2/v1/certs\",\"client_x509_cert_url\": \"https://www.googleapis.com/robot/v1/metadata/x509/github-access%40gxjavacloudstorageunittests.iam.gserviceaccount.com\"}"); + Environment.SetEnvironmentVariable($"STORAGE_{providerName}_PROJECT_ID", "gxjavacloudstorageunittests"); + Environment.SetEnvironmentVariable($"STORAGE_{providerName}_BUCKET_NAME", "javaclasses-unittests"); + Environment.SetEnvironmentVariable($"STORAGE_{providerName}_FOLDER_NAME", "gxclasses"); + Environment.SetEnvironmentVariable($"STORAGE_{providerName}_APPLICATION_NAME", "gxjavacloudstorageunittests"); + } + + bool testEnabled = Environment.GetEnvironmentVariable(providerName + "_TEST_ENABLED") == "true"; + + Assume.True(testEnabled); + + provider = (GeneXus.Services.ExternalProvider)Activator.CreateInstance(externalProviderType); + + Assert.NotNull(provider); + } + + [Fact] + public void TestUploadPublicMethod() + { + String upload = provider.Upload(TEST_SAMPLE_FILE_PATH, TEST_SAMPLE_FILE_NAME, GxFileType.PublicRead); + EnsureUrl(upload, GxFileType.PublicRead); + } + + [Fact] + public void TestUploadDefaultMethod() + { + String upload = provider.Upload(TEST_SAMPLE_FILE_PATH, TEST_SAMPLE_FILE_NAME, GxFileType.Default); + EnsureUrl(upload, GxFileType.Default); + } + + [Fact] + public void TestUploadAndCopyDefault() + { + TestUploadAndCopyByAcl(GxFileType.Default, GxFileType.Default); + } + + [Fact] + public void TestUploadAndCopyPrivate() + { + TestUploadAndCopyByAcl(GxFileType.Private, GxFileType.Private); + } + + [Fact] + public void TestUploadAndCopyPublic() + { + TestUploadAndCopyByAcl(GxFileType.PublicRead, GxFileType.PublicRead); + } + + [Fact] + public void TestUploadAndCopyMixed() + { + TestUploadAndCopyByAcl(GxFileType.Default, GxFileType.Private); + } + + [Fact] + public void TestUploadAndCopyPrivateToPublic() + { + TestUploadAndCopyByAcl(GxFileType.Private, GxFileType.PublicRead); + } + + [Fact] + public void TestUploadAndCopyPublicToPrivate() + { + TestUploadAndCopyByAcl(GxFileType.PublicRead, GxFileType.Private); + } + + public void TestUploadAndCopyByAcl(GxFileType aclUpload, GxFileType aclCopy) + { + String copyFileName = "test-upload-and-copy.txt"; + DeleteSafe(TEST_SAMPLE_FILE_PATH); + DeleteSafe(copyFileName); + String upload = provider.Upload(TEST_SAMPLE_FILE_PATH, TEST_SAMPLE_FILE_NAME, aclUpload); + Assert.True(UrlExists(upload), "Not found URL: " + upload); + + String copyUrl = TryGet(copyFileName, aclCopy); + Assert.False(UrlExists(copyUrl), "URL cannot exist: " + copyUrl); + + provider.Copy(TEST_SAMPLE_FILE_NAME, aclUpload, copyFileName, aclCopy); + upload = provider.Get(copyFileName, aclCopy, 100); + EnsureUrl(upload, aclCopy); + } + + [Fact] + public void TestCopyMethod() + { + String copyFileName = "copy-text.txt"; + Copy(copyFileName, GxFileType.PublicRead); + } + + [Fact] + public void TestCopyPrivateMethod() + { + String copyFileName = "copy-text-private.txt"; + Copy(copyFileName, GxFileType.Private); + } + + private void Copy(String copyFileName, GxFileType acl) + { + provider.Upload(TEST_SAMPLE_FILE_PATH, TEST_SAMPLE_FILE_NAME, acl); + String upload = provider.Get(TEST_SAMPLE_FILE_NAME, acl, 100); + EnsureUrl(upload, acl); + + DeleteSafe(copyFileName); + Wait(1000); //Google CDN replication seems to be delayed. + + String urlCopy = TryGet(copyFileName, GxFileType.PublicRead); + Assert.False(UrlExists(urlCopy), "URL cannot exist: " + urlCopy); + + provider.Copy("text.txt", acl, copyFileName, acl); + upload = provider.Get(copyFileName, acl, 100); + EnsureUrl(upload, acl); + } + + [Fact] + public void TestMultimediaUpload() + { + String copyFileName = "copy-text-private.txt"; + GxFileType acl = GxFileType.Private; + + provider.Upload(TEST_SAMPLE_FILE_PATH, TEST_SAMPLE_FILE_NAME, acl); + String upload = provider.Get(TEST_SAMPLE_FILE_NAME, acl, 100); + EnsureUrl(upload, acl); + + provider.Delete(copyFileName, acl); + provider.Copy("text.txt", acl, copyFileName, acl); + upload = TryGet(copyFileName, acl); + EnsureUrl(upload, acl); + } + + [Fact] + public void TestGetMethod() + { + TestUploadPublicMethod(); + String url = provider.Get("text.txt", GxFileType.PublicRead, 10); + EnsureUrl(url, GxFileType.PublicRead); + } + + [Fact] + public void TestGetObjectName() + { + TestUploadPublicMethod(); + String url = provider.Get(TEST_SAMPLE_FILE_NAME, GxFileType.PublicRead, 10); + Assert.True(UrlExists(url)); + String objectName = ""; + provider.GetObjectNameFromURL(url, out objectName); + Assert.Equal("text.txt", objectName); + } + + [Fact] + public void TestDownloadMethod() + { + String downloadPath = Path.Combine("resources", "test", TEST_SAMPLE_FILE_NAME); + TestUploadPublicMethod(); + + try + { + File.Delete(downloadPath); + } + catch (Exception) { } + + provider.Download(TEST_SAMPLE_FILE_NAME, downloadPath, GxFileType.PublicRead); + Assert.True(File.Exists(downloadPath)); + } + + [Fact] + public void TestDeleteFile() + { + GxFileType acl = GxFileType.PublicRead; + TestUploadPublicMethod(); + String url = TryGet(TEST_SAMPLE_FILE_NAME, acl); + EnsureUrl(url, acl); + provider.Delete(TEST_SAMPLE_FILE_NAME, acl); + + url = TryGet(TEST_SAMPLE_FILE_NAME, acl); + Assert.False(UrlExists(url)); + } + + [Fact] + public void TestDeleteFilePrivate() + { + GxFileType acl = GxFileType.Private; + TestUploadPrivateMethod(); + String url = TryGet(TEST_SAMPLE_FILE_NAME, acl); + EnsureUrl(url, acl); + provider.Delete(TEST_SAMPLE_FILE_NAME, acl); + + url = TryGet(TEST_SAMPLE_FILE_NAME, acl); + Assert.False(UrlExists(url)); + } + + [Fact] + public void TestUploadPrivateMethod() + { + GxFileType acl = GxFileType.Private; + String externalFileName = "text-private-2.txt"; + + DeleteSafe(externalFileName); + String signedUrl = provider.Upload(TEST_SAMPLE_FILE_PATH, externalFileName, acl); + EnsureUrl(signedUrl, acl); + signedUrl = provider.Get(externalFileName, acl, 10); + EnsureUrl(signedUrl, acl); + + } + + + private String TryGet(String objectName, GxFileType acl) + { + String getValue = ""; + try + { + getValue = provider.Get(objectName, acl, 5); + } + catch (Exception) + { + + } + return getValue; + } + + private String GetSafe(String objectName, GxFileType acl) + { + try + { + return provider.Get(objectName, acl, 100); + } + catch (Exception) + { + + } + return String.Empty; + } + + private bool DeleteSafe(String objectName) + { + DeleteSafeImpl(objectName, GxFileType.Private); + DeleteSafeImpl(objectName, GxFileType.PublicRead); + return true; + } + + private bool DeleteSafeImpl(String objectName, GxFileType acl) + { + try + { + provider.Delete(objectName, acl); + } + catch (Exception) + { + + } + return true; + } + + private static void Wait(int milliseconds) + { + System.Threading.Thread.Sleep(milliseconds); + } + + private static void EnsureUrl(String signedOrUnsignedUrl, GxFileType acl) + { + Assert.True(UrlExists(signedOrUnsignedUrl), "Resource not found: " + signedOrUnsignedUrl); + if (acl == GxFileType.Private) + { + String noSignedUrl = signedOrUnsignedUrl.Substring(0, signedOrUnsignedUrl.IndexOf('?') + 1); + Assert.False(UrlExists(noSignedUrl), "Resource must be private: " + noSignedUrl); + } + } + +#pragma warning disable CA1054 // Uri parameters should not be strings + protected static bool UrlExists(string url) + { + if (string.IsNullOrEmpty(url)) + { + return false; + } + bool exists = false; + HttpWebRequest request = (HttpWebRequest)WebRequest.Create(new Uri(url)); + request.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate; + + try + { + using (HttpWebResponse response = (HttpWebResponse)request.GetResponse()) + { + exists = response.StatusCode == HttpStatusCode.OK; + } + } + catch (WebException) + { + + } + return exists; + } +#pragma warning restore CA1054 // Uri parameters should not be strings + } +} +#pragma warning restore CA1031 // Do not catch general exception types \ No newline at end of file diff --git a/dotnet/test/DotNetUnitTest/Properties/launchSettings.json b/dotnet/test/DotNetUnitTest/Properties/launchSettings.json new file mode 100644 index 000000000..6d2484989 --- /dev/null +++ b/dotnet/test/DotNetUnitTest/Properties/launchSettings.json @@ -0,0 +1,11 @@ +{ + "profiles": { + "DotNetUnitTest": { + "commandName": "Project", + "environmentVariables": { + "test": "test" + }, + "remoteDebugEnabled": false + } + } +} \ No newline at end of file diff --git a/dotnet/test/DotNetUnitTest/resources/text.txt b/dotnet/test/DotNetUnitTest/resources/text.txt new file mode 100644 index 000000000..e04a19003 --- /dev/null +++ b/dotnet/test/DotNetUnitTest/resources/text.txt @@ -0,0 +1 @@ +My uploaded file test \ No newline at end of file From fb0a546b8353b7c9e518d0f0d29aa643a014fafa Mon Sep 17 00:00:00 2001 From: Gonzalo Date: Thu, 18 Jun 2020 15:14:32 -0300 Subject: [PATCH 04/33] Support for Path Style urls (cherry picked from commit 48fe32c4abc8f373117c9f9ce6812ded495dac3e) --- .../Storage/GXAmazonS3/ExternalProviderS3.cs | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs b/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs index 54accf387..64c66fdd1 100644 --- a/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs +++ b/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs @@ -27,7 +27,7 @@ public class ExternalProviderS3 : ExternalProviderBase, ExternalProvider const string STORAGE_CUSTOM_ENDPOINT_VALUE = "custom"; const string DEFAULT_REGION = "us-east-1"; - + [Obsolete("Use Property ACCESS_KEY instead", false)] const string ACCESS_KEY_ID_DEPRECATED = "STORAGE_PROVIDER_ACCESSKEYID"; [Obsolete("Use Property SECRET_ACCESS_KEY instead", false)] @@ -57,7 +57,7 @@ public class ExternalProviderS3 : ExternalProviderBase, ExternalProvider public string StorageUri { get { - return (ForcePathStyle)? $"{Endpoint}/": $"https://{Bucket}.{Endpoint}/"; + return _storageUri; } } @@ -133,7 +133,7 @@ private void SetURI() { if (Region == DEFAULT_REGION) { - _storageUri = $"https://{Bucket}.{Endpoint}/"; + _storageUri = (ForcePathStyle) ? $"{Endpoint}/" : $"https://{Bucket}.{Endpoint}/"; } else { @@ -240,16 +240,20 @@ private bool IsPrivateUpload(GxFileType fileType) public string Get(string objectName, GxFileType fileType, int urlMinutes = DefaultExpirationMinutes) { - bool isPrivate = IsPrivateUpload(fileType); if (Exists(objectName, fileType)) - if (isPrivate) - return GetPreSignedUrl(objectName, urlMinutes); - else - return StorageUri + StorageUtils.EncodeUrl(objectName); + { + return GetUrlImpl(objectName, fileType, urlMinutes); + } else return string.Empty; } + private string GetUrlImpl(string objectName, GxFileType fileType, int urlMinutes = DefaultExpirationMinutes) + { + bool isPrivate = IsPrivateUpload(fileType); + return (isPrivate)? GetPreSignedUrl(objectName, urlMinutes): StorageUri + StorageUtils.EncodeUrl(objectName); + } + private string GetPreSignedUrl(string objectName, int urlMinutes) { GetPreSignedUrlRequest request = new GetPreSignedUrlRequest From a7b076f361c24e3747e0d55d293e2c09058a587f Mon Sep 17 00:00:00 2001 From: Gonzalo Date: Thu, 10 Jun 2021 19:29:16 -0300 Subject: [PATCH 05/33] Add Minio Unit Tests (cherry picked from commit cb9d170e76347aa45cf66f809aaf064b1994fa61) --- .../Storage/GXAmazonS3/ExternalProviderS3.cs | 32 ++++++++++++------- .../GXGoogleCloud/ExternalProviderGoogle.cs | 2 ++ .../ExternalProviderMinioTest.cs | 13 ++++++++ .../ExternalProvider/ExternalProviderTest.cs | 18 +++++++++-- 4 files changed, 51 insertions(+), 14 deletions(-) create mode 100644 dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderMinioTest.cs diff --git a/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs b/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs index 64c66fdd1..28ca58f00 100644 --- a/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs +++ b/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs @@ -52,8 +52,9 @@ public class ExternalProviderS3 : ExternalProviderBase, ExternalProvider string Endpoint { get; set; } string Region { get; set; } - bool ForcePathStyle = false; - + bool forcePathStyle = false; + bool customEndpoint = false; + public string StorageUri { get { @@ -84,7 +85,7 @@ private void Initialize() { credentials = new BasicAWSCredentials(keyId, keySecret); } - var region = Amazon.RegionEndpoint.GetBySystemName(GetPropertyValue(REGION, REGION_DEPRECATED)); + var region = Amazon.RegionEndpoint.GetBySystemName(GetPropertyValue(REGION, REGION_DEPRECATED, DEFAULT_REGION)); AmazonS3Config config = new AmazonS3Config() { @@ -95,11 +96,12 @@ private void Initialize() { if (Endpoint == STORAGE_CUSTOM_ENDPOINT_VALUE) { Endpoint = GetPropertyValue(STORAGE_CUSTOM_ENDPOINT, STORAGE_CUSTOM_ENDPOINT_DEPRECATED); - ForcePathStyle = true; - config.ForcePathStyle = ForcePathStyle; + forcePathStyle = true; + config.ForcePathStyle = forcePathStyle; config.ServiceURL = Endpoint; + customEndpoint = true; } - + #if NETCORE if (credentials != null) { @@ -131,13 +133,20 @@ private void Initialize() { private void SetURI() { - if (Region == DEFAULT_REGION) + if (customEndpoint) { - _storageUri = (ForcePathStyle) ? $"{Endpoint}/" : $"https://{Bucket}.{Endpoint}/"; + _storageUri = !Endpoint.EndsWith("/") ? $"{Endpoint}/{Bucket}/": $"{Endpoint}{Bucket}/"; } else { - _storageUri = $"https://{Bucket}.{Endpoint.Replace("s3.amazonaws.com", $"s3.{Region.ToLower()}.amazonaws.com")}/"; + if (Region == DEFAULT_REGION) + { + _storageUri = (forcePathStyle) ? $"{Endpoint}/" : $"https://{Bucket}.{Endpoint}/"; + } + else + { + _storageUri = $"https://{Bucket}.{Endpoint.Replace("s3.amazonaws.com", $"s3.{Region.ToLower()}.amazonaws.com")}/"; + } } } @@ -229,8 +238,8 @@ public string Upload(string localFile, string objectName, GxFileType fileType) FilePath = localFile, CannedACL = GetCannedACL(fileType) }; - PutObjectResponse result = PutObject(objectRequest); - return Get(objectName, fileType); + PutObject(objectRequest); + return GetUrlImpl(objectName, fileType); } private bool IsPrivateUpload(GxFileType fileType) @@ -252,6 +261,7 @@ private string GetUrlImpl(string objectName, GxFileType fileType, int urlMinutes { bool isPrivate = IsPrivateUpload(fileType); return (isPrivate)? GetPreSignedUrl(objectName, urlMinutes): StorageUri + StorageUtils.EncodeUrl(objectName); + } private string GetPreSignedUrl(string objectName, int urlMinutes) diff --git a/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs b/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs index c53ab783f..040ca64b8 100644 --- a/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs +++ b/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs @@ -32,6 +32,8 @@ public class ExternalProviderGoogle : ExternalProviderBase, ExternalProvider String Bucket { get; set; } String Folder { get; set; } UrlSigner Signer { get; set; } + + public string StorageUri { get { return $"https://{Bucket}.storage.googleapis.com/"; } diff --git a/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderMinioTest.cs b/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderMinioTest.cs new file mode 100644 index 000000000..2695bd232 --- /dev/null +++ b/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderMinioTest.cs @@ -0,0 +1,13 @@ +using GeneXus.Storage.GXAmazonS3; +using UnitTesting; + +namespace DotNetUnitTest +{ + public class ExternalProviderMinioTest : ExternalProviderTest + { + public ExternalProviderMinioTest(): base(ExternalProviderS3.Name, typeof(ExternalProviderS3)) + { + } + + } +} diff --git a/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs b/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs index 2469d0e70..5c6f0fcee 100644 --- a/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs +++ b/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs @@ -2,6 +2,7 @@ using System.Globalization; using System.IO; using System.Net; +using DotNetUnitTest; using GeneXus.Storage.GXAmazonS3; using GeneXus.Storage.GXAzureStorage; using GeneXus.Storage.GXGoogleCloud; @@ -19,7 +20,7 @@ public abstract class ExternalProviderTest public ExternalProviderTest(string providerName, Type externalProviderType) { - if (externalProviderType == typeof(ExternalProviderS3)) + if (this.GetType() == typeof(ExternalProviderS3Test)) { Environment.SetEnvironmentVariable(providerName + "_TEST_ENABLED", "true"); Environment.SetEnvironmentVariable("STORAGE_AWSS3_ACCESS_KEY", "AKIAJMQ6SF3Y4IULKD5A"); @@ -29,7 +30,18 @@ public ExternalProviderTest(string providerName, Type externalProviderType) Environment.SetEnvironmentVariable("STORAGE_AWSS3_REGION", "us-east-1"); Environment.SetEnvironmentVariable("STORAGE_AWSS3_ENDPOINT", "s3.amazonaws.com"); } - if (externalProviderType == typeof(AzureStorageExternalProvider)) + + if (this.GetType() == typeof(ExternalProviderMinioTest)) + { + Environment.SetEnvironmentVariable(providerName + "_TEST_ENABLED", "true"); + Environment.SetEnvironmentVariable("STORAGE_AWSS3_ACCESS_KEY", "desaint"); + Environment.SetEnvironmentVariable("STORAGE_AWSS3_SECRET_KEY", "6YafTT3U2YtHS7am"); + Environment.SetEnvironmentVariable("STORAGE_AWSS3_BUCKET_NAME", "java-classes-unittests"); + Environment.SetEnvironmentVariable("STORAGE_AWSS3_FOLDER_NAME", "test-minio"); + Environment.SetEnvironmentVariable("STORAGE_AWSS3_ENDPOINT", "custom"); + Environment.SetEnvironmentVariable("STORAGE_AWSS3_CUSTOM_ENDPOINT", "http://192.168.254.78:9000"); + } + if (this.GetType() == typeof(ExternalProviderAzureTest)) { Environment.SetEnvironmentVariable(providerName + "_TEST_ENABLED", "true"); Environment.SetEnvironmentVariable($"STORAGE_{providerName}_ACCESS_KEY", "DNiutn2Evl+MNs0TsGUqgg0IDzYWMoMAhLM7Oju/PLi4BEsIsrSY917M6li3Ml7sxj8W+KFD/LZU49OGI40Slg=="); @@ -39,7 +51,7 @@ public ExternalProviderTest(string providerName, Type externalProviderType) Environment.SetEnvironmentVariable($"STORAGE_{providerName}_PRIVATE_CONTAINER_NAME", "contluisprivate"); } - if (externalProviderType == typeof(ExternalProviderGoogle)) + if (this.GetType() == typeof(ExternalProviderGoogleTest)) { Environment.SetEnvironmentVariable(providerName + "_TEST_ENABLED", "true"); Environment.SetEnvironmentVariable($"STORAGE_{providerName}_KEY", "{\"type\": \"service_account\",\"project_id\": \"gxjavacloudstorageunittests\",\"private_key_id\": \"37e8c0566cd56c7bb07542fdddd9f46ac64a5ba0\",\"private_key\": \"-----BEGIN PRIVATE KEY-----\nMIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDQFJ0NHKkmZNCP\n824SBcXHa0DeoDy1i7zToD1DYiJzQXZAZg9tm91SKEbbCZWMgslBdUP10c7UsnuU\nkvnZTUpZ2YQ680ySiykjAWildp28e/nLBDVZy0f8N7H94484kOS4fxQs2VoW8MH6\nVzsxxm+3yhHjQfQOD+Obes3YBMXUxb7rs/blp7fYWbXNzyGGcpM0kEm4mRNslTbi\nZmVnuwpkaaYfQEPPYLfxWcwfgfpbXEPIJB+bNw+EN5JOUvYvCdpwdPgNyA6MvQFj\nGLTAbsXCOtFh6TtRjE9LkjTZrLTAT0HJ0kODxCyM4pBgag87iZAJ4nsq/355DIMJ\nJtEy3ZtTAgMBAAECggEAYmS89wxMeBlH/inwLJmKMohm/l7rFjXjrnahQZHQFIwp\n7L3WIdCIUWc2SjE4BF9753YaEs2Jbk6P3Wu6taS0udP/kRinZsxjQWhTIZr7b7t4\nHSX6TGGxwnRbuGC4wtjRLuT4l1SYIyzprQU+uoTJIzFsT/hJ/bRJvqXNXI61Na0J\n/ahzB640y75xe8H5Yw06yWXisqD+eiAxX8TU2SRdcZgcIVWaWLDiKhERk7sBctol\nWYgxm3qzpkA+dcvIoykrhLMZGaX9yDu7V9ueXeksqehZrQjSVx0CYSRi5ONy9bcJ\nnQIO4B0lo5oCh6EzKsYlvzJlAPCLJbs4K9Fxviu7QQKBgQDuRBiqAG9J7CR06TqW\nohdCGG6byZZ6x9wmWz3QVLsijFpiQTfu6XHDgh2A4NDau1hOzoJ21rGN5H3XNC9/\ni+8k4Je1nM14aiomlsgNFEp+gUhR7uR54da31q7Vf1DqcQ8ykqOCZitb6/AF0zhU\nlm8syJF1QAi3H7LMN0Jk3LnQsQKBgQDfkV14rRp0nUIdEp0O49bmD+qphKXFAsYn\nWCt37jq3mgZ2IVOfX94+m6bLUAjb+Ipwn9dT+3PrDlZqHRPMKTGvWdm9+41EyBK3\nvjKiNSyqnw/G/gapJn3aN/Nhp8qzOC+xNUTT9xXRf4cEuw9f19zXSm9niDgN5LS3\nE163+ZMNQwKBgB+J7gXaxuBvHKhJExNLY27BUyrV9VBNUkvVegownRDGqVQmM+Qx\nDHkHqSYdHChH8jmERmq6oogYvbuV0c+9Uyt7ezl0BxKwYuH2xYZNsEqsjEkkKSQl\nC8oL5dqm3qwZyRw1ouUo5wZk5cGvot43h4HTDsYJct3imUVE70nwmbwRAoGAQjh4\ni0oaz/fUoW/l/YcXHEYSp+uWfmh38Sd4mKmD0uZYi50Le+WVms3X9djbBuzzdLCj\nw0hz6WfxyLScLJj3Eo12pYNhMMJiaPJ5ZPqDJHbA4ZxUtL2mAYEZIg/lRniaB89T\nd8V0PP2dLJWL1EPIMizmGrCKifL4ZFHkeHIAUKkCgYAhJsPSmUc852/arUFqzhzM\nmtI/bWUUFeULygoARTSN4SVz/RjDj2MJYSV2N9MAskUKFXmmwNMq/NpJ2KkxlHrV\ned1O6cGAg8VYPpQvf6GOmpnHauePQXWXR3xVB++omRH4lD+J4gMT73dOU5o70V5e\nwY7LJ21G5lim4X1G3zRPOQ==\n-----END PRIVATE KEY-----\n\",\"client_email\": \"github-access@gxjavacloudstorageunittests.iam.gserviceaccount.com\",\"client_id\": \"110489372256464678127\",\"auth_uri\": \"https://accounts.google.com/o/oauth2/auth\",\"token_uri\": \"https://oauth2.googleapis.com/token\",\"auth_provider_x509_cert_url\": \"https://www.googleapis.com/oauth2/v1/certs\",\"client_x509_cert_url\": \"https://www.googleapis.com/robot/v1/metadata/x509/github-access%40gxjavacloudstorageunittests.iam.gserviceaccount.com\"}"); From 5390fcaa1d7d66a3aba0b326a8ca9e964f921ba2 Mon Sep 17 00:00:00 2001 From: Gonzalo Date: Thu, 10 Jun 2021 19:32:26 -0300 Subject: [PATCH 06/33] Remove launchSettings.json (cherry picked from commit c7f0550b12de7623db882c6421f9944141c25fbf) --- .../DotNetUnitTest/Properties/launchSettings.json | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 dotnet/test/DotNetUnitTest/Properties/launchSettings.json diff --git a/dotnet/test/DotNetUnitTest/Properties/launchSettings.json b/dotnet/test/DotNetUnitTest/Properties/launchSettings.json deleted file mode 100644 index 6d2484989..000000000 --- a/dotnet/test/DotNetUnitTest/Properties/launchSettings.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "profiles": { - "DotNetUnitTest": { - "commandName": "Project", - "environmentVariables": { - "test": "test" - }, - "remoteDebugEnabled": false - } - } -} \ No newline at end of file From c1b4cfd3fcf3b90bc276eadf367a2e37ed11175b Mon Sep 17 00:00:00 2001 From: Gonzalo Date: Thu, 10 Jun 2021 21:45:21 -0300 Subject: [PATCH 07/33] Remove Secrets (cherry picked from commit d759a2a412bf9fed5574e4550fd89e9e0e061647) --- .../ExternalProvider/ExternalProviderTest.cs | 41 +------------------ 1 file changed, 1 insertion(+), 40 deletions(-) diff --git a/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs b/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs index 5c6f0fcee..b7dae0295 100644 --- a/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs +++ b/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs @@ -20,46 +20,7 @@ public abstract class ExternalProviderTest public ExternalProviderTest(string providerName, Type externalProviderType) { - if (this.GetType() == typeof(ExternalProviderS3Test)) - { - Environment.SetEnvironmentVariable(providerName + "_TEST_ENABLED", "true"); - Environment.SetEnvironmentVariable("STORAGE_AWSS3_ACCESS_KEY", "AKIAJMQ6SF3Y4IULKD5A"); - Environment.SetEnvironmentVariable("STORAGE_AWSS3_SECRET_KEY", "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"); - Environment.SetEnvironmentVariable("STORAGE_AWSS3_BUCKET_NAME", "genexuss3test"); - Environment.SetEnvironmentVariable("STORAGE_AWSS3_FOLDER_NAME", "gxclasses"); - Environment.SetEnvironmentVariable("STORAGE_AWSS3_REGION", "us-east-1"); - Environment.SetEnvironmentVariable("STORAGE_AWSS3_ENDPOINT", "s3.amazonaws.com"); - } - - if (this.GetType() == typeof(ExternalProviderMinioTest)) - { - Environment.SetEnvironmentVariable(providerName + "_TEST_ENABLED", "true"); - Environment.SetEnvironmentVariable("STORAGE_AWSS3_ACCESS_KEY", "desaint"); - Environment.SetEnvironmentVariable("STORAGE_AWSS3_SECRET_KEY", "6YafTT3U2YtHS7am"); - Environment.SetEnvironmentVariable("STORAGE_AWSS3_BUCKET_NAME", "java-classes-unittests"); - Environment.SetEnvironmentVariable("STORAGE_AWSS3_FOLDER_NAME", "test-minio"); - Environment.SetEnvironmentVariable("STORAGE_AWSS3_ENDPOINT", "custom"); - Environment.SetEnvironmentVariable("STORAGE_AWSS3_CUSTOM_ENDPOINT", "http://192.168.254.78:9000"); - } - if (this.GetType() == typeof(ExternalProviderAzureTest)) - { - Environment.SetEnvironmentVariable(providerName + "_TEST_ENABLED", "true"); - Environment.SetEnvironmentVariable($"STORAGE_{providerName}_ACCESS_KEY", "DNiutn2Evl+MNs0TsGUqgg0IDzYWMoMAhLM7Oju/PLi4BEsIsrSY917M6li3Ml7sxj8W+KFD/LZU49OGI40Slg=="); - Environment.SetEnvironmentVariable($"STORAGE_{providerName}_ACCOUNT_NAME", "luistest1"); - Environment.SetEnvironmentVariable($"STORAGE_{providerName}_FOLDER_NAME", "luistest1"); - Environment.SetEnvironmentVariable($"STORAGE_{providerName}_PUBLIC_CONTAINER_NAME", "contluispublic"); - Environment.SetEnvironmentVariable($"STORAGE_{providerName}_PRIVATE_CONTAINER_NAME", "contluisprivate"); - } - - if (this.GetType() == typeof(ExternalProviderGoogleTest)) - { - Environment.SetEnvironmentVariable(providerName + "_TEST_ENABLED", "true"); - Environment.SetEnvironmentVariable($"STORAGE_{providerName}_KEY", "{\"type\": \"service_account\",\"project_id\": \"gxjavacloudstorageunittests\",\"private_key_id\": \"37e8c0566cd56c7bb07542fdddd9f46ac64a5ba0\",\"private_key\": \"-----BEGIN PRIVATE KEY-----\nMIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDQFJ0NHKkmZNCP\n824SBcXHa0DeoDy1i7zToD1DYiJzQXZAZg9tm91SKEbbCZWMgslBdUP10c7UsnuU\nkvnZTUpZ2YQ680ySiykjAWildp28e/nLBDVZy0f8N7H94484kOS4fxQs2VoW8MH6\nVzsxxm+3yhHjQfQOD+Obes3YBMXUxb7rs/blp7fYWbXNzyGGcpM0kEm4mRNslTbi\nZmVnuwpkaaYfQEPPYLfxWcwfgfpbXEPIJB+bNw+EN5JOUvYvCdpwdPgNyA6MvQFj\nGLTAbsXCOtFh6TtRjE9LkjTZrLTAT0HJ0kODxCyM4pBgag87iZAJ4nsq/355DIMJ\nJtEy3ZtTAgMBAAECggEAYmS89wxMeBlH/inwLJmKMohm/l7rFjXjrnahQZHQFIwp\n7L3WIdCIUWc2SjE4BF9753YaEs2Jbk6P3Wu6taS0udP/kRinZsxjQWhTIZr7b7t4\nHSX6TGGxwnRbuGC4wtjRLuT4l1SYIyzprQU+uoTJIzFsT/hJ/bRJvqXNXI61Na0J\n/ahzB640y75xe8H5Yw06yWXisqD+eiAxX8TU2SRdcZgcIVWaWLDiKhERk7sBctol\nWYgxm3qzpkA+dcvIoykrhLMZGaX9yDu7V9ueXeksqehZrQjSVx0CYSRi5ONy9bcJ\nnQIO4B0lo5oCh6EzKsYlvzJlAPCLJbs4K9Fxviu7QQKBgQDuRBiqAG9J7CR06TqW\nohdCGG6byZZ6x9wmWz3QVLsijFpiQTfu6XHDgh2A4NDau1hOzoJ21rGN5H3XNC9/\ni+8k4Je1nM14aiomlsgNFEp+gUhR7uR54da31q7Vf1DqcQ8ykqOCZitb6/AF0zhU\nlm8syJF1QAi3H7LMN0Jk3LnQsQKBgQDfkV14rRp0nUIdEp0O49bmD+qphKXFAsYn\nWCt37jq3mgZ2IVOfX94+m6bLUAjb+Ipwn9dT+3PrDlZqHRPMKTGvWdm9+41EyBK3\nvjKiNSyqnw/G/gapJn3aN/Nhp8qzOC+xNUTT9xXRf4cEuw9f19zXSm9niDgN5LS3\nE163+ZMNQwKBgB+J7gXaxuBvHKhJExNLY27BUyrV9VBNUkvVegownRDGqVQmM+Qx\nDHkHqSYdHChH8jmERmq6oogYvbuV0c+9Uyt7ezl0BxKwYuH2xYZNsEqsjEkkKSQl\nC8oL5dqm3qwZyRw1ouUo5wZk5cGvot43h4HTDsYJct3imUVE70nwmbwRAoGAQjh4\ni0oaz/fUoW/l/YcXHEYSp+uWfmh38Sd4mKmD0uZYi50Le+WVms3X9djbBuzzdLCj\nw0hz6WfxyLScLJj3Eo12pYNhMMJiaPJ5ZPqDJHbA4ZxUtL2mAYEZIg/lRniaB89T\nd8V0PP2dLJWL1EPIMizmGrCKifL4ZFHkeHIAUKkCgYAhJsPSmUc852/arUFqzhzM\nmtI/bWUUFeULygoARTSN4SVz/RjDj2MJYSV2N9MAskUKFXmmwNMq/NpJ2KkxlHrV\ned1O6cGAg8VYPpQvf6GOmpnHauePQXWXR3xVB++omRH4lD+J4gMT73dOU5o70V5e\nwY7LJ21G5lim4X1G3zRPOQ==\n-----END PRIVATE KEY-----\n\",\"client_email\": \"github-access@gxjavacloudstorageunittests.iam.gserviceaccount.com\",\"client_id\": \"110489372256464678127\",\"auth_uri\": \"https://accounts.google.com/o/oauth2/auth\",\"token_uri\": \"https://oauth2.googleapis.com/token\",\"auth_provider_x509_cert_url\": \"https://www.googleapis.com/oauth2/v1/certs\",\"client_x509_cert_url\": \"https://www.googleapis.com/robot/v1/metadata/x509/github-access%40gxjavacloudstorageunittests.iam.gserviceaccount.com\"}"); - Environment.SetEnvironmentVariable($"STORAGE_{providerName}_PROJECT_ID", "gxjavacloudstorageunittests"); - Environment.SetEnvironmentVariable($"STORAGE_{providerName}_BUCKET_NAME", "javaclasses-unittests"); - Environment.SetEnvironmentVariable($"STORAGE_{providerName}_FOLDER_NAME", "gxclasses"); - Environment.SetEnvironmentVariable($"STORAGE_{providerName}_APPLICATION_NAME", "gxjavacloudstorageunittests"); - } + bool testEnabled = Environment.GetEnvironmentVariable(providerName + "_TEST_ENABLED") == "true"; From 8da1dc862c49baebae046cf1cc536a0d08d6be1b Mon Sep 17 00:00:00 2001 From: Gonzalo Date: Fri, 11 Jun 2021 14:42:48 -0300 Subject: [PATCH 08/33] Every file is now private (cherry picked from commit 32542a28643ec7ff6f51126397077648d6022990) --- .../dotnetframework/GxClasses/Core/GXUtils.cs | 15 ++++---- .../GxClasses/Data/GXDataNTierADO.cs | 8 ++--- .../GxClasses/Domain/GXFileIO.cs | 15 ++++---- .../Services/Storage/ExternalProviderBase.cs | 18 ++++++++-- .../Storage/ExternalProviderCommon.cs | 36 +++++++++++++++++++ .../test/DotNetUnitTest/DotNetUnitTest.csproj | 3 ++ 6 files changed, 74 insertions(+), 21 deletions(-) create mode 100644 dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderCommon.cs diff --git a/dotnet/src/dotnetframework/GxClasses/Core/GXUtils.cs b/dotnet/src/dotnetframework/GxClasses/Core/GXUtils.cs index e4354f935..0cb3458a0 100644 --- a/dotnet/src/dotnetframework/GxClasses/Core/GXUtils.cs +++ b/dotnet/src/dotnetframework/GxClasses/Core/GXUtils.cs @@ -1139,8 +1139,9 @@ public bool UploadPrivate(string filefullpath, string storageobjectfullname, GxF { storageobjectfullname=Path.GetFileName(filefullpath); } - string url = provider.Upload(filefullpath, storageobjectfullname, GxFileType.Private); - uploadedFile.FileInfo = new GxExternalFileInfo(storageobjectfullname, url, provider); + GxFileType acl = GxFileType.Private; + string url = provider.Upload(filefullpath, storageobjectfullname, acl); + uploadedFile.FileInfo = new GxExternalFileInfo(storageobjectfullname, url, provider, acl); return true; } catch (Exception ex) @@ -1197,7 +1198,8 @@ public bool Get(string storageobjectfullname, GxFile externalFile, GXBaseCollect try { ValidProvider(); - string url = provider.Get(storageobjectfullname, GxFileType.PublicRead, 0); + GxFileType acl = GxFileType.PublicRead; + string url = provider.Get(storageobjectfullname, acl, 0); if (String.IsNullOrEmpty(url)) { GXUtil.ErrorToMessages("Get Error", "File doesn't exists", messages); @@ -1205,7 +1207,7 @@ public bool Get(string storageobjectfullname, GxFile externalFile, GXBaseCollect } else { - externalFile.FileInfo = new GxExternalFileInfo(storageobjectfullname, url, provider); + externalFile.FileInfo = new GxExternalFileInfo(storageobjectfullname, url, provider, acl); return true; } } @@ -1222,7 +1224,8 @@ public bool GetPrivate(string storageobjectfullname, GxFile externalFile, int ex try { ValidProvider(); - string url = provider.Get(storageobjectfullname, GxFileType.Private, expirationMinutes); + GxFileType acl = GxFileType.Private; + string url = provider.Get(storageobjectfullname, acl, expirationMinutes); if (String.IsNullOrEmpty(url)) { GXUtil.ErrorToMessages("Get Error", "File doesn't exists", messages); @@ -1230,7 +1233,7 @@ public bool GetPrivate(string storageobjectfullname, GxFile externalFile, int ex } else { - externalFile.FileInfo = new GxExternalFileInfo(storageobjectfullname, url, provider, GxFileType.Private); + externalFile.FileInfo = new GxExternalFileInfo(storageobjectfullname, url, provider, acl); return true; } } diff --git a/dotnet/src/dotnetframework/GxClasses/Data/GXDataNTierADO.cs b/dotnet/src/dotnetframework/GxClasses/Data/GXDataNTierADO.cs index 10befd758..4ea624d2b 100644 --- a/dotnet/src/dotnetframework/GxClasses/Data/GXDataNTierADO.cs +++ b/dotnet/src/dotnetframework/GxClasses/Data/GXDataNTierADO.cs @@ -617,7 +617,7 @@ public void SetParameterMultimedia(int id, string image_gxi, string image, strin { try { - multimediaUri = ServiceFactory.GetExternalProvider().Copy(image_gxi, GXDbFile.GenerateUri(image_gxi, !GXDbFile.HasToken(image_gxi), false), tableName, fieldName, GxFileType.PublicRead); + multimediaUri = ServiceFactory.GetExternalProvider().Copy(image_gxi, GXDbFile.GenerateUri(image_gxi, !GXDbFile.HasToken(image_gxi), false), tableName, fieldName, GxFileType.Default); GXLogging.Debug(log, "Copy file already in ExternalProvider:", multimediaUri); } catch (Exception ex) @@ -635,7 +635,7 @@ public void SetParameterMultimedia(int id, string image_gxi, string image, strin using (var fileStream = new MemoryStream(new WebClient().DownloadData(image_gxi))) { //Cannot pass Http Stream directly, because some Providers (AWS S3) does not support Http Stream. - multimediaUri = ServiceFactory.GetExternalProvider().Save(fileStream, GXDbFile.GenerateUri(image_gxi, !GXDbFile.HasToken(image_gxi), false), tableName, fieldName, GxFileType.PublicRead); + multimediaUri = ServiceFactory.GetExternalProvider().Save(fileStream, GXDbFile.GenerateUri(image_gxi, !GXDbFile.HasToken(image_gxi), false), tableName, fieldName, GxFileType.Default); GXLogging.Debug(log, "Upload external file to ExternalProvider:", multimediaUri); } #pragma warning disable SYSLIB0014 // WebClient @@ -657,7 +657,7 @@ public void SetParameterMultimedia(int id, string image_gxi, string image, strin String fileName = PathUtil.GetValidFileName(fileFullName, "_"); using (fileStream) { - multimediaUri = ServiceFactory.GetExternalProvider().Save(fileStream, GXDbFile.GenerateUri(fileName, !GXDbFile.HasToken(fileName), false), tableName, fieldName, GxFileType.PublicRead); + multimediaUri = ServiceFactory.GetExternalProvider().Save(fileStream, GXDbFile.GenerateUri(fileName, !GXDbFile.HasToken(fileName), false), tableName, fieldName, GxFileType.Default); GXLogging.Debug(log, "Upload file (_gxi) to ExternalProvider:", multimediaUri); } } @@ -665,7 +665,7 @@ public void SetParameterMultimedia(int id, string image_gxi, string image, strin { try { - multimediaUri = ServiceFactory.GetExternalProvider().Copy(image, GXDbFile.GenerateUri(image_gxi, !GXDbFile.HasToken(image_gxi), false), tableName, fieldName, GxFileType.PublicRead); + multimediaUri = ServiceFactory.GetExternalProvider().Copy(image, GXDbFile.GenerateUri(image_gxi, !GXDbFile.HasToken(image_gxi), false), tableName, fieldName, GxFileType.Default); GXLogging.Debug(log, "Copy external file in ExternalProvider:", multimediaUri); } catch(Exception e) diff --git a/dotnet/src/dotnetframework/GxClasses/Domain/GXFileIO.cs b/dotnet/src/dotnetframework/GxClasses/Domain/GXFileIO.cs index 963328abf..34cee20b8 100644 --- a/dotnet/src/dotnetframework/GxClasses/Domain/GXFileIO.cs +++ b/dotnet/src/dotnetframework/GxClasses/Domain/GXFileIO.cs @@ -17,6 +17,7 @@ using GeneXus; using GeneXus.Storage; using GeneXus.Attributes; +using GxClasses.Services.Storage; public interface IGxDirectoryInfo { @@ -412,7 +413,7 @@ public class GxExternalFileInfo : IGxFileInfo private string _name; private ExternalProvider _provider; private string _url; - private GxFileType _fileTypeAtt = GxFileType.PublicRead; + private GxFileType _fileTypeAtt = GxFileType.Private; public GxExternalFileInfo(ExternalProvider provider) { @@ -432,20 +433,16 @@ public GxExternalFileInfo(string storageObjectFullname, ExternalProvider provide _url = storageObjectFullname; } else { - if (fileType.HasFlag(GxFileType.Private)) //Attributes multimedia consider Storage Provider Folder + /*if (fileType.HasFlag(GxFileType.Private)) //Attributes multimedia consider Storage Provider Folder { _url = provider.GetBaseURL() + storageObjectFullname; - _name = _url.Replace(provider.StorageUri, string.Empty); - if (_name.StartsWith("/")) - { - _name = _name.Substring(1, _name.Length - 1); - } - } + _name = ExternalProviderCommon.getProviderObjectName(provider, storageObjectFullname); + }*/ } _fileTypeAtt = fileType; } - public GxExternalFileInfo(string storageObjectFullname, string url, ExternalProvider provider, GxFileType fileType = GxFileType.PublicRead) + public GxExternalFileInfo(string storageObjectFullname, string url, ExternalProvider provider, GxFileType fileType = GxFileType.Private) { _name = storageObjectFullname; _provider = provider; diff --git a/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderBase.cs b/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderBase.cs index a495f88a3..c80108011 100644 --- a/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderBase.cs +++ b/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderBase.cs @@ -27,6 +27,16 @@ public ExternalProviderBase() public ExternalProviderBase(GXService s) { + if (s == null) { + try + { + s = ServiceFactory.GetGXServices().Get(GXServices.STORAGE_SERVICE); + } + catch (Exception) { + logger.Warn("STORAGE_SERVICE is not activated in CloudServices.config"); + } + } + this.service = s; Initialize(); } @@ -39,6 +49,10 @@ private void Initialize() if (!String.IsNullOrEmpty(aclS)) { GxFileType.TryParse(aclS, out this.defaultAcl); + } + if (this.defaultAcl == GxFileType.Default) + { + this.defaultAcl = GxFileType.PublicRead; } } @@ -85,7 +99,7 @@ protected String GetPropertyValue(String propertyName, String alternativePropert protected String GetPropertyValue(String propertyName, String alternativePropertyName, String defaultValue) { - String value = String.Empty; + String value = null; value = string.IsNullOrEmpty(value) ? GetPropertyValueImpl(ResolvePropertyName(propertyName)) : value; value = string.IsNullOrEmpty(value) ? GetPropertyValueImpl(propertyName) : value; value = string.IsNullOrEmpty(value) ? GetPropertyValueImpl(alternativePropertyName) : value; @@ -95,7 +109,7 @@ protected String GetPropertyValue(String propertyName, String alternativePropert private string GetPropertyValueImpl(string propertyName) { - String value = String.Empty; + String value = null; if (!string.IsNullOrEmpty(propertyName)) { value = Environment.GetEnvironmentVariable(propertyName); diff --git a/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderCommon.cs b/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderCommon.cs new file mode 100644 index 000000000..c4c625709 --- /dev/null +++ b/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderCommon.cs @@ -0,0 +1,36 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using GeneXus.Services; + +namespace GxClasses.Services.Storage +{ + public static class ExternalProviderCommon + { + public static String getProviderObjectName(ExternalProvider provider, String objectNameOrUrl) + { + String providerObjectName = null; + if (provider != null) + { + //providerObjectName = provider.GetObjectNameFromURL(objectNameOrUrl); + if (providerObjectName != null && providerObjectName.IndexOf("?") > 0) + { + providerObjectName = providerObjectName.Substring(0, providerObjectName.IndexOf("?")); + } + } + return providerObjectName; + } + + public static String getProviderObjectAbsoluteUriSafe(ExternalProvider provider, String rawObjectUri) + { + String providerObjectName = getProviderObjectName(provider, rawObjectUri); + if (providerObjectName != null && rawObjectUri.IndexOf("?") > 0) + { + rawObjectUri = rawObjectUri.Substring(0, rawObjectUri.IndexOf("?")); + } + return rawObjectUri; + } + + } +} diff --git a/dotnet/test/DotNetUnitTest/DotNetUnitTest.csproj b/dotnet/test/DotNetUnitTest/DotNetUnitTest.csproj index be0c33de9..6ae0848b1 100644 --- a/dotnet/test/DotNetUnitTest/DotNetUnitTest.csproj +++ b/dotnet/test/DotNetUnitTest/DotNetUnitTest.csproj @@ -52,6 +52,9 @@ Always + + Always + From 6b2bee02d4bdf9847024374ca0e83f63065b8b4e Mon Sep 17 00:00:00 2001 From: Claudia Murialdo Date: Fri, 4 Jun 2021 10:39:02 -0300 Subject: [PATCH 09/33] Always add a GUID suffix to the name of uploaded image files. (cherry picked from commit 57988be16a30a68a24cbc3e291d06a3bdf311db0) --- .../dotnetframework/GxClasses/Core/GXUtilsCommon.cs | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/dotnet/src/dotnetframework/GxClasses/Core/GXUtilsCommon.cs b/dotnet/src/dotnetframework/GxClasses/Core/GXUtilsCommon.cs index a3dc44cb7..6f56fa6f1 100644 --- a/dotnet/src/dotnetframework/GxClasses/Core/GXUtilsCommon.cs +++ b/dotnet/src/dotnetframework/GxClasses/Core/GXUtilsCommon.cs @@ -3373,16 +3373,8 @@ public static string getTempFileName(string baseDir, string name, string extensi lock (syncObj) { - name = FixFileName(name, string.Empty); - if (string.IsNullOrEmpty(name)) - { - fileName = tempFileName(baseDir, name, extension); - } - else - { - fileName = PathUtil.SafeCombine(baseDir, name + "." + extension); - } + fileName = tempFileName(baseDir, name, extension); GxFile file = new GxFile(baseDir, fileName, fileType); while (file.FileInfo.Exist(fileName)) { From 671899b66ffc79d86217f39dfcc20012cfc11f95 Mon Sep 17 00:00:00 2001 From: Claudia Murialdo Date: Mon, 7 Jun 2021 16:28:00 -0300 Subject: [PATCH 10/33] Support for Private storage (cherry picked from commit 08dcb7fb577e91980b882ecfd2a695bd389d488e) --- .../GxClasses/Core/GXApplication.cs | 7 +-- .../dotnetframework/GxClasses/Core/GXUtils.cs | 5 +- .../GxClasses/Core/GXUtilsCommon.cs | 48 ++++++++++--------- .../GxClasses/Data/GXDataNTierADO.cs | 15 +++--- .../GxClasses/Domain/GXFileIO.cs | 16 +++---- .../Services/Storage/ExternalProviderBase.cs | 15 +++++- .../Storage/ExternalProviderCommon.cs | 36 -------------- .../GxClasses/Services/Storage/GXServices.cs | 4 +- .../GxClasses/Storage/StorageFactory.cs | 29 ++++++++++- .../src/dotnetframework/GxExcel/GxExcelI.cs | 2 +- .../Storage/GXAmazonS3/ExternalProviderS3.cs | 9 ++-- .../AzureStorageExternalProvider.cs | 4 +- .../GXGoogleCloud/ExternalProviderGoogle.cs | 8 ++-- .../GXOpenStack/ExternalProviderOpenStack.cs | 2 +- .../ExternalProvider/ExternalProviderTest.cs | 6 +-- 15 files changed, 102 insertions(+), 104 deletions(-) delete mode 100644 dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderCommon.cs diff --git a/dotnet/src/dotnetframework/GxClasses/Core/GXApplication.cs b/dotnet/src/dotnetframework/GxClasses/Core/GXApplication.cs index 1d4b532ea..e90e307a3 100644 --- a/dotnet/src/dotnetframework/GxClasses/Core/GXApplication.cs +++ b/dotnet/src/dotnetframework/GxClasses/Core/GXApplication.cs @@ -679,13 +679,10 @@ public static bool GetHttpRequestPostedFile(IGxContext gxContext, string varName if (httpContext != null) { HttpPostedFile pf = httpContext.Request.GetFile(varName); - if (pf != null) + if (pf != null && pf.ContentLength > 0) { -#pragma warning disable SCS0018 // Path traversal: injection possible in {1} argument passed to '{0}' - FileInfo fi = new FileInfo(pf.FileName); -#pragma warning restore SCS0018 // Path traversal: injection possible in {1} argument passed to '{0}' string tempDir = Preferences.getTMP_MEDIA_PATH(); - string ext = fi.Extension; + string ext = Path.GetExtension(pf.FileName); if (ext != null) ext = ext.TrimStart('.'); string filePath = FileUtil.getTempFileName(tempDir); diff --git a/dotnet/src/dotnetframework/GxClasses/Core/GXUtils.cs b/dotnet/src/dotnetframework/GxClasses/Core/GXUtils.cs index 0cb3458a0..23d3d8c80 100644 --- a/dotnet/src/dotnetframework/GxClasses/Core/GXUtils.cs +++ b/dotnet/src/dotnetframework/GxClasses/Core/GXUtils.cs @@ -1116,12 +1116,13 @@ public bool Upload(string filefullpath, string storageobjectfullname, GxFile upl try { ValidProvider(); + GxFileType acl = GxFileType.PublicRead; if (String.IsNullOrEmpty(storageobjectfullname)) { storageobjectfullname = Path.GetFileName(filefullpath); } - string url = provider.Upload(filefullpath, storageobjectfullname, GxFileType.PublicRead); - uploadedFile.FileInfo = new GxExternalFileInfo(storageobjectfullname, url, provider); + string url = provider.Upload(filefullpath, storageobjectfullname, acl); + uploadedFile.FileInfo = new GxExternalFileInfo(storageobjectfullname, url, provider, acl); return true; } catch (Exception ex) diff --git a/dotnet/src/dotnetframework/GxClasses/Core/GXUtilsCommon.cs b/dotnet/src/dotnetframework/GxClasses/Core/GXUtilsCommon.cs index 6f56fa6f1..2ec2b3ecd 100644 --- a/dotnet/src/dotnetframework/GxClasses/Core/GXUtilsCommon.cs +++ b/dotnet/src/dotnetframework/GxClasses/Core/GXUtilsCommon.cs @@ -37,6 +37,8 @@ using System.Security.Cryptography; using System.Collections.Concurrent; using System.Drawing.Drawing2D; +using GeneXus.Storage; +using GeneXus.Services; namespace GeneXus.Utils { @@ -3272,8 +3274,6 @@ static public DateTime FromTimeZone(DateTime dt, String sTZ, IGxContext context) public class FileUtil { static readonly ILog log = log4net.LogManager.GetLogger(typeof(GeneXus.Utils.FileUtil)); - - private static object syncObj = new object(); public static byte DeleteFile(string fileName) { try @@ -3366,22 +3366,10 @@ public static string UriToPath(string uriString) return uriString; } } - public static string getTempFileName(string baseDir, string name, string extension, GxFileType fileType = GxFileType.PublicRead) + public static string getTempFileName(string baseDir, string name="", string extension="tmp", GxFileType fileType = GxFileType.Private) { - String fileName; - name = FileUtil.FileNamePrettify(name); - - lock (syncObj) - { - name = FixFileName(name, string.Empty); - fileName = tempFileName(baseDir, name, extension); - GxFile file = new GxFile(baseDir, fileName, fileType); - while (file.FileInfo.Exist(fileName)) - { - fileName = tempFileName(baseDir, name, extension); - } - } - return fileName.Trim(); + name = FixFileName(FileUtil.FileNamePrettify(name), string.Empty); + return tempFileName(baseDir, name, extension); } private static string tempFileName(string baseDir, string name, string extension) @@ -3442,10 +3430,17 @@ public static string GetFileType(string FileName) public static string FileNamePrettify(string s) { - string str = GXUtil.RemoveDiacritics(s.Trim().ToLower()); //remove accents - str = Regex.Replace(str, @"\s+", " ").Trim(); // convert multiple spaces into one space - str = Regex.Replace(str, @"\s", "-"); // //Replace spaces by dashes - return str; + if (!string.IsNullOrEmpty(s)) + { + string str = GXUtil.RemoveDiacritics(s.Trim().ToLower()); //remove accents + str = Regex.Replace(str, @"\s+", " ").Trim(); // convert multiple spaces into one space + str = Regex.Replace(str, @"\s", "-"); // //Replace spaces by dashes + return str; + } + else + { + return s; + } } public static string GetFileName(string FileName) @@ -5249,7 +5244,14 @@ public static string ResolveUri(string uriString, IGxContext context = null) public static string ResolveUri(string uriString, bool absUrl, IGxContext context = null) { if (String.IsNullOrEmpty(uriString)) - return ""; + return string.Empty; + + string providerObjectName; + if (PathUtil.IsAbsoluteUrl(uriString) && StorageFactory.TryGetProviderObjectName(ServiceFactory.GetExternalProvider(), uriString, out providerObjectName)) + { + return new GxFile(string.Empty, providerObjectName).GetURI(); + } + if (schemeRegex.IsMatch(uriString)) { string fileName = schemeRegex.Replace(uriString, ""); @@ -5257,7 +5259,7 @@ public static string ResolveUri(string uriString, bool absUrl, IGxContext contex string basePath = Path.Combine(Path.Combine(Preferences.getBLOB_PATH(), MultimediaDirectory)); try { - GxFile file = new GxFile(string.Empty, PathUtil.SafeCombine(basePath, fileName), GxFileType.PublicRead); + GxFile file = new GxFile(string.Empty, PathUtil.SafeCombine(basePath, fileName)); return PathToUrl(file.GetURI(), absUrl, context); } catch (ArgumentException ex) diff --git a/dotnet/src/dotnetframework/GxClasses/Data/GXDataNTierADO.cs b/dotnet/src/dotnetframework/GxClasses/Data/GXDataNTierADO.cs index 4ea624d2b..8aa134c91 100644 --- a/dotnet/src/dotnetframework/GxClasses/Data/GXDataNTierADO.cs +++ b/dotnet/src/dotnetframework/GxClasses/Data/GXDataNTierADO.cs @@ -10,8 +10,7 @@ using System.Collections.Generic; using GeneXus.Services; using System.Net; -using System.Diagnostics; -using System.Linq; +using GeneXus.Storage; namespace GeneXus.Data.NTier.ADO { @@ -149,7 +148,7 @@ public string getMultimediaFile(int id, string gxdbFileUri) string filePath = PathUtil.SafeCombine(_gxDbCommand.Conn.MultimediaPath, fileName); try { - GxFile file = new GxFile(string.Empty, filePath, GxFileType.PublicRead); + GxFile file = new GxFile(string.Empty, filePath, GxFileType.Default); if (file.Exists()) { return filePath; @@ -433,7 +432,7 @@ public string getMultimediaFile(int id, string gxdbFileUri) try { - GxFile file = new GxFile(string.Empty, filePath, GxFileType.PublicRead); + GxFile file = new GxFile(string.Empty, filePath, GxFileType.Default); if (file.Exists()) { @@ -441,7 +440,7 @@ public string getMultimediaFile(int id, string gxdbFileUri) } else { - return getBLOBFile(id, FileUtil.GetFileType(gxdbFileUri), FileUtil.GetFileName(gxdbFileUri), filePath, false, GxFileType.PublicRead); + return getBLOBFile(id, FileUtil.GetFileType(gxdbFileUri), FileUtil.GetFileName(gxdbFileUri), filePath, false, GxFileType.Default); } } catch (ArgumentException) @@ -456,7 +455,7 @@ public string getMultimediaFile(int id, string gxdbFileUri) public string getMultimediaUri(int id) { - return GXDbFile.ResolveUri(getVarchar(id), true, _gxDbCommand.Conn.DataStore.Context); + return getMultimediaUri(id, true); } public string getMultimediaUri(int id, bool absUrl) @@ -613,7 +612,7 @@ public void SetParameterMultimedia(int id, string image_gxi, string image, strin string objectName; //file is already on the cloud p.e. https://s3.amazonaws.com/Test/PublicTempStorage/multimedia/Image_ad013b5b050c4bf199f544b5561d9b92.png //Must be copied to https://s3.amazonaws.com/Test/TableName/FieldName/Image_ad013b5b050c4bf199f544b5561d9b92.png - if (ServiceFactory.GetExternalProvider().GetObjectNameFromURL(image_gxi, out objectName)) + if (ServiceFactory.GetExternalProvider().TryGetObjectNameFromURL(image_gxi, out objectName)) { try { @@ -726,7 +725,7 @@ private static string PushToExternalProvider(Stream fileStream, string externalF string multimediaUri; using (fileStream) { - multimediaUri = ServiceFactory.GetExternalProvider().Save(fileStream, externalFileName, tableName, fieldName, GxFileType.PublicRead); + multimediaUri = ServiceFactory.GetExternalProvider().Save(fileStream, externalFileName, tableName, fieldName, GxFileType.Default); GXLogging.Debug(log, "Upload file to ExternalProvider:", multimediaUri); } diff --git a/dotnet/src/dotnetframework/GxClasses/Domain/GXFileIO.cs b/dotnet/src/dotnetframework/GxClasses/Domain/GXFileIO.cs index 34cee20b8..d2fd14d78 100644 --- a/dotnet/src/dotnetframework/GxClasses/Domain/GXFileIO.cs +++ b/dotnet/src/dotnetframework/GxClasses/Domain/GXFileIO.cs @@ -17,7 +17,6 @@ using GeneXus; using GeneXus.Storage; using GeneXus.Attributes; -using GxClasses.Services.Storage; public interface IGxDirectoryInfo { @@ -210,7 +209,7 @@ public IGxFileInfo[] GetFiles(string searchPattern) searchPattern = ""; List files = _provider.GetFiles(_name, searchPattern); - GxExternalFileInfo[] externalFiles = files.Select(elem => new GxExternalFileInfo(elem, _provider, GxFileType.PublicRead)).ToArray(); + GxExternalFileInfo[] externalFiles = files.Select(elem => new GxExternalFileInfo(elem, _provider, GxFileType.Default)).ToArray(); return externalFiles; } @@ -443,9 +442,9 @@ public GxExternalFileInfo(string storageObjectFullname, ExternalProvider provide } public GxExternalFileInfo(string storageObjectFullname, string url, ExternalProvider provider, GxFileType fileType = GxFileType.Private) - { - _name = storageObjectFullname; - _provider = provider; + { + _name = StorageFactory.GetProviderObjectAbsoluteUriSafe(provider, storageObjectFullname); + _provider = provider; _url = url; _fileTypeAtt = fileType; } @@ -667,13 +666,13 @@ public GxFile(string baseDirectory) _baseDirectory = _baseDirectory.Substring(0, _baseDirectory.Length - 1); } - public GxFile(string baseDirectory, IGxFileInfo file, GxFileType fileType = GxFileType.PublicRead) + public GxFile(string baseDirectory, IGxFileInfo file, GxFileType fileType = GxFileType.Private) : this(baseDirectory) { _file = file; } - public GxFile(string baseDirectory, string fileName, GxFileType fileType = GxFileType.PublicRead) + public GxFile(string baseDirectory, string fileName, GxFileType fileType = GxFileType.Private) : this(baseDirectory) { if (GxUploadHelper.IsUpload(fileName)) @@ -739,8 +738,7 @@ public string Source if (IsAbsoluteUrl(value)) { string objName = string.Empty; - ExternalProvider p = StorageFactory.GetExternalProviderFromUrl(value, out objName); - _file = new GxExternalFileInfo(objName, value, p); + _file = new GxExternalFileInfo(objName, value, ServiceFactory.GetExternalProvider()); } else { diff --git a/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderBase.cs b/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderBase.cs index c80108011..5a9bc4b7c 100644 --- a/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderBase.cs +++ b/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderBase.cs @@ -14,8 +14,8 @@ public abstract class ExternalProviderBase protected static String DEFAULT_EXPIRATION = "DEFAULT_EXPIRATION"; protected static String FOLDER = "FOLDER_NAME"; protected static String DEFAULT_ACL_DEPRECATED = "STORAGE_PROVIDER_DEFAULT_ACL"; - protected static String DEFAULT_EXPIRATION_DEPRECATED = "STORAGE_PROVIDER_DEFAULT_EXPIRATION"; - protected const int DefaultExpirationMinutes = 24 * 60; + protected static String DEFAULT_EXPIRATION_DEPRECATED = "STORAGE_PROVIDER_DEFAULT_EXPIRATION"; + protected int defaultExpirationMinutes = 1440; protected GxFileType defaultAcl = GxFileType.Private; @@ -54,6 +54,17 @@ private void Initialize() { this.defaultAcl = GxFileType.PublicRead; } + + String expirationS = GetPropertyValue(DEFAULT_EXPIRATION, DEFAULT_EXPIRATION, defaultExpirationMinutes.ToString()); + if (!String.IsNullOrEmpty(expirationS)) + { + Int32.TryParse(expirationS, out this.defaultExpirationMinutes); + } + } + + protected int ResolveExpirationMinutes(int expirationMinutes) + { + return expirationMinutes > 0 ? expirationMinutes : defaultExpirationMinutes; } protected String GetEncryptedPropertyValue(String propertyName, String alternativePropertyName = null) diff --git a/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderCommon.cs b/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderCommon.cs deleted file mode 100644 index c4c625709..000000000 --- a/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderCommon.cs +++ /dev/null @@ -1,36 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; -using GeneXus.Services; - -namespace GxClasses.Services.Storage -{ - public static class ExternalProviderCommon - { - public static String getProviderObjectName(ExternalProvider provider, String objectNameOrUrl) - { - String providerObjectName = null; - if (provider != null) - { - //providerObjectName = provider.GetObjectNameFromURL(objectNameOrUrl); - if (providerObjectName != null && providerObjectName.IndexOf("?") > 0) - { - providerObjectName = providerObjectName.Substring(0, providerObjectName.IndexOf("?")); - } - } - return providerObjectName; - } - - public static String getProviderObjectAbsoluteUriSafe(ExternalProvider provider, String rawObjectUri) - { - String providerObjectName = getProviderObjectName(provider, rawObjectUri); - if (providerObjectName != null && rawObjectUri.IndexOf("?") > 0) - { - rawObjectUri = rawObjectUri.Substring(0, rawObjectUri.IndexOf("?")); - } - return rawObjectUri; - } - - } -} diff --git a/dotnet/src/dotnetframework/GxClasses/Services/Storage/GXServices.cs b/dotnet/src/dotnetframework/GxClasses/Services/Storage/GXServices.cs index 399e02a56..f78e0a202 100644 --- a/dotnet/src/dotnetframework/GxClasses/Services/Storage/GXServices.cs +++ b/dotnet/src/dotnetframework/GxClasses/Services/Storage/GXServices.cs @@ -247,9 +247,9 @@ public interface ExternalProvider List GetFiles(string directoryName, string filter = ""); List GetSubDirectories(string directoryName); Stream GetStream(string objectName, GxFileType fileType); - bool GetMessageFromException(Exception ex, SdtMessages_Message msg); - bool GetObjectNameFromURL(string url, out string objectName); + bool GetMessageFromException(Exception ex, SdtMessages_Message msg); string GetBaseURL(); + bool TryGetObjectNameFromURL(string objectNameOrUrl, out string providerObjectName); string StorageUri { get; } } } diff --git a/dotnet/src/dotnetframework/GxClasses/Storage/StorageFactory.cs b/dotnet/src/dotnetframework/GxClasses/Storage/StorageFactory.cs index 0ef2a2d3f..eeb36091f 100644 --- a/dotnet/src/dotnetframework/GxClasses/Storage/StorageFactory.cs +++ b/dotnet/src/dotnetframework/GxClasses/Storage/StorageFactory.cs @@ -5,13 +5,13 @@ namespace GeneXus.Storage public class StorageFactory { const char QUESTION_MARK = '?'; - public static ExternalProvider GetExternalProviderFromUrl(string url, out string objectName) + /*public static ExternalProvider GetExternalProviderFromUrl(string url, out string objectName) { objectName = null; ExternalProvider provider = ServiceFactory.GetExternalProvider(); if (provider != null) { - if (provider.GetObjectNameFromURL(url, out objectName)) + if (provider.TryGetObjectNameFromURL(url, out objectName)) { var questionMarkIndex = objectName.IndexOf(QUESTION_MARK); objectName = questionMarkIndex >= 0 ? objectName.Substring(0, questionMarkIndex): objectName.Substring(0); @@ -20,5 +20,30 @@ public static ExternalProvider GetExternalProviderFromUrl(string url, out string } return null; } + */ + public static bool TryGetProviderObjectName(ExternalProvider provider, string objectNameOrUrl, out string providerObjectName) + { + providerObjectName = null; + if (provider != null && provider.TryGetObjectNameFromURL(objectNameOrUrl, out providerObjectName)) + { + int idx = providerObjectName.IndexOf(QUESTION_MARK); + if (idx > 0) + { + providerObjectName = providerObjectName.Substring(0, idx); + } + return true; + } + return false; + } + + public static string GetProviderObjectAbsoluteUriSafe(ExternalProvider provider, string rawObjectUri) + { + string providerObjectName; + if (TryGetProviderObjectName(provider, rawObjectUri, out providerObjectName)) + { + rawObjectUri = providerObjectName; + } + return rawObjectUri; + } } } diff --git a/dotnet/src/dotnetframework/GxExcel/GxExcelI.cs b/dotnet/src/dotnetframework/GxExcel/GxExcelI.cs index 79168b9f3..27d4c25a9 100644 --- a/dotnet/src/dotnetframework/GxExcel/GxExcelI.cs +++ b/dotnet/src/dotnetframework/GxExcel/GxExcelI.cs @@ -144,7 +144,7 @@ public String Template string localTemplate = value; if (!Path.IsPathRooted(localTemplate)) localTemplate = Path.Combine(GxContext.StaticPhysicalPath(), localTemplate); - ServiceFactory.GetExternalProvider().Upload(localTemplate, template, GxFileType.PublicRead); + ServiceFactory.GetExternalProvider().Upload(localTemplate, template, GxFileType.Private); } } else if (!Path.IsPathRooted(value)) diff --git a/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs b/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs index 28ca58f00..afebd64ad 100644 --- a/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs +++ b/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs @@ -247,7 +247,7 @@ private bool IsPrivateUpload(GxFileType fileType) return GetCannedACL(fileType) != S3CannedACL.PublicRead; } - public string Get(string objectName, GxFileType fileType, int urlMinutes = DefaultExpirationMinutes) + public string Get(string objectName, GxFileType fileType, int urlMinutes = 0) { if (Exists(objectName, fileType)) { @@ -257,10 +257,10 @@ public string Get(string objectName, GxFileType fileType, int urlMinutes = Defau return string.Empty; } - private string GetUrlImpl(string objectName, GxFileType fileType, int urlMinutes = DefaultExpirationMinutes) + private string GetUrlImpl(string objectName, GxFileType fileType, int urlMinutes = 0) { bool isPrivate = IsPrivateUpload(fileType); - return (isPrivate)? GetPreSignedUrl(objectName, urlMinutes): StorageUri + StorageUtils.EncodeUrl(objectName); + return (isPrivate)? GetPreSignedUrl(objectName, ResolveExpirationMinutes(urlMinutes)): StorageUri + StorageUtils.EncodeUrl(objectName); } @@ -521,7 +521,7 @@ public bool GetMessageFromException(Exception ex, SdtMessages_Message msg) } } - public bool GetObjectNameFromURL(string url, out string objectName) + public bool TryGetObjectNameFromURL(string url, out string objectName) { if (url.StartsWith(StorageUri)) { @@ -536,5 +536,6 @@ public override string GetName() { return Name; } + } } diff --git a/dotnet/src/dotnetframework/Providers/Storage/GXAzureStorage/AzureStorageExternalProvider.cs b/dotnet/src/dotnetframework/Providers/Storage/GXAzureStorage/AzureStorageExternalProvider.cs index 1b46ada6c..5c5351db8 100644 --- a/dotnet/src/dotnetframework/Providers/Storage/GXAzureStorage/AzureStorageExternalProvider.cs +++ b/dotnet/src/dotnetframework/Providers/Storage/GXAzureStorage/AzureStorageExternalProvider.cs @@ -125,7 +125,7 @@ private string GetURL(CloudBlockBlob blob, GxFileType fileType, int urlMinutes = { SharedAccessBlobPolicy sasConstraints = new SharedAccessBlobPolicy(); sasConstraints.SharedAccessStartTime = DateTime.UtcNow; - sasConstraints.SharedAccessExpiryTime = DateTime.UtcNow.AddMinutes((urlMinutes > 0) ? urlMinutes : 60 * 24 * 7); + sasConstraints.SharedAccessExpiryTime = DateTime.UtcNow.AddMinutes(ResolveExpirationMinutes(urlMinutes)); sasConstraints.Permissions = SharedAccessBlobPermissions.Read; url += blob.GetSharedAccessSignature(sasConstraints); } @@ -352,7 +352,7 @@ public string Save(Stream fileStream, string fileName, string tableName, string return Upload(newName, fileStream, fileType); } - public bool GetObjectNameFromURL(string url, out string objectName) + public bool TryGetObjectNameFromURL(string url, out string objectName) { string baseUrl = StorageUri + StorageUtils.DELIMITER + PublicContainer.Name + StorageUtils.DELIMITER; if (url.StartsWith(baseUrl)) diff --git a/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs b/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs index 040ca64b8..e1fcdbdb1 100644 --- a/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs +++ b/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs @@ -192,7 +192,7 @@ public string Upload(string localFile, string objectName, GxFileType fileType) using (FileStream stream = new FileStream(localFile, FileMode.Open)) { Google.Apis.Storage.v1.Data.Object obj = Client.UploadObject(Bucket, objectName, "application/octet-stream", stream, GetUploadOptions(fileType)); - return GetURL(objectName, fileType, DefaultExpirationMinutes); + return GetURL(objectName, fileType); } } @@ -245,10 +245,10 @@ public string Get(string objectName, GxFileType fileType, int urlMinutes) return string.Empty; } - private string GetURL(string objectName, GxFileType fileType, int urlMinutes = 60 * 24 * 7) + private string GetURL(string objectName, GxFileType fileType, int urlMinutes = 0) { if (fileType.HasFlag(GxFileType.Private)) - return Signer.Sign(Bucket, StorageUtils.EncodeUrl(objectName), TimeSpan.FromMinutes(urlMinutes), HttpMethod.Get); + return Signer.Sign(Bucket, StorageUtils.EncodeUrl(objectName), TimeSpan.FromMinutes(ResolveExpirationMinutes(urlMinutes)), HttpMethod.Get); else { return StorageUri + StorageUtils.EncodeUrl(objectName); @@ -433,7 +433,7 @@ public string Save(Stream fileStream, string fileName, string tableName, string fileName = Folder + StorageUtils.DELIMITER + tableName + StorageUtils.DELIMITER + fieldName + StorageUtils.DELIMITER + fileName; return Upload(fileName, fileStream, fileType); } - public bool GetObjectNameFromURL(string url, out string objectName) + public bool TryGetObjectNameFromURL(string url, out string objectName) { string baseUrl = StorageUri; if (url.StartsWith(baseUrl)) diff --git a/dotnet/src/dotnetframework/Providers/Storage/GXOpenStack/ExternalProviderOpenStack.cs b/dotnet/src/dotnetframework/Providers/Storage/GXOpenStack/ExternalProviderOpenStack.cs index a8aaddff1..bd9a94348 100644 --- a/dotnet/src/dotnetframework/Providers/Storage/GXOpenStack/ExternalProviderOpenStack.cs +++ b/dotnet/src/dotnetframework/Providers/Storage/GXOpenStack/ExternalProviderOpenStack.cs @@ -362,7 +362,7 @@ public string GetBaseURL() return storageUrl + StorageUtils.DELIMITER + publicBucketName + StorageUtils.DELIMITER; } - public bool GetObjectNameFromURL(string url, out string objectName) + public bool TryGetObjectNameFromURL(string url, out string objectName) { string baseUrl = storageUrl + StorageUtils.DELIMITER + publicBucketName + StorageUtils.DELIMITER; if (url.StartsWith(baseUrl)) diff --git a/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs b/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs index b7dae0295..bbe764a12 100644 --- a/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs +++ b/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs @@ -156,10 +156,10 @@ public void TestGetMethod() public void TestGetObjectName() { TestUploadPublicMethod(); - String url = provider.Get(TEST_SAMPLE_FILE_NAME, GxFileType.PublicRead, 10); + string url = provider.Get(TEST_SAMPLE_FILE_NAME, GxFileType.PublicRead, 10); Assert.True(UrlExists(url)); - String objectName = ""; - provider.GetObjectNameFromURL(url, out objectName); + string objectName; + provider.TryGetObjectNameFromURL(url, out objectName); Assert.Equal("text.txt", objectName); } From 1d2236e07179ff6765a09d118d8b65e59a826b6a Mon Sep 17 00:00:00 2001 From: Gonzalo Date: Mon, 14 Jun 2021 14:51:14 -0300 Subject: [PATCH 11/33] Add Workflow for External Storage Unit Tests (cherry picked from commit de2ba310e66c0df1efd33de32fcfe67fe17a11e9) --- .github/workflows/External-Storage-Tests.yml | 99 ++++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 .github/workflows/External-Storage-Tests.yml diff --git a/.github/workflows/External-Storage-Tests.yml b/.github/workflows/External-Storage-Tests.yml new file mode 100644 index 000000000..c9ca8874c --- /dev/null +++ b/.github/workflows/External-Storage-Tests.yml @@ -0,0 +1,99 @@ +name: Build + +on: + pull_request: + branches: + - 'master' + - 'release-*' + push: + branches: + - 'master' + - 'beta' + - 'release-*' + +jobs: + build: + env: + GIT_REF: ${{ github.ref }} + GIT_SHA: ${{ github.sha }} + Configuration: Release + SolutionFile: dotnet\DotNetStandardClasses.sln + + runs-on: windows-latest + + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Install .NET Core 3.1 + uses: actions/setup-dotnet@v1 + with: + dotnet-version: '3.1.x' + + - name: Install .NET 6 + uses: actions/setup-dotnet@v1.7.2 + with: + dotnet-version: '6.0.100-preview.3.21202.5' + + - uses: actions/setup-dotnet@v1 + with: + source-url: https://nuget.pkg.github.com/genexuslabs/index.json + env: + NUGET_AUTH_TOKEN: ${{secrets.GITHUB_TOKEN}} + + - name: Calculate environment variables + run: | + $IsPrerelease = !($Env:GIT_REF -match 'release-[0-9]+(?:\.[0-9]+)?$') + echo "IsPrerelease=$IsPrerelease" >> $env:GITHUB_ENV + + $COMMIT_NUMBER = @($(git rev-list --count origin/master..), $(git rev-list --count HEAD))[$IsPrerelease] + + echo "COMMIT_NUMBER=$COMMIT_NUMBER" >> $env:GITHUB_ENV + + - name: Calculate package version + env: + PackageVersionString: ./.github/generatePackageVersion.ps1 + run: | + $NuGetPackageVersion = & "$Env:PackageVersionString" + + Write-Output "Packge version to be used: $NuGetPackageVersion" + + echo "NuGetPackageVersion=$NuGetPackageVersion" >> $env:GITHUB_ENV + + - name: Restore packages + run: dotnet restore $Env:SolutionFile + + - name: Build + run: dotnet build $Env:SolutionFile --no-restore --configuration $Env:Configuration + + - name: Test External Storage + run: | + export AWSS3_TEST_ENABLED=true + export STORAGE_AWSS3_ACCESS_KEY="${{ secrets.AWSS3_ACCESS_KEY }}" + export STORAGE_AWSS3_SECRET_KEY="${{ secrets.AWSS3_SECRET_KEY }}" + export STORAGE_AWSS3_BUCKET_NAME=genexuss3test + export STORAGE_AWSS3_FOLDER_NAME=gxclasses + export STORAGE_AWSS3_REGION=us-east-1 + export IBMCOS_TEST_ENABLED=true + export STORAGE_IBMCOS_ACCESS_KEY="${{ secrets.IBMCOS_ACCESS_KEY }}" + export STORAGE_IBMCOS_SECRET_KEY="${{ secrets.IBMCOS_SECRET_KEY }}" + export STORAGE_IBMCOS_BUCKET_NAME=gxclasses-unit-tests + export STORAGE_IBMCOS_FOLDER_NAME=tests + export STORAGE_IBMCOS_REGION=us-south + export AZUREBS_TEST_ENABLED=true + export STORAGE_AZUREBS_ACCESS_KEY="${{ secrets.AZUREBS_ACCESS_KEY }}" + export STORAGE_AZUREBS_ACCOUNT_NAME="${{ secrets.AZUREBS_ACCOUNT_NAME }}" + export STORAGE_AZUREBS_FOLDER_NAME=tests + export STORAGE_AZUREBS_PUBLIC_CONTAINER_NAME=contluispublic + export STORAGE_AZUREBS_PRIVATE_CONTAINER_NAME=contluisprivate + export GOOGLECS_TEST_ENABLED=true + export STORAGE_GOOGLECS_KEY="${{ secrets.GOOGLECS_KEY }}" + export STORAGE_GOOGLECS_PROJECT_ID=gxjavacloudstorageunittests + export STORAGE_GOOGLECS_BUCKET_NAME=javaclasses-unittests + export STORAGE_GOOGLECS_FOLDER_NAME=gxclasses + export STORAGE_GOOGLECS_APPLICATION_NAME=gxjavacloudstorageunittests + + dotnet test $Env:SolutionFile --no-restore --no-build --configuration $Env:Configuration + From 83d7f738612f60b47febebac78ce5434613a010d Mon Sep 17 00:00:00 2001 From: Gonzalo Date: Mon, 14 Jun 2021 14:58:07 -0300 Subject: [PATCH 12/33] Workflow renamed (cherry picked from commit 05b99b33a91bbe5e3eb3a7dc636207c5691018f6) --- .github/workflows/External-Storage-Tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/External-Storage-Tests.yml b/.github/workflows/External-Storage-Tests.yml index c9ca8874c..2ae7522de 100644 --- a/.github/workflows/External-Storage-Tests.yml +++ b/.github/workflows/External-Storage-Tests.yml @@ -1,4 +1,4 @@ -name: Build +name: External Storage Tests on: pull_request: From 226b578a5eb952d09560b213022169f23084eb6f Mon Sep 17 00:00:00 2001 From: Gonzalo Date: Mon, 14 Jun 2021 15:24:20 -0300 Subject: [PATCH 13/33] Set Environment Variables (cherry picked from commit fa68e1b3ef4cfeed46cb2ad9ae4d767c04b82051) --- .github/workflows/External-Storage-Tests.yml | 48 ++++++++++---------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/.github/workflows/External-Storage-Tests.yml b/.github/workflows/External-Storage-Tests.yml index 2ae7522de..daab0e351 100644 --- a/.github/workflows/External-Storage-Tests.yml +++ b/.github/workflows/External-Storage-Tests.yml @@ -70,30 +70,30 @@ jobs: - name: Test External Storage run: | - export AWSS3_TEST_ENABLED=true - export STORAGE_AWSS3_ACCESS_KEY="${{ secrets.AWSS3_ACCESS_KEY }}" - export STORAGE_AWSS3_SECRET_KEY="${{ secrets.AWSS3_SECRET_KEY }}" - export STORAGE_AWSS3_BUCKET_NAME=genexuss3test - export STORAGE_AWSS3_FOLDER_NAME=gxclasses - export STORAGE_AWSS3_REGION=us-east-1 - export IBMCOS_TEST_ENABLED=true - export STORAGE_IBMCOS_ACCESS_KEY="${{ secrets.IBMCOS_ACCESS_KEY }}" - export STORAGE_IBMCOS_SECRET_KEY="${{ secrets.IBMCOS_SECRET_KEY }}" - export STORAGE_IBMCOS_BUCKET_NAME=gxclasses-unit-tests - export STORAGE_IBMCOS_FOLDER_NAME=tests - export STORAGE_IBMCOS_REGION=us-south - export AZUREBS_TEST_ENABLED=true - export STORAGE_AZUREBS_ACCESS_KEY="${{ secrets.AZUREBS_ACCESS_KEY }}" - export STORAGE_AZUREBS_ACCOUNT_NAME="${{ secrets.AZUREBS_ACCOUNT_NAME }}" - export STORAGE_AZUREBS_FOLDER_NAME=tests - export STORAGE_AZUREBS_PUBLIC_CONTAINER_NAME=contluispublic - export STORAGE_AZUREBS_PRIVATE_CONTAINER_NAME=contluisprivate - export GOOGLECS_TEST_ENABLED=true - export STORAGE_GOOGLECS_KEY="${{ secrets.GOOGLECS_KEY }}" - export STORAGE_GOOGLECS_PROJECT_ID=gxjavacloudstorageunittests - export STORAGE_GOOGLECS_BUCKET_NAME=javaclasses-unittests - export STORAGE_GOOGLECS_FOLDER_NAME=gxclasses - export STORAGE_GOOGLECS_APPLICATION_NAME=gxjavacloudstorageunittests + $Env:AWSS3_TEST_ENABLED=true + $Env:STORAGE_AWSS3_ACCESS_KEY="${{ secrets.AWSS3_ACCESS_KEY }}" + $Env:STORAGE_AWSS3_SECRET_KEY="${{ secrets.AWSS3_SECRET_KEY }}" + $Env:STORAGE_AWSS3_BUCKET_NAME=genexuss3test + $Env:STORAGE_AWSS3_FOLDER_NAME=gxclasses + $Env:STORAGE_AWSS3_REGION=us-east-1 + $Env:IBMCOS_TEST_ENABLED=true + $Env:STORAGE_IBMCOS_ACCESS_KEY="${{ secrets.IBMCOS_ACCESS_KEY }}" + $Env:STORAGE_IBMCOS_SECRET_KEY="${{ secrets.IBMCOS_SECRET_KEY }}" + $Env:STORAGE_IBMCOS_BUCKET_NAME=gxclasses-unit-tests + $Env:STORAGE_IBMCOS_FOLDER_NAME=tests + $Env:STORAGE_IBMCOS_REGION=us-south + $Env:AZUREBS_TEST_ENABLED=true + $Env:STORAGE_AZUREBS_ACCESS_KEY="${{ secrets.AZUREBS_ACCESS_KEY }}" + $Env:STORAGE_AZUREBS_ACCOUNT_NAME="${{ secrets.AZUREBS_ACCOUNT_NAME }}" + $Env:STORAGE_AZUREBS_FOLDER_NAME=tests + $Env:STORAGE_AZUREBS_PUBLIC_CONTAINER_NAME=contluispublic + $Env:STORAGE_AZUREBS_PRIVATE_CONTAINER_NAME=contluisprivate + $Env:GOOGLECS_TEST_ENABLED=true + $Env:STORAGE_GOOGLECS_KEY="${{ secrets.GOOGLECS_KEY }}" + $Env:STORAGE_GOOGLECS_PROJECT_ID=gxjavacloudstorageunittests + $Env:STORAGE_GOOGLECS_BUCKET_NAME=javaclasses-unittests + $Env:STORAGE_GOOGLECS_FOLDER_NAME=gxclasses + $Env:STORAGE_GOOGLECS_APPLICATION_NAME=gxjavacloudstorageunittests dotnet test $Env:SolutionFile --no-restore --no-build --configuration $Env:Configuration From c79759901170bf4669df1c8fccdf0510b5ecff00 Mon Sep 17 00:00:00 2001 From: Gonzalo Date: Wed, 16 Jun 2021 10:18:28 -0300 Subject: [PATCH 14/33] Remove commented code (cherry picked from commit 73dac973ca613b71ebee1fb0ba72eabea2a277a5) --- dotnet/src/dotnetframework/GxClasses/Domain/GXFileIO.cs | 7 ------- 1 file changed, 7 deletions(-) diff --git a/dotnet/src/dotnetframework/GxClasses/Domain/GXFileIO.cs b/dotnet/src/dotnetframework/GxClasses/Domain/GXFileIO.cs index d2fd14d78..55901a29d 100644 --- a/dotnet/src/dotnetframework/GxClasses/Domain/GXFileIO.cs +++ b/dotnet/src/dotnetframework/GxClasses/Domain/GXFileIO.cs @@ -431,13 +431,6 @@ public GxExternalFileInfo(string storageObjectFullname, ExternalProvider provide { _url = storageObjectFullname; } - else { - /*if (fileType.HasFlag(GxFileType.Private)) //Attributes multimedia consider Storage Provider Folder - { - _url = provider.GetBaseURL() + storageObjectFullname; - _name = ExternalProviderCommon.getProviderObjectName(provider, storageObjectFullname); - }*/ - } _fileTypeAtt = fileType; } From a9f167a4d728aac1531711cdf782e386047d3ef9 Mon Sep 17 00:00:00 2001 From: Gonzalo Date: Wed, 16 Jun 2021 10:19:46 -0300 Subject: [PATCH 15/33] Remove commented code (cherry picked from commit ef2692d46807b442b0e8652680c4617407d3d551) --- .../GxClasses/Storage/StorageFactory.cs | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/dotnet/src/dotnetframework/GxClasses/Storage/StorageFactory.cs b/dotnet/src/dotnetframework/GxClasses/Storage/StorageFactory.cs index eeb36091f..f4bc8fdda 100644 --- a/dotnet/src/dotnetframework/GxClasses/Storage/StorageFactory.cs +++ b/dotnet/src/dotnetframework/GxClasses/Storage/StorageFactory.cs @@ -5,22 +5,6 @@ namespace GeneXus.Storage public class StorageFactory { const char QUESTION_MARK = '?'; - /*public static ExternalProvider GetExternalProviderFromUrl(string url, out string objectName) - { - objectName = null; - ExternalProvider provider = ServiceFactory.GetExternalProvider(); - if (provider != null) - { - if (provider.TryGetObjectNameFromURL(url, out objectName)) - { - var questionMarkIndex = objectName.IndexOf(QUESTION_MARK); - objectName = questionMarkIndex >= 0 ? objectName.Substring(0, questionMarkIndex): objectName.Substring(0); - return provider; - } - } - return null; - } - */ public static bool TryGetProviderObjectName(ExternalProvider provider, string objectNameOrUrl, out string providerObjectName) { providerObjectName = null; From e0c76741b1090cac4da3c7459b385c6ff506b2b6 Mon Sep 17 00:00:00 2001 From: Gonzalo Date: Wed, 16 Jun 2021 10:33:02 -0300 Subject: [PATCH 16/33] Use TimeSpan instead of Minutes (cherry picked from commit 58c7cfa5c276e5410816e2636fe0e602298bfd02) --- .../Services/Storage/ExternalProviderBase.cs | 18 +++++++++++------- .../Storage/GXAmazonS3/ExternalProviderS3.cs | 13 ++++--------- .../AzureStorageExternalProvider.cs | 2 +- .../GXGoogleCloud/ExternalProviderGoogle.cs | 2 +- 4 files changed, 17 insertions(+), 18 deletions(-) diff --git a/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderBase.cs b/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderBase.cs index 5a9bc4b7c..e6f3bbbea 100644 --- a/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderBase.cs +++ b/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderBase.cs @@ -14,10 +14,10 @@ public abstract class ExternalProviderBase protected static String DEFAULT_EXPIRATION = "DEFAULT_EXPIRATION"; protected static String FOLDER = "FOLDER_NAME"; protected static String DEFAULT_ACL_DEPRECATED = "STORAGE_PROVIDER_DEFAULT_ACL"; - protected static String DEFAULT_EXPIRATION_DEPRECATED = "STORAGE_PROVIDER_DEFAULT_EXPIRATION"; - protected int defaultExpirationMinutes = 1440; - + protected static String DEFAULT_EXPIRATION_DEPRECATED = "STORAGE_PROVIDER_DEFAULT_EXPIRATION"; + protected TimeSpan defaultExpiration = new TimeSpan(24, 0, 0); + protected GxFileType defaultAcl = GxFileType.Private; public ExternalProviderBase() @@ -55,16 +55,20 @@ private void Initialize() this.defaultAcl = GxFileType.PublicRead; } - String expirationS = GetPropertyValue(DEFAULT_EXPIRATION, DEFAULT_EXPIRATION, defaultExpirationMinutes.ToString()); + String expirationS = GetPropertyValue(DEFAULT_EXPIRATION, DEFAULT_EXPIRATION, defaultExpiration.ToString()); if (!String.IsNullOrEmpty(expirationS)) { - Int32.TryParse(expirationS, out this.defaultExpirationMinutes); + int minutes; + if (Int32.TryParse(expirationS, out minutes) && minutes > 0) + { + defaultExpiration = new TimeSpan(0, minutes, 0); + } } } - protected int ResolveExpirationMinutes(int expirationMinutes) + protected TimeSpan ResolveExpiration(int expirationMinutes) { - return expirationMinutes > 0 ? expirationMinutes : defaultExpirationMinutes; + return expirationMinutes > 0 ? new TimeSpan(0, expirationMinutes, 0) : defaultExpiration; } protected String GetEncryptedPropertyValue(String propertyName, String alternativePropertyName = null) diff --git a/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs b/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs index afebd64ad..19e756900 100644 --- a/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs +++ b/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs @@ -39,11 +39,6 @@ public class ExternalProviderS3 : ExternalProviderBase, ExternalProvider [Obsolete("Use Property STORAGE_CUSTOM_ENDPOINT instead", false)] const string STORAGE_CUSTOM_ENDPOINT_DEPRECATED = "STORAGE_CUSTOM_ENDPOINT"; - [Obsolete("Use Property BUCKET instead", false)] - const string BUCKET_DEPRECATED = "BUCKET_NAME"; - [Obsolete("Use Property FOLDER instead", false)] - const string FOLDER_DEPRECATED = "FOLDER_NAME"; - string _storageUri; IAmazonS3 Client { get; set; } @@ -92,7 +87,7 @@ private void Initialize() { RegionEndpoint = region }; - Endpoint = GetPropertyValue( STORAGE_ENDPOINT , ENDPOINT_DEPRECATED); + Endpoint = GetPropertyValue(STORAGE_ENDPOINT, ENDPOINT_DEPRECATED); if (Endpoint == STORAGE_CUSTOM_ENDPOINT_VALUE) { Endpoint = GetPropertyValue(STORAGE_CUSTOM_ENDPOINT, STORAGE_CUSTOM_ENDPOINT_DEPRECATED); @@ -122,8 +117,8 @@ private void Initialize() { } #endif - Bucket = GetEncryptedPropertyValue(BUCKET, BUCKET_DEPRECATED); - Folder = GetPropertyValue(FOLDER, FOLDER_DEPRECATED); + Bucket = GetEncryptedPropertyValue(BUCKET); + Folder = GetPropertyValue(FOLDER); Region = region.SystemName; SetURI(); @@ -260,7 +255,7 @@ public string Get(string objectName, GxFileType fileType, int urlMinutes = 0) private string GetUrlImpl(string objectName, GxFileType fileType, int urlMinutes = 0) { bool isPrivate = IsPrivateUpload(fileType); - return (isPrivate)? GetPreSignedUrl(objectName, ResolveExpirationMinutes(urlMinutes)): StorageUri + StorageUtils.EncodeUrl(objectName); + return (isPrivate)? GetPreSignedUrl(objectName, ResolveExpiration(urlMinutes).Minutes): StorageUri + StorageUtils.EncodeUrl(objectName); } diff --git a/dotnet/src/dotnetframework/Providers/Storage/GXAzureStorage/AzureStorageExternalProvider.cs b/dotnet/src/dotnetframework/Providers/Storage/GXAzureStorage/AzureStorageExternalProvider.cs index 5c5351db8..f1beae904 100644 --- a/dotnet/src/dotnetframework/Providers/Storage/GXAzureStorage/AzureStorageExternalProvider.cs +++ b/dotnet/src/dotnetframework/Providers/Storage/GXAzureStorage/AzureStorageExternalProvider.cs @@ -125,7 +125,7 @@ private string GetURL(CloudBlockBlob blob, GxFileType fileType, int urlMinutes = { SharedAccessBlobPolicy sasConstraints = new SharedAccessBlobPolicy(); sasConstraints.SharedAccessStartTime = DateTime.UtcNow; - sasConstraints.SharedAccessExpiryTime = DateTime.UtcNow.AddMinutes(ResolveExpirationMinutes(urlMinutes)); + sasConstraints.SharedAccessExpiryTime = DateTime.UtcNow.AddMinutes(ResolveExpiration(urlMinutes).Minutes); sasConstraints.Permissions = SharedAccessBlobPermissions.Read; url += blob.GetSharedAccessSignature(sasConstraints); } diff --git a/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs b/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs index e1fcdbdb1..ca19cd134 100644 --- a/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs +++ b/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs @@ -248,7 +248,7 @@ public string Get(string objectName, GxFileType fileType, int urlMinutes) private string GetURL(string objectName, GxFileType fileType, int urlMinutes = 0) { if (fileType.HasFlag(GxFileType.Private)) - return Signer.Sign(Bucket, StorageUtils.EncodeUrl(objectName), TimeSpan.FromMinutes(ResolveExpirationMinutes(urlMinutes)), HttpMethod.Get); + return Signer.Sign(Bucket, StorageUtils.EncodeUrl(objectName), TimeSpan.FromMinutes(ResolveExpiration(urlMinutes).Minutes), HttpMethod.Get); else { return StorageUri + StorageUtils.EncodeUrl(objectName); From 3c25ca954d0721c5176bf5f7e30f22fb7f4eb7e7 Mon Sep 17 00:00:00 2001 From: Gonzalo Date: Wed, 16 Jun 2021 10:40:42 -0300 Subject: [PATCH 17/33] Use TimeSpan instead of Minutes (cherry picked from commit fa818dd2a540aece3a02ef14d27655d893cca763) --- .../Providers/Storage/GXAmazonS3/ExternalProviderS3.cs | 4 ++-- .../Storage/GXAzureStorage/AzureStorageExternalProvider.cs | 2 +- .../Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs b/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs index 19e756900..58deb23a9 100644 --- a/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs +++ b/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs @@ -255,11 +255,11 @@ public string Get(string objectName, GxFileType fileType, int urlMinutes = 0) private string GetUrlImpl(string objectName, GxFileType fileType, int urlMinutes = 0) { bool isPrivate = IsPrivateUpload(fileType); - return (isPrivate)? GetPreSignedUrl(objectName, ResolveExpiration(urlMinutes).Minutes): StorageUri + StorageUtils.EncodeUrl(objectName); + return (isPrivate)? GetPreSignedUrl(objectName, ResolveExpiration(urlMinutes).TotalMinutes): StorageUri + StorageUtils.EncodeUrl(objectName); } - private string GetPreSignedUrl(string objectName, int urlMinutes) + private string GetPreSignedUrl(string objectName, double urlMinutes) { GetPreSignedUrlRequest request = new GetPreSignedUrlRequest { diff --git a/dotnet/src/dotnetframework/Providers/Storage/GXAzureStorage/AzureStorageExternalProvider.cs b/dotnet/src/dotnetframework/Providers/Storage/GXAzureStorage/AzureStorageExternalProvider.cs index f1beae904..4f84fdbeb 100644 --- a/dotnet/src/dotnetframework/Providers/Storage/GXAzureStorage/AzureStorageExternalProvider.cs +++ b/dotnet/src/dotnetframework/Providers/Storage/GXAzureStorage/AzureStorageExternalProvider.cs @@ -125,7 +125,7 @@ private string GetURL(CloudBlockBlob blob, GxFileType fileType, int urlMinutes = { SharedAccessBlobPolicy sasConstraints = new SharedAccessBlobPolicy(); sasConstraints.SharedAccessStartTime = DateTime.UtcNow; - sasConstraints.SharedAccessExpiryTime = DateTime.UtcNow.AddMinutes(ResolveExpiration(urlMinutes).Minutes); + sasConstraints.SharedAccessExpiryTime = DateTime.UtcNow.AddMinutes(ResolveExpiration(urlMinutes).TotalMinutes); sasConstraints.Permissions = SharedAccessBlobPermissions.Read; url += blob.GetSharedAccessSignature(sasConstraints); } diff --git a/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs b/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs index ca19cd134..94561466d 100644 --- a/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs +++ b/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs @@ -248,7 +248,7 @@ public string Get(string objectName, GxFileType fileType, int urlMinutes) private string GetURL(string objectName, GxFileType fileType, int urlMinutes = 0) { if (fileType.HasFlag(GxFileType.Private)) - return Signer.Sign(Bucket, StorageUtils.EncodeUrl(objectName), TimeSpan.FromMinutes(ResolveExpiration(urlMinutes).Minutes), HttpMethod.Get); + return Signer.Sign(Bucket, StorageUtils.EncodeUrl(objectName), ResolveExpiration(urlMinutes), HttpMethod.Get); else { return StorageUri + StorageUtils.EncodeUrl(objectName); From 2707d4db8bc350451d5571f54ba09ace50b2256b Mon Sep 17 00:00:00 2001 From: Gonzalo Date: Wed, 16 Jun 2021 15:13:23 -0300 Subject: [PATCH 18/33] Do not require Folder Name (cherry picked from commit 9b07823a039072f8749758d65fdce08104a14665) --- .../GxClasses/Services/Storage/ExternalProviderBase.cs | 2 ++ .../Providers/Storage/GXAmazonS3/ExternalProviderS3.cs | 2 -- .../Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs | 2 -- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderBase.cs b/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderBase.cs index e6f3bbbea..7bafcdb04 100644 --- a/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderBase.cs +++ b/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderBase.cs @@ -19,6 +19,7 @@ public abstract class ExternalProviderBase protected GxFileType defaultAcl = GxFileType.Private; + protected string Folder { get; set; } public ExternalProviderBase() { @@ -64,6 +65,7 @@ private void Initialize() defaultExpiration = new TimeSpan(0, minutes, 0); } } + Folder = GetPropertyValue(FOLDER, null, string.Empty); } protected TimeSpan ResolveExpiration(int expirationMinutes) diff --git a/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs b/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs index 58deb23a9..1d781f428 100644 --- a/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs +++ b/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs @@ -43,7 +43,6 @@ public class ExternalProviderS3 : ExternalProviderBase, ExternalProvider IAmazonS3 Client { get; set; } string Bucket { get; set; } - string Folder { get; set; } string Endpoint { get; set; } string Region { get; set; } @@ -118,7 +117,6 @@ private void Initialize() { #endif Bucket = GetEncryptedPropertyValue(BUCKET); - Folder = GetPropertyValue(FOLDER); Region = region.SystemName; SetURI(); diff --git a/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs b/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs index 94561466d..a9a351f54 100644 --- a/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs +++ b/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs @@ -30,7 +30,6 @@ public class ExternalProviderGoogle : ExternalProviderBase, ExternalProvider StorageService Service { get; set; } String Project { get; set; } String Bucket { get; set; } - String Folder { get; set; } UrlSigner Signer { get; set; } @@ -80,7 +79,6 @@ private void Initialize() Bucket = GetEncryptedPropertyValue(BUCKET); Project = GetPropertyValue(PROJECT_ID); - Folder = GetPropertyValue(FOLDER); CreateBucket(); CreateFolder(Folder); From 1e91f04a8dc5faac36c736073547d7df0d24f926 Mon Sep 17 00:00:00 2001 From: Gonzalo Date: Wed, 16 Jun 2021 17:02:51 -0300 Subject: [PATCH 19/33] Default expiration fix (cherry picked from commit 69d95084e183b3917ac25d315e7788bb6f7e7a60) --- .../GxClasses/Services/Storage/ExternalProviderBase.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderBase.cs b/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderBase.cs index 7bafcdb04..78bd167ee 100644 --- a/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderBase.cs +++ b/dotnet/src/dotnetframework/GxClasses/Services/Storage/ExternalProviderBase.cs @@ -56,7 +56,7 @@ private void Initialize() this.defaultAcl = GxFileType.PublicRead; } - String expirationS = GetPropertyValue(DEFAULT_EXPIRATION, DEFAULT_EXPIRATION, defaultExpiration.ToString()); + String expirationS = GetPropertyValue(DEFAULT_EXPIRATION, DEFAULT_EXPIRATION, defaultExpiration.TotalMinutes.ToString()); if (!String.IsNullOrEmpty(expirationS)) { int minutes; From 303988576f6071d5baa40075226b91080b7ba05b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Echag=C3=BCe?= Date: Thu, 17 Jun 2021 09:11:11 -0300 Subject: [PATCH 20/33] Add an environment to the testing workflow (cherry picked from commit 0e630cffe2427a5271eab929f70bb9bfa559b7e2) --- .github/workflows/External-Storage-Tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/External-Storage-Tests.yml b/.github/workflows/External-Storage-Tests.yml index daab0e351..d8e2eb1e6 100644 --- a/.github/workflows/External-Storage-Tests.yml +++ b/.github/workflows/External-Storage-Tests.yml @@ -20,6 +20,7 @@ jobs: SolutionFile: dotnet\DotNetStandardClasses.sln runs-on: windows-latest + environment: external-storage-tests steps: - name: Checkout From 2b740603057c6a7ded62516178560077d0904baa Mon Sep 17 00:00:00 2001 From: Gonzalo Date: Thu, 17 Jun 2021 09:53:16 -0300 Subject: [PATCH 21/33] Do not require S3 Endpoint. (cherry picked from commit 6d175b9957c5e06e3aba675858773057cd9dd975) --- .../Storage/GXAmazonS3/ExternalProviderS3.cs | 5 ++++- .../ExternalProvider/ExternalProviderTest.cs | 15 ++++++++++----- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs b/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs index 1d781f428..cb0834192 100644 --- a/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs +++ b/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs @@ -26,6 +26,7 @@ public class ExternalProviderS3 : ExternalProviderBase, ExternalProvider const string REGION = "REGION"; const string STORAGE_CUSTOM_ENDPOINT_VALUE = "custom"; + const string DEFAULT_ENDPOINT = "s3.amazonaws.com"; const string DEFAULT_REGION = "us-east-1"; [Obsolete("Use Property ACCESS_KEY instead", false)] @@ -86,7 +87,7 @@ private void Initialize() { RegionEndpoint = region }; - Endpoint = GetPropertyValue(STORAGE_ENDPOINT, ENDPOINT_DEPRECATED); + Endpoint = GetPropertyValue(STORAGE_ENDPOINT, ENDPOINT_DEPRECATED, DEFAULT_ENDPOINT); if (Endpoint == STORAGE_CUSTOM_ENDPOINT_VALUE) { Endpoint = GetPropertyValue(STORAGE_CUSTOM_ENDPOINT, STORAGE_CUSTOM_ENDPOINT_DEPRECATED); @@ -350,7 +351,9 @@ public string Upload(string fileName, Stream stream, GxFileType destFileType) CannedACL = GetCannedACL(destFileType) }; if (Path.GetExtension(fileName).Equals(".tmp")) + { objectRequest.ContentType = "image/jpeg"; + } PutObjectResponse result = PutObject(objectRequest); return Get(fileName, destFileType); } diff --git a/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs b/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs index bbe764a12..00c102e2e 100644 --- a/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs +++ b/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs @@ -138,8 +138,9 @@ public void TestMultimediaUpload() String upload = provider.Get(TEST_SAMPLE_FILE_NAME, acl, 100); EnsureUrl(upload, acl); - provider.Delete(copyFileName, acl); - provider.Copy("text.txt", acl, copyFileName, acl); + DeleteSafe(copyFileName); + upload = provider.Copy(TEST_SAMPLE_FILE_NAME, copyFileName, "Table", "Field", acl); + provider.TryGetObjectNameFromURL(upload, out copyFileName); upload = TryGet(copyFileName, acl); EnsureUrl(upload, acl); } @@ -165,16 +166,20 @@ public void TestGetObjectName() [Fact] public void TestDownloadMethod() - { - String downloadPath = Path.Combine("resources", "test", TEST_SAMPLE_FILE_NAME); + { TestUploadPublicMethod(); + String downloadPath = Path.Combine("resources", "test", TEST_SAMPLE_FILE_NAME); try { File.Delete(downloadPath); } catch (Exception) { } - + try + { + Directory.CreateDirectory(Path.Combine("resources", "test")); + } + catch (Exception) { } provider.Download(TEST_SAMPLE_FILE_NAME, downloadPath, GxFileType.PublicRead); Assert.True(File.Exists(downloadPath)); } From f58d1574f41b4089ba7b781816391b91f0e1e87a Mon Sep 17 00:00:00 2001 From: Gonzalo Date: Thu, 17 Jun 2021 13:48:53 -0300 Subject: [PATCH 22/33] Skip tests if Env Variables not set (cherry picked from commit 69a825853fe992a1710ff2e860360f17a0a1eb0a) --- .../test/DotNetUnitTest/DotNetUnitTest.csproj | 2 +- .../ExternalProvider/ExternalProviderTest.cs | 77 +++++++++---------- 2 files changed, 39 insertions(+), 40 deletions(-) diff --git a/dotnet/test/DotNetUnitTest/DotNetUnitTest.csproj b/dotnet/test/DotNetUnitTest/DotNetUnitTest.csproj index 6ae0848b1..96b6ea3da 100644 --- a/dotnet/test/DotNetUnitTest/DotNetUnitTest.csproj +++ b/dotnet/test/DotNetUnitTest/DotNetUnitTest.csproj @@ -28,11 +28,11 @@ - all runtime; build; native; contentfiles; analyzers; buildtransitive + diff --git a/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs b/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs index 00c102e2e..3dff9861d 100644 --- a/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs +++ b/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs @@ -16,7 +16,7 @@ public abstract class ExternalProviderTest private GeneXus.Services.ExternalProvider provider; private static String TEST_SAMPLE_FILE_NAME = "text.txt"; private static String TEST_SAMPLE_FILE_PATH = Path.Combine("resources", TEST_SAMPLE_FILE_NAME).ToString(CultureInfo.InvariantCulture); - + public ExternalProviderTest(string providerName, Type externalProviderType) { @@ -24,58 +24,58 @@ public ExternalProviderTest(string providerName, Type externalProviderType) bool testEnabled = Environment.GetEnvironmentVariable(providerName + "_TEST_ENABLED") == "true"; - Assume.True(testEnabled); - + Skip.IfNot(testEnabled, "Environment variables not set"); provider = (GeneXus.Services.ExternalProvider)Activator.CreateInstance(externalProviderType); + Skip.IfNot(testEnabled); Assert.NotNull(provider); } - [Fact] + [SkippableFact] public void TestUploadPublicMethod() - { + { String upload = provider.Upload(TEST_SAMPLE_FILE_PATH, TEST_SAMPLE_FILE_NAME, GxFileType.PublicRead); EnsureUrl(upload, GxFileType.PublicRead); } - [Fact] + [SkippableFact] public void TestUploadDefaultMethod() { String upload = provider.Upload(TEST_SAMPLE_FILE_PATH, TEST_SAMPLE_FILE_NAME, GxFileType.Default); EnsureUrl(upload, GxFileType.Default); } - [Fact] + [SkippableFact] public void TestUploadAndCopyDefault() { TestUploadAndCopyByAcl(GxFileType.Default, GxFileType.Default); } - [Fact] + [SkippableFact] public void TestUploadAndCopyPrivate() { TestUploadAndCopyByAcl(GxFileType.Private, GxFileType.Private); } - [Fact] + [SkippableFact] public void TestUploadAndCopyPublic() { TestUploadAndCopyByAcl(GxFileType.PublicRead, GxFileType.PublicRead); } - [Fact] + [SkippableFact] public void TestUploadAndCopyMixed() { TestUploadAndCopyByAcl(GxFileType.Default, GxFileType.Private); } - [Fact] + [SkippableFact] public void TestUploadAndCopyPrivateToPublic() { TestUploadAndCopyByAcl(GxFileType.Private, GxFileType.PublicRead); } - [Fact] + [SkippableFact] public void TestUploadAndCopyPublicToPrivate() { TestUploadAndCopyByAcl(GxFileType.PublicRead, GxFileType.Private); @@ -97,38 +97,21 @@ public void TestUploadAndCopyByAcl(GxFileType aclUpload, GxFileType aclCopy) EnsureUrl(upload, aclCopy); } - [Fact] + [SkippableFact] public void TestCopyMethod() { String copyFileName = "copy-text.txt"; Copy(copyFileName, GxFileType.PublicRead); } - [Fact] + [SkippableFact] public void TestCopyPrivateMethod() { String copyFileName = "copy-text-private.txt"; Copy(copyFileName, GxFileType.Private); - } - - private void Copy(String copyFileName, GxFileType acl) - { - provider.Upload(TEST_SAMPLE_FILE_PATH, TEST_SAMPLE_FILE_NAME, acl); - String upload = provider.Get(TEST_SAMPLE_FILE_NAME, acl, 100); - EnsureUrl(upload, acl); - - DeleteSafe(copyFileName); - Wait(1000); //Google CDN replication seems to be delayed. - - String urlCopy = TryGet(copyFileName, GxFileType.PublicRead); - Assert.False(UrlExists(urlCopy), "URL cannot exist: " + urlCopy); - - provider.Copy("text.txt", acl, copyFileName, acl); - upload = provider.Get(copyFileName, acl, 100); - EnsureUrl(upload, acl); - } + } - [Fact] + [SkippableFact] public void TestMultimediaUpload() { String copyFileName = "copy-text-private.txt"; @@ -145,7 +128,7 @@ public void TestMultimediaUpload() EnsureUrl(upload, acl); } - [Fact] + [SkippableFact] public void TestGetMethod() { TestUploadPublicMethod(); @@ -153,7 +136,7 @@ public void TestGetMethod() EnsureUrl(url, GxFileType.PublicRead); } - [Fact] + [SkippableFact] public void TestGetObjectName() { TestUploadPublicMethod(); @@ -164,7 +147,7 @@ public void TestGetObjectName() Assert.Equal("text.txt", objectName); } - [Fact] + [SkippableFact] public void TestDownloadMethod() { TestUploadPublicMethod(); @@ -184,7 +167,7 @@ public void TestDownloadMethod() Assert.True(File.Exists(downloadPath)); } - [Fact] + [SkippableFact] public void TestDeleteFile() { GxFileType acl = GxFileType.PublicRead; @@ -197,7 +180,7 @@ public void TestDeleteFile() Assert.False(UrlExists(url)); } - [Fact] + [SkippableFact] public void TestDeleteFilePrivate() { GxFileType acl = GxFileType.Private; @@ -210,7 +193,7 @@ public void TestDeleteFilePrivate() Assert.False(UrlExists(url)); } - [Fact] + [SkippableFact] public void TestUploadPrivateMethod() { GxFileType acl = GxFileType.Private; @@ -224,6 +207,22 @@ public void TestUploadPrivateMethod() } + private void Copy(String copyFileName, GxFileType acl) + { + provider.Upload(TEST_SAMPLE_FILE_PATH, TEST_SAMPLE_FILE_NAME, acl); + String upload = provider.Get(TEST_SAMPLE_FILE_NAME, acl, 100); + EnsureUrl(upload, acl); + + DeleteSafe(copyFileName); + Wait(1000); //Google CDN replication seems to be delayed. + + String urlCopy = TryGet(copyFileName, GxFileType.PublicRead); + Assert.False(UrlExists(urlCopy), "URL cannot exist: " + urlCopy); + + provider.Copy("text.txt", acl, copyFileName, acl); + upload = provider.Get(copyFileName, acl, 100); + EnsureUrl(upload, acl); + } private String TryGet(String objectName, GxFileType acl) { From 8aa4221d0e51b9d96a8fe5e402555c2f9950386c Mon Sep 17 00:00:00 2001 From: Gonzalo Date: Fri, 18 Jun 2021 12:29:20 -0300 Subject: [PATCH 23/33] Workflow file --- .github/workflows/External-Storage-Tests.yml | 36 ++++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/.github/workflows/External-Storage-Tests.yml b/.github/workflows/External-Storage-Tests.yml index d8e2eb1e6..fdd2ecc3c 100644 --- a/.github/workflows/External-Storage-Tests.yml +++ b/.github/workflows/External-Storage-Tests.yml @@ -71,30 +71,30 @@ jobs: - name: Test External Storage run: | - $Env:AWSS3_TEST_ENABLED=true + $Env:AWSS3_TEST_ENABLED="true" $Env:STORAGE_AWSS3_ACCESS_KEY="${{ secrets.AWSS3_ACCESS_KEY }}" $Env:STORAGE_AWSS3_SECRET_KEY="${{ secrets.AWSS3_SECRET_KEY }}" - $Env:STORAGE_AWSS3_BUCKET_NAME=genexuss3test - $Env:STORAGE_AWSS3_FOLDER_NAME=gxclasses - $Env:STORAGE_AWSS3_REGION=us-east-1 - $Env:IBMCOS_TEST_ENABLED=true + $Env:STORAGE_AWSS3_BUCKET_NAME="genexus-s3-test" + $Env:STORAGE_AWSS3_FOLDER_NAME="gxclasses" + $Env:STORAGE_AWSS3_REGION="us-east-1" + $Env:IBMCOS_TEST_ENABLED="true" $Env:STORAGE_IBMCOS_ACCESS_KEY="${{ secrets.IBMCOS_ACCESS_KEY }}" $Env:STORAGE_IBMCOS_SECRET_KEY="${{ secrets.IBMCOS_SECRET_KEY }}" - $Env:STORAGE_IBMCOS_BUCKET_NAME=gxclasses-unit-tests - $Env:STORAGE_IBMCOS_FOLDER_NAME=tests - $Env:STORAGE_IBMCOS_REGION=us-south - $Env:AZUREBS_TEST_ENABLED=true + $Env:STORAGE_IBMCOS_BUCKET_NAME="gxclasses-unit-tests" + $Env:STORAGE_IBMCOS_FOLDER_NAME="tests" + $Env:STORAGE_IBMCOS_REGION="us-south" + $Env:AZUREBS_TEST_ENABLED="true" $Env:STORAGE_AZUREBS_ACCESS_KEY="${{ secrets.AZUREBS_ACCESS_KEY }}" $Env:STORAGE_AZUREBS_ACCOUNT_NAME="${{ secrets.AZUREBS_ACCOUNT_NAME }}" - $Env:STORAGE_AZUREBS_FOLDER_NAME=tests - $Env:STORAGE_AZUREBS_PUBLIC_CONTAINER_NAME=contluispublic - $Env:STORAGE_AZUREBS_PRIVATE_CONTAINER_NAME=contluisprivate - $Env:GOOGLECS_TEST_ENABLED=true - $Env:STORAGE_GOOGLECS_KEY="${{ secrets.GOOGLECS_KEY }}" - $Env:STORAGE_GOOGLECS_PROJECT_ID=gxjavacloudstorageunittests - $Env:STORAGE_GOOGLECS_BUCKET_NAME=javaclasses-unittests - $Env:STORAGE_GOOGLECS_FOLDER_NAME=gxclasses - $Env:STORAGE_GOOGLECS_APPLICATION_NAME=gxjavacloudstorageunittests + $Env:STORAGE_AZUREBS_FOLDER_NAME="tests" + $Env:STORAGE_AZUREBS_PUBLIC_CONTAINER_NAME="contluispublic" + $Env:STORAGE_AZUREBS_PRIVATE_CONTAINER_NAME="contluisprivate" + $Env:GOOGLECS_TEST_ENABLED="true" + $Env:STORAGE_GOOGLECS_KEY='${{ secrets.GOOGLECS_KEY }}' + $Env:STORAGE_GOOGLECS_PROJECT_ID="gxjavacloudstorageunittests" + $Env:STORAGE_GOOGLECS_BUCKET_NAME="javaclasses-unittests" + $Env:STORAGE_GOOGLECS_FOLDER_NAME="gxclasses" + $Env:STORAGE_GOOGLECS_APPLICATION_NAME="gxjavacloudstorageunittests" dotnet test $Env:SolutionFile --no-restore --no-build --configuration $Env:Configuration From 05f22b9d7bd3d1c238e15d39f66d998c8417c2bc Mon Sep 17 00:00:00 2001 From: Gonzalo Date: Fri, 18 Jun 2021 16:52:42 -0300 Subject: [PATCH 24/33] Fix error: The project exceeded the rate limit for bucket operations (cherry picked from commit 77b46c89a984113d8999d01e6ca91d0550dd8274) --- .../GXGoogleCloud/ExternalProviderGoogle.cs | 46 +++++++++++-------- 1 file changed, 27 insertions(+), 19 deletions(-) diff --git a/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs b/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs index a9a351f54..9aabf8b70 100644 --- a/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs +++ b/dotnet/src/dotnetframework/Providers/Storage/GXGoogleCloud/ExternalProviderGoogle.cs @@ -96,25 +96,33 @@ private Stream KeyStream(string key) private void CreateBucket() { - //objects in bucket are public by default - ObjectAccessControl defaultAccess = new ObjectAccessControl(); - defaultAccess.Entity = "allUsers"; - defaultAccess.Role = "READER"; - - Bucket bucket = new Bucket(); - bucket.Name = Bucket; - bucket.DefaultObjectAcl = new List { defaultAccess }; - - try - { - Client.CreateBucket(Project, bucket); - } - catch (GoogleApiException ex) - { - if (ex.Error.Code != BUCKET_EXISTS) - throw ex; - } - + try + { + //objects in bucket are public by default + ObjectAccessControl defaultAccess = new ObjectAccessControl(); + defaultAccess.Entity = "allUsers"; + defaultAccess.Role = "READER"; + + Bucket bucket = new Bucket(); + bucket.Name = Bucket; + bucket.DefaultObjectAcl = new List { defaultAccess }; + + if (Client.GetBucket(Bucket) == null) + { + try + { + Client.CreateBucket(Project, bucket); + } + catch (GoogleApiException ex) + { + if (ex.Error.Code != BUCKET_EXISTS) + throw ex; + } + } + } + catch (Exception) { + //We should not fail when Bucket cannot be created. Bucket should be created beforehand as Credentials may not allow Bucket creation. + } } private void CreateFolder(String name, string table = null, string field = null) From 7a835da41fb52653484ed5a79c5e872077a4e192 Mon Sep 17 00:00:00 2001 From: Gonzalo Date: Fri, 18 Jun 2021 17:22:25 -0300 Subject: [PATCH 25/33] Do not run Tests in parallel, as Google fails by rate limiter. (cherry picked from commit 65542b3d56b9a20091a2f9c3655e15a994e3af25) --- .../ExternalProvider/ExternalProviderTest.cs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs b/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs index 3dff9861d..1b2862950 100644 --- a/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs +++ b/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs @@ -3,14 +3,14 @@ using System.IO; using System.Net; using DotNetUnitTest; -using GeneXus.Storage.GXAmazonS3; -using GeneXus.Storage.GXAzureStorage; -using GeneXus.Storage.GXGoogleCloud; +using GeneXus.Storage; using Xunit; + #pragma warning disable CA1031 // Do not catch general exception types namespace UnitTesting { + [Collection("Sequential")] public abstract class ExternalProviderTest { private GeneXus.Services.ExternalProvider provider; @@ -123,8 +123,12 @@ public void TestMultimediaUpload() DeleteSafe(copyFileName); upload = provider.Copy(TEST_SAMPLE_FILE_NAME, copyFileName, "Table", "Field", acl); - provider.TryGetObjectNameFromURL(upload, out copyFileName); - upload = TryGet(copyFileName, acl); + + copyFileName = StorageFactory.GetProviderObjectAbsoluteUriSafe(provider, upload); + if (!copyFileName.StartsWith("http", StringComparison.OrdinalIgnoreCase)) + { + upload = TryGet(copyFileName, acl); + } EnsureUrl(upload, acl); } From 59c63e82a38cfa6ecf068421b12bc46d6659ede4 Mon Sep 17 00:00:00 2001 From: Gonzalo Date: Tue, 22 Jun 2021 17:06:15 -0300 Subject: [PATCH 26/33] Spacing issue --- .../Providers/Storage/GXAmazonS3/ExternalProviderS3.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs b/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs index cb0834192..ddc37b588 100644 --- a/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs +++ b/dotnet/src/dotnetframework/Providers/Storage/GXAmazonS3/ExternalProviderS3.cs @@ -534,4 +534,4 @@ public override string GetName() } } -} +} \ No newline at end of file From 38bcbd79f9945eafdecbe317c1ce0f0c72eb4725 Mon Sep 17 00:00:00 2001 From: Gonzalo Date: Tue, 22 Jun 2021 17:13:46 -0300 Subject: [PATCH 27/33] Missing cherry pick --- .../GxClasses/Core/GXUtilsCommon.cs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/dotnet/src/dotnetframework/GxClasses/Core/GXUtilsCommon.cs b/dotnet/src/dotnetframework/GxClasses/Core/GXUtilsCommon.cs index 2ec2b3ecd..8671f055f 100644 --- a/dotnet/src/dotnetframework/GxClasses/Core/GXUtilsCommon.cs +++ b/dotnet/src/dotnetframework/GxClasses/Core/GXUtilsCommon.cs @@ -3408,6 +3408,12 @@ public static string GetFileType(string FileName) { if (FileName.Trim().Length == 0) return string.Empty; + + if (GxUploadHelper.IsUpload(FileName)) + { + return new GxFile(string.Empty, FileName, GxFileType.Private).GetExtension(); + } + string extension = string.Empty; try { @@ -3447,6 +3453,11 @@ public static string GetFileName(string FileName) { if (FileName.Trim().Length == 0) return ""; + + if (GxUploadHelper.IsUpload(FileName)) + { + FileName = new GxFile(string.Empty, FileName, GxFileType.Private).GetName(); + } try { return Path.GetFileNameWithoutExtension(FileName);//FileNames with URI or local (exist) @@ -3665,6 +3676,10 @@ public static string GetValidFileName(string path, string replaceStr) if (!IsValidFilePath(path)) validPath = GetValidPath(path, "_"); + Uri result; + if (Uri.TryCreate(validPath, UriKind.Absolute, out result)) + validPath = result.LocalPath; + string fileName = Path.GetFileName(validPath); if (!IsValidFileName(fileName)) { @@ -5310,6 +5325,10 @@ public static string GetUriFromFile(string name, string type, string path) { if (String.IsNullOrEmpty(name) && String.IsNullOrEmpty(type) && !String.IsNullOrEmpty(path)) { + if (GxUploadHelper.IsUpload(path)) + { + return new GxFile(string.Empty, path, GxFileType.Private).GetName(); + } string fromPathType = Path.GetExtension(path); if (!String.IsNullOrEmpty(fromPathType) && fromPathType != "tmp") { From 8e5b8ebd82cdc31819731e91d50bacb078806409 Mon Sep 17 00:00:00 2001 From: Gonzalo Date: Tue, 22 Jun 2021 17:16:15 -0300 Subject: [PATCH 28/33] Add WF to BETA --- .github/workflows/External-Storage-Tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/External-Storage-Tests.yml b/.github/workflows/External-Storage-Tests.yml index fdd2ecc3c..7260d469d 100644 --- a/.github/workflows/External-Storage-Tests.yml +++ b/.github/workflows/External-Storage-Tests.yml @@ -4,6 +4,7 @@ on: pull_request: branches: - 'master' + - 'beta' - 'release-*' push: branches: From 6c33dc5a9e0018a44206972ba1f9ed6252d5ae8a Mon Sep 17 00:00:00 2001 From: Gonzalo Date: Tue, 22 Jun 2021 17:28:19 -0300 Subject: [PATCH 29/33] Add some random to Test because of Google Rate Limit exception --- .../DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs b/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs index 1b2862950..23fd8f9ec 100644 --- a/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs +++ b/dotnet/test/DotNetUnitTest/ExternalProvider/ExternalProviderTest.cs @@ -83,7 +83,7 @@ public void TestUploadAndCopyPublicToPrivate() public void TestUploadAndCopyByAcl(GxFileType aclUpload, GxFileType aclCopy) { - String copyFileName = "test-upload-and-copy.txt"; + String copyFileName = $"test-upload-and-copy_{new Random().Next()}.txt"; DeleteSafe(TEST_SAMPLE_FILE_PATH); DeleteSafe(copyFileName); String upload = provider.Upload(TEST_SAMPLE_FILE_PATH, TEST_SAMPLE_FILE_NAME, aclUpload); From 0b5b6a2e4eeb6ec87d975eaa0c53ffb5bdf553d6 Mon Sep 17 00:00:00 2001 From: Gonzalo Date: Tue, 22 Jun 2021 17:41:45 -0300 Subject: [PATCH 30/33] Fix cherry-pick conflicts --- .../GxClasses/Core/GXUtilsCommon.cs | 71 +++++++++---------- .../test/DotNetUnitTest/DotNetUnitTest.csproj | 13 ++-- 2 files changed, 41 insertions(+), 43 deletions(-) diff --git a/dotnet/src/dotnetframework/GxClasses/Core/GXUtilsCommon.cs b/dotnet/src/dotnetframework/GxClasses/Core/GXUtilsCommon.cs index 8671f055f..a12c52973 100644 --- a/dotnet/src/dotnetframework/GxClasses/Core/GXUtilsCommon.cs +++ b/dotnet/src/dotnetframework/GxClasses/Core/GXUtilsCommon.cs @@ -2632,19 +2632,19 @@ static public string CMonth(DateTime dt, string lang) public string Time() { return DateTime.Now.ToString(TimeFormatFromSize(8, -1, ":"), cultureInfo); - } + } - static public DateTime AddMth(DateTime dt, int cantMonths) - { - if (dt == nullDate) - return nullDate; - return dt.AddMonths(cantMonths); - } - static public DateTime AddYr(DateTime dt, int cantYears) - { - if (dt == nullDate) - return nullDate; - return dt.AddYears(cantYears); + static public DateTime AddMth(DateTime dt, int cantMonths) + { + if (dt == nullDate && cantMonths < 0) + return nullDate; + return dt.AddMonths(cantMonths); + } + static public DateTime AddYr(DateTime dt, int cantYears) + { + if (dt == nullDate && cantYears < 0) + return nullDate; + return dt.AddYears(cantYears); } static public DateTime DateEndOfMonth(DateTime dt) { @@ -2693,30 +2693,29 @@ static public long TDiff(DateTime dtMinu, DateTime dtSust) } static public int DDiff(DateTime dtMinu, DateTime dtSust) { - return Convert.ToInt32((dtMinu - dtSust).TotalDays); - } - static public DateTime TAdd(DateTime dt, int seconds) - { - if (dt == nullDate) - return nullDate; - return dt.AddSeconds(seconds); - } - static public DateTime TAddMs(DateTime dt, double seconds) - { - - if (dt == nullDate) - return nullDate; - if (seconds % 1 == 0) - return dt.AddSeconds((int)seconds); - else - return dt.AddMilliseconds(seconds * 1000); - } - static public DateTime DAdd(DateTime dt, int days) - { - if (dt == nullDate) - return nullDate; - return dt.AddDays(days); - } + return Convert.ToInt32((dtMinu - dtSust).TotalDays); + } + static public DateTime TAdd(DateTime dt, int seconds) + { + if (dt == nullDate && seconds < 0) + return nullDate; + return dt.AddSeconds(seconds); + } + static public DateTime TAddMs(DateTime dt, double seconds) + { + if (dt == nullDate && seconds < 0) + return nullDate; + if (seconds % 1 == 0) + return dt.AddSeconds((int)seconds); + else + return dt.AddMilliseconds(seconds * 1000); + } + static public DateTime DAdd(DateTime dt, int days) + { + if (dt == nullDate && days < 0) + return nullDate; + return dt.AddDays(days); + } public DateTime CToT(string strDate, int picFmt, int ampmFmt) { if (isNullDateTime(strDate, picFmt, ampmFmt)) diff --git a/dotnet/test/DotNetUnitTest/DotNetUnitTest.csproj b/dotnet/test/DotNetUnitTest/DotNetUnitTest.csproj index 96b6ea3da..2d2120052 100644 --- a/dotnet/test/DotNetUnitTest/DotNetUnitTest.csproj +++ b/dotnet/test/DotNetUnitTest/DotNetUnitTest.csproj @@ -5,9 +5,7 @@ DotNetUnitTest - - @@ -36,6 +34,7 @@ + @@ -49,12 +48,12 @@ - - Always - - - Always + + PreserveNewest + + + From c2c75b9c3cdf4f5f915dffef3d8a428b66177fac Mon Sep 17 00:00:00 2001 From: Gonzalo Date: Tue, 22 Jun 2021 17:49:41 -0300 Subject: [PATCH 31/33] Add test.txt resource --- dotnet/test/DotNetUnitTest/DotNetUnitTest.csproj | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dotnet/test/DotNetUnitTest/DotNetUnitTest.csproj b/dotnet/test/DotNetUnitTest/DotNetUnitTest.csproj index 2d2120052..ff1adfe46 100644 --- a/dotnet/test/DotNetUnitTest/DotNetUnitTest.csproj +++ b/dotnet/test/DotNetUnitTest/DotNetUnitTest.csproj @@ -51,6 +51,9 @@ PreserveNewest + + Always + From 2b71d8391045b0c4773ee10d5ba4fc81d113d662 Mon Sep 17 00:00:00 2001 From: Gonzalo Date: Tue, 22 Jun 2021 18:36:55 -0300 Subject: [PATCH 32/33] Bump .NET FW Version for workflow --- .github/workflows/External-Storage-Tests.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/External-Storage-Tests.yml b/.github/workflows/External-Storage-Tests.yml index 7260d469d..bee648cc2 100644 --- a/.github/workflows/External-Storage-Tests.yml +++ b/.github/workflows/External-Storage-Tests.yml @@ -34,10 +34,16 @@ jobs: with: dotnet-version: '3.1.x' + - name: Install .NET Core 5.0 + uses: actions/setup-dotnet@v1 + with: + dotnet-version: '5.0.x' + - name: Install .NET 6 - uses: actions/setup-dotnet@v1.7.2 + uses: actions/setup-dotnet@v1 with: - dotnet-version: '6.0.100-preview.3.21202.5' + dotnet-version: '6.0.x' + include-prerelease: True - uses: actions/setup-dotnet@v1 with: From 3f9f755d0256a5c9ed2bb877a0ad0a14b1c65ba3 Mon Sep 17 00:00:00 2001 From: Gonzalo Date: Tue, 22 Jun 2021 18:46:01 -0300 Subject: [PATCH 33/33] Remove Workflow from beta branches --- .github/workflows/External-Storage-Tests.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/External-Storage-Tests.yml b/.github/workflows/External-Storage-Tests.yml index bee648cc2..0e35129c5 100644 --- a/.github/workflows/External-Storage-Tests.yml +++ b/.github/workflows/External-Storage-Tests.yml @@ -3,8 +3,7 @@ name: External Storage Tests on: pull_request: branches: - - 'master' - - 'beta' + - 'master' - 'release-*' push: branches: