[Docker] Update docker ignore - clouds ymls (#5647)

.dockerignore

@@ -7,16 +7,28 @@
 venv
 __pycache__
 .idea
-.DS_Store
+**/.DS_Store
 .git-blame-ignore-revs
 .gitignore
 .hypothesis
 .ruff_cache
 kubeconfig
 **/env.yml
-**/patch_env.yml
+**/patch[_\-]env.yml
 **/mlrun.env
 mlrun.egg-info
 
+**/*.coverage
+**/.aws-secrets
+**/credentials.txt
+**/credentials.json
+**/test-alibaba-oss.yml
+**/test-azure-blob.yml
+**/test-aws-s3.yml
+**/test-google-cloud-storage.yml
+**/test-dbfs-store.yml
+
+**/google-big-query-credentials.json
+
 # this is just a template
 !dockerfiles/jupyter/mlrun.env

.gitignore

@@ -38,7 +38,7 @@ project.yaml
 *.bak
 docs/contributing.md
 server/api/proto/*pb2*.py
-automation/patch_igz/patch_env.yml
+automation/patch_igz/patch[_-]env.yml
 kubeconfig
 
 # used for local development.
@@ -51,3 +51,7 @@ mlrun_bc_*.json
 
 # used for local development.
 playground
+credentials.txt
+credentials.json
+google-big-query-credentials.json
+*.coverage

tests/datastore/databricks_utils.py

@@ -22,6 +22,8 @@
 
 
 def is_databricks_configured(config_file_path=None):
+    if not os.path.exists(config_file_path):
+        return False
     if not config_file_path:
         return all(os.environ.get(var) for var in MUST_HAVE_VARIABLES)
     with open(config_file_path) as yaml_file:

tests/integration/alibaba_oss/test_alibaba_oss.py

@@ -29,8 +29,11 @@
 
 here = Path(__file__).absolute().parent
 config_file_path = here / "test-alibaba-oss.yml"
-with config_file_path.open() as fp:
-    config = yaml.safe_load(fp)
+config = {}
+if os.path.exists(config_file_path):
+    with config_file_path.open() as fp:
+        config = yaml.safe_load(fp)
+
 
 test_filename = here / "test.txt"
 with open(test_filename) as f:
@@ -47,8 +50,10 @@
 
 
 def alibaba_oss_configured(extra_params=None):
+    if not os.path.exists(config_file_path):
+        return False
     extra_params = extra_params or []
-    env_params = config["env"]
+    env_params = config.get("env", {})
     needed_params = ["bucket_name", *credential_params, *extra_params]
     for param in needed_params:
         if not env_params.get(param):

tests/integration/aws_s3/test_aws_s3.py

@@ -35,8 +35,11 @@
 
 here = os.path.dirname(__file__)
 config_file_path = os.path.join(here, "test-aws-s3.yml")
-with open(config_file_path) as yaml_file:
-    config = yaml.safe_load(yaml_file)
+
+config = {}
+if os.path.exists(config_file_path):
+    with open(config_file_path) as yaml_file:
+        config = yaml.safe_load(yaml_file)
 
 # Used to test dataframe functionality (will be saved as csv)
 test_df_string = "col1,col2,col3\n1,2,3"
@@ -45,8 +48,10 @@
 
 
 def aws_s3_configured(extra_params=None):
+    if not os.path.exists(config_file_path):
+        return False
     extra_params = extra_params or []
-    env_params = config["env"]
+    env_params = config.get("env", {})
     needed_params = ["bucket_name", *credential_params, *extra_params]
     for param in needed_params:
         if not env_params.get(param):
@@ -58,7 +63,7 @@ def aws_s3_configured(extra_params=None):
 @pytest.mark.parametrize("use_datastore_profile", [False, True])
 class TestAwsS3:
     assets_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), "assets")
-    env = config["env"]
+    env = config.get("env", {})
     bucket_name = env.get("bucket_name")
     access_key_id = env.get("AWS_ACCESS_KEY_ID")
     _secret_access_key = env.get("AWS_SECRET_ACCESS_KEY")

tests/integration/azure_blob/test_azure_blob.py

@@ -39,8 +39,10 @@
 csv_dir = "csv"
 
 config_file_path = here / "test-azure-blob.yml"
-with config_file_path.open() as fp:
-    config = yaml.safe_load(fp)
+config = {}
+if os.path.exists(str(config_file_path)):
+    with config_file_path.open() as fp:
+        config = yaml.safe_load(fp)
 
 AUTH_METHODS_AND_REQUIRED_PARAMS = {
     "env_conn_str": ["AZURE_STORAGE_CONNECTION_STRING"],
@@ -71,15 +73,19 @@
     "auth_method ,use_datastore_profile", generated_pytest_parameters
 )
 @pytest.mark.skipif(
-    not config["env"].get("AZURE_CONTAINER"),
+    not config.get("env", {}).get("AZURE_CONTAINER"),
     reason="AZURE_CONTAINER is not set",
 )
+@pytest.mark.skipif(
+    not os.path.exists(str(config_file_path)),
+    reason="azure credentials file is not exists",
+)
 class TestAzureBlob:
     assets_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), "assets")
     profile_name = "azure_blob_ds_profile"
     test_dir = "test_mlrun_azure_blob"
     run_dir = f"{test_dir}/run_{uuid.uuid4()}"
-    bucket_name = config["env"].get("AZURE_CONTAINER", None)
+    bucket_name = config.get("env", {}).get("AZURE_CONTAINER", None)
     test_file = os.path.join(assets_path, "test.txt")
 
     @classmethod

tests/integration/azure_blob/test_dask_dataitem_to_azure_blob.py

@@ -27,8 +27,9 @@
 
 here = Path(__file__).absolute().parent
 config_file_path = here / "test-azure-blob.yml"
-with config_file_path.open() as fp:
-    config = yaml.safe_load(fp)
+if os.path.exists(str(config_file_path)):
+    with config_file_path.open() as fp:
+        config = yaml.safe_load(fp)
 
 
 AUTH_METHODS_AND_REQUIRED_PARAMS = {
@@ -37,6 +38,8 @@
 
 
 def verify_auth_parameters_and_configure_env(auth_method):
+    if not os.path.exists(str(config_file_path)):
+        return False
     if not config["env"].get("AZURE_CONTAINER"):
         return False
 

tests/integration/google_cloud_storage/test_google_cloud_storage.py

@@ -35,16 +35,18 @@
 from mlrun.utils import logger
 
 here = os.path.dirname(__file__)
+config = {}
 config_file_path = os.path.join(here, "test-google-cloud-storage.yml")
-with open(config_file_path) as yaml_file:
-    config = yaml.safe_load(yaml_file)
+if os.path.exists(config_file_path):
+    with open(config_file_path) as yaml_file:
+        config = yaml.safe_load(yaml_file)
 
 
 credential_params = ["credentials_json_file"]
 
 
 def google_cloud_storage_configured():
-    env_params = config["env"]
+    env_params = config.get("env", {})
     needed_params = ["bucket_name", *credential_params]
     for param in needed_params:
         if not env_params.get(param):
@@ -56,14 +58,18 @@ def google_cloud_storage_configured():
     not google_cloud_storage_configured(),
     reason="Google cloud storage parameters not configured",
 )
+@pytest.mark.skipif(
+    not os.path.exists(config_file_path),
+    reason="Google cloud storage credentials file is not exists",
+)
 @pytest.mark.parametrize("use_datastore_profile", [False, True])
 class TestGoogleCloudStorage:
     assets_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), "assets")
-    bucket_name = config["env"].get("bucket_name")
+    bucket_name = config.get("env", {}).get("bucket_name")
     test_dir = "test_mlrun_gcs_objects"
     run_dir = f"{test_dir}/run_{uuid.uuid4()}"
     profile_name = "gcs_profile"
-    credentials_path = config["env"].get("credentials_json_file")
+    credentials_path = config.get("env", {}).get("credentials_json_file")
     test_file = os.path.join(assets_path, "test.txt")
 
     @classmethod