diff --git a/samples/features/sql-big-data-cluster/spark/data-virtualization/mssql_spark_connector_ad_pyspark.ipynb b/samples/features/sql-big-data-cluster/spark/data-virtualization/mssql_spark_connector_ad_pyspark.ipynb index 2f2ea8ffc5..b1d9817b97 100644 --- a/samples/features/sql-big-data-cluster/spark/data-virtualization/mssql_spark_connector_ad_pyspark.ipynb +++ b/samples/features/sql-big-data-cluster/spark/data-virtualization/mssql_spark_connector_ad_pyspark.ipynb @@ -29,7 +29,7 @@ "- SQL Server 2019 big data cluster is deployed with AD\r\n", "- Have access to AD controller to create keytabs that we will use in this sample. \r\n", "- Download [AdultCensusIncome.csv]( https://amldockerdatasets.azureedge.net/AdultCensusIncome.csv ) to your local machine. Upload this file to hdfs folder named *spark_data*. \r\n", - "- The sample uses a SQL database *spark_sql_db* to create/update tables. The database needs to be created before the sample is run.\r\n", + "- The sample uses a SQL database *spark_mssql_db* to create/update tables. Refer **data-virtualization/mssql_spark_connector_user_creation.ipynb** on steps to create this database anduser.\r\n", " " ], "metadata": { @@ -80,7 +80,7 @@ "kinit -kt testusera1.keytab testusera1@AZDATA.LOCAL\r\n", "```\r\n", "\r\n", - "Load Keytab to HDFS for use\r\n", + "Load Keytab to HDFS for use, you can do it from ADS, or use hadoop commandline inside a BDC container:\r\n", "\r\n", "```sh\r\n", "hadoop fs -mkdir -p /user/testusera1/\r\n", @@ -95,61 +95,6 @@ "azdata_cell_guid": "5e733ddd-cbf8-4aa1-9f97-9fff40c28332" } }, - { - "cell_type": "markdown", - "source": [ - "# Create a database user\r\n", - "\r\n", - "``` sql\r\n", - "IF NOT EXISTS (select name from sys.server_principals where name='AZDATA.LOCAL\\testusera1')\r\n", - "BEGIN\r\n", - " CREATE LOGIN [AZDATA.LOCAL\\testusera1] FROM WINDOWS\r\n", - "END\r\n", - "\r\n", - "ALTER SERVER ROLE dbcreator ADD MEMBER [AZDATA.LOCAL\\testusera1]\r\n", - "GRANT VIEW SERVER STATE to [AZDATA.LOCAL\\testusera1]\r\n", - "\r\n", - "# Create a database named \"spark_mssql_db\"\r\n", - "IF NOT EXISTS (SELECT * FROM sys.databases WHERE name = N'spark_mssql_db')\r\n", - " CREATE DATABASE spark_mssql_db\r\n", - "```" - ], - "metadata": { - "azdata_cell_guid": "453e7b2f-e590-4b95-9fbd-5dc8b9d1f02c" - } - }, - { - "cell_type": "markdown", - "source": [ - "# Create Data Pool user\r\n", - "\r\n", - "```\r\n", - "-- To create external tables in data pools\r\n", - "grant alter any external data source to [aris\\testuser];\r\n", - "\r\n", - "-- To create external table\r\n", - "grant create table to [aris\\testuser];\r\n", - "grant alter any schema to [aris\\testuser];\r\n", - "\r\n", - "ALTER ROLE [db_datareader] ADD MEMBER [aris\\testuser]\r\n", - "ALTER ROLE [db_datawriter] ADD MEMBER [aris\\testuser]\r\n", - "```\r\n", - "\r\n", - "```\r\n", - "CREATE EXTERNAL DATA SOURCE connector_ds WITH (LOCATION = 'sqldatapool://controller-svc/default');\r\n", - "EXECUTE('USE spark_mssql_db; CREATE EXTERNAL TABLE [dummy3] ([number] int, [word] nvarchar(2048)) WITH (DATA_SOURCE = connector_ds, DISTRIBUTION = ROUND_ROBIN)')\r\n", - "\r\n", - "-- Create a login in data pools and Provide right permissions to this user\r\n", - "EXECUTE( ' Use spark_mssql_db; CREATE LOGIN [aris\\testusera1] FROM WINDOWS ' ) AT DATA_SOURCE connector_ds;\r\n", - "\r\n", - "EXECUTE( ' Use spark_mssql_db; CREATE USER [aris\\testusera1] ; ALTER ROLE [db_datareader] ADD MEMBER [aris\\testusera1]; ALTER ROLE [db_datawriter] ADD MEMBER [aris\\testusera1] ;') AT DATA_SOURCE connector_ds;\r\n", - "\r\n", - "```" - ], - "metadata": { - "azdata_cell_guid": "15b7588a-d1ad-4e17-84c6-bf4a862d1905" - } - }, { "cell_type": "markdown", "source": [ @@ -184,53 +129,23 @@ }, "outputs": [ { - "output_type": "stream", - "name": "stdout", - "text": "Starting Spark application\n" - }, - { - "output_type": "display_data", - "data": { - "text/plain": "", - "text/html": "\n
IDYARN Application IDKindStateSpark UIDriver logCurrent session?
12application_1581458669418_0041pysparkidleLinkLink✔
" - }, - "metadata": {} - }, - { - "output_type": "display_data", - "data": { - "text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…", - "application/vnd.jupyter.widget-view+json": { - "version_major": 2, - "version_minor": 0, - "model_id": "ae4ec82192fb473e8b77e4065f4c1927" - } - }, - "metadata": {} - }, - { - "output_type": "stream", - "name": "stdout", - "text": "SparkSession available as 'spark'.\n" - }, - { - "output_type": "display_data", "data": { "text/plain": "", - "text/html": "Current session configs: {'conf': {'spark.files': '/user/user1/user1.keytab', 'spark.executor.memory': '4g', 'spark.driver.memory': '4g', 'spark.executor.cores': 2, 'spark.driver.cores': 1, 'spark.executor.instances': 4}, 'kind': 'pyspark'}
" + "text/html": "Current session configs: {'conf': {'spark.files': '/user/testusera1/testusera1.keytab', 'spark.executor.memory': '4g', 'spark.driver.memory': '4g', 'spark.executor.cores': 2, 'spark.driver.cores': 1, 'spark.executor.instances': 4}, 'kind': 'pyspark'}
" }, - "metadata": {} + "metadata": {}, + "output_type": "display_data" }, { - "output_type": "display_data", "data": { "text/plain": "", - "text/html": "\n
IDYARN Application IDKindStateSpark UIDriver logCurrent session?
6application_1581458669418_0035pysparkidleLinkLink✔
" + "text/html": "No active sessions." }, - "metadata": {} + "metadata": {}, + "output_type": "display_data" } ], - "execution_count": 8 + "execution_count": 2 }, { "cell_type": "markdown", @@ -268,54 +183,54 @@ }, "outputs": [ { - "output_type": "stream", "name": "stdout", - "text": "Starting Spark application\n" + "text": "Starting Spark application\n", + "output_type": "stream" }, { - "output_type": "display_data", "data": { "text/plain": "", - "text/html": "\n
IDYARN Application IDKindStateSpark UIDriver logCurrent session?
7application_1581458669418_0036pysparkidleLinkLink✔
" + "text/html": "\n
IDYARN Application IDKindStateSpark UIDriver logCurrent session?
4application_1597953190209_0006pysparkidleLinkLink✔
" }, - "metadata": {} + "metadata": {}, + "output_type": "display_data" }, { - "output_type": "display_data", "data": { "text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…", "application/vnd.jupyter.widget-view+json": { "version_major": 2, "version_minor": 0, - "model_id": "52afbfffaf3545e28a2d09864d1c4ced" + "model_id": "0f33eba1feef441fa9b77e82bdf2c5ee" } }, - "metadata": {} + "metadata": {}, + "output_type": "display_data" }, { - "output_type": "stream", "name": "stdout", - "text": "SparkSession available as 'spark'.\n" + "text": "SparkSession available as 'spark'.\n", + "output_type": "stream" }, { - "output_type": "display_data", "data": { "text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…", "application/vnd.jupyter.widget-view+json": { "version_major": 2, "version_minor": 0, - "model_id": "7e03eb6a81cd41e4878f12df5ee31dc7" + "model_id": "e97636b10a734032a1cdae8c2a16e08a" } }, - "metadata": {} + "metadata": {}, + "output_type": "display_data" }, { - "output_type": "stream", "name": "stdout", - "text": "+---+----------------+------+---------+-------------+------------------+-----------------+-------------+-----+------+------------+------------+--------------+--------------+------+\n|age| workclass|fnlwgt|education|education-num| marital-status| occupation| relationship| race| sex|capital-gain|capital-loss|hours-per-week|native-country|income|\n+---+----------------+------+---------+-------------+------------------+-----------------+-------------+-----+------+------------+------------+--------------+--------------+------+\n| 39| State-gov| 77516|Bachelors| 13| Never-married| Adm-clerical|Not-in-family|White| Male| 2174| 0| 40| United-States| <=50K|\n| 50|Self-emp-not-inc| 83311|Bachelors| 13|Married-civ-spouse| Exec-managerial| Husband|White| Male| 0| 0| 13| United-States| <=50K|\n| 38| Private|215646| HS-grad| 9| Divorced|Handlers-cleaners|Not-in-family|White| Male| 0| 0| 40| United-States| <=50K|\n| 53| Private|234721| 11th| 7|Married-civ-spouse|Handlers-cleaners| Husband|Black| Male| 0| 0| 40| United-States| <=50K|\n| 28| Private|338409|Bachelors| 13|Married-civ-spouse| Prof-specialty| Wife|Black|Female| 0| 0| 40| Cuba| <=50K|\n+---+----------------+------+---------+-------------+------------------+-----------------+-------------+-----+------+------------+------------+--------------+--------------+------+\nonly showing top 5 rows\n\n+---+----------------+------+---------+-------------+------------------+-----------------+-------------+-----+------+------------+------------+--------------+--------------+------+\n|age| workclass|fnlwgt|education|education_num| marital_status| occupation| relationship| race| sex|capital_gain|capital_loss|hours_per_week|native_country|income|\n+---+----------------+------+---------+-------------+------------------+-----------------+-------------+-----+------+------------+------------+--------------+--------------+------+\n| 39| State-gov| 77516|Bachelors| 13| Never-married| Adm-clerical|Not-in-family|White| Male| 2174| 0| 40| United-States| <=50K|\n| 50|Self-emp-not-inc| 83311|Bachelors| 13|Married-civ-spouse| Exec-managerial| Husband|White| Male| 0| 0| 13| United-States| <=50K|\n| 38| Private|215646| HS-grad| 9| Divorced|Handlers-cleaners|Not-in-family|White| Male| 0| 0| 40| United-States| <=50K|\n| 53| Private|234721| 11th| 7|Married-civ-spouse|Handlers-cleaners| Husband|Black| Male| 0| 0| 40| United-States| <=50K|\n| 28| Private|338409|Bachelors| 13|Married-civ-spouse| Prof-specialty| Wife|Black|Female| 0| 0| 40| Cuba| <=50K|\n+---+----------------+------+---------+-------------+------------------+-----------------+-------------+-----+------+------------+------------+--------------+--------------+------+\nonly showing top 5 rows" + "text": "+---+----------------+------+---------+-------------+------------------+-----------------+-------------+-----+------+------------+------------+--------------+--------------+------+\n|age| workclass|fnlwgt|education|education-num| marital-status| occupation| relationship| race| sex|capital-gain|capital-loss|hours-per-week|native-country|income|\n+---+----------------+------+---------+-------------+------------------+-----------------+-------------+-----+------+------------+------------+--------------+--------------+------+\n| 39| State-gov| 77516|Bachelors| 13| Never-married| Adm-clerical|Not-in-family|White| Male| 2174| 0| 40| United-States| <=50K|\n| 50|Self-emp-not-inc| 83311|Bachelors| 13|Married-civ-spouse| Exec-managerial| Husband|White| Male| 0| 0| 13| United-States| <=50K|\n| 38| Private|215646| HS-grad| 9| Divorced|Handlers-cleaners|Not-in-family|White| Male| 0| 0| 40| United-States| <=50K|\n| 53| Private|234721| 11th| 7|Married-civ-spouse|Handlers-cleaners| Husband|Black| Male| 0| 0| 40| United-States| <=50K|\n| 28| Private|338409|Bachelors| 13|Married-civ-spouse| Prof-specialty| Wife|Black|Female| 0| 0| 40| Cuba| <=50K|\n+---+----------------+------+---------+-------------+------------------+-----------------+-------------+-----+------+------------+------------+--------------+--------------+------+\nonly showing top 5 rows\n\n+---+----------------+------+---------+-------------+------------------+-----------------+-------------+-----+------+------------+------------+--------------+--------------+------+\n|age| workclass|fnlwgt|education|education_num| marital_status| occupation| relationship| race| sex|capital_gain|capital_loss|hours_per_week|native_country|income|\n+---+----------------+------+---------+-------------+------------------+-----------------+-------------+-----+------+------------+------------+--------------+--------------+------+\n| 39| State-gov| 77516|Bachelors| 13| Never-married| Adm-clerical|Not-in-family|White| Male| 2174| 0| 40| United-States| <=50K|\n| 50|Self-emp-not-inc| 83311|Bachelors| 13|Married-civ-spouse| Exec-managerial| Husband|White| Male| 0| 0| 13| United-States| <=50K|\n| 38| Private|215646| HS-grad| 9| Divorced|Handlers-cleaners|Not-in-family|White| Male| 0| 0| 40| United-States| <=50K|\n| 53| Private|234721| 11th| 7|Married-civ-spouse|Handlers-cleaners| Husband|Black| Male| 0| 0| 40| United-States| <=50K|\n| 28| Private|338409|Bachelors| 13|Married-civ-spouse| Prof-specialty| Wife|Black|Female| 0| 0| 40| Cuba| <=50K|\n+---+----------------+------+---------+-------------+------------------+-----------------+-------------+-----+------+------------+------------+--------------+--------------+------+\nonly showing top 5 rows", + "output_type": "stream" } ], - "execution_count": 5 + "execution_count": 3 }, { "cell_type": "markdown", @@ -336,7 +251,7 @@ "#Write from Spark to SQL table using MSSQL Spark Connector\r\n", "print(\"MSSQL Spark Connector write(overwrite) start \")\r\n", "\r\n", - "servername = \"jdbc:sqlserver://master-p-svc:1433\"\r\n", + "servername = \"jdbc:sqlserver://mssql.azdata.local:31433\"\r\n", "dbname = \"spark_mssql_db\"\r\n", "security_spec = \";integratedSecurity=true;authenticationScheme=JavaKerberos;\"\r\n", "url = servername + \";\" + \"databaseName=\" + dbname + security_spec\r\n", @@ -364,24 +279,24 @@ }, "outputs": [ { - "output_type": "display_data", "data": { "text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…", "application/vnd.jupyter.widget-view+json": { "version_major": 2, "version_minor": 0, - "model_id": "644bfe3ba48142f38f52717cab9eb19d" + "model_id": "82fe4fdfbb0c4110871ab3f0e4011600" } }, - "metadata": {} + "metadata": {}, + "output_type": "display_data" }, { - "output_type": "stream", "name": "stdout", - "text": "Use MSSQL connector to write to master SQL instance \nMSSQL Connector write(overwrite) succeeded" + "text": "MSSQL Spark Connector write(overwrite) start \nMSSQL Connector write(overwrite) done", + "output_type": "stream" } ], - "execution_count": 7 + "execution_count": 11 }, { "cell_type": "code", @@ -406,12 +321,24 @@ }, "outputs": [ { - "output_type": "stream", + "data": { + "text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…", + "application/vnd.jupyter.widget-view+json": { + "version_major": 2, + "version_minor": 0, + "model_id": "3148b605ab214b408412efb9bfe6b14c" + } + }, + "metadata": {}, + "output_type": "display_data" + }, + { "name": "stdout", - "text": "read data from SQL server table \n+---+----------------+------+---------+-------------+------------------+-----------------+-------------+-----+------+------------+------------+--------------+--------------+------+\n|age| workclass|fnlwgt|education|education_num| marital_status| occupation| relationship| race| sex|capital_gain|capital_loss|hours_per_week|native_country|income|\n+---+----------------+------+---------+-------------+------------------+-----------------+-------------+-----+------+------------+------------+--------------+--------------+------+\n| 39| State-gov| 77516|Bachelors| 13| Never-married| Adm-clerical|Not-in-family|White| Male| 2174| 0| 40| United-States| <=50K|\n| 50|Self-emp-not-inc| 83311|Bachelors| 13|Married-civ-spouse| Exec-managerial| Husband|White| Male| 0| 0| 13| United-States| <=50K|\n| 38| Private|215646| HS-grad| 9| Divorced|Handlers-cleaners|Not-in-family|White| Male| 0| 0| 40| United-States| <=50K|\n| 53| Private|234721| 11th| 7|Married-civ-spouse|Handlers-cleaners| Husband|Black| Male| 0| 0| 40| United-States| <=50K|\n| 28| Private|338409|Bachelors| 13|Married-civ-spouse| Prof-specialty| Wife|Black|Female| 0| 0| 40| Cuba| <=50K|\n+---+----------------+------+---------+-------------+------------------+-----------------+-------------+-----+------+------------+------------+--------------+--------------+------+\nonly showing top 5 rows" + "text": "MSSQL Spark Connector read start \n+---+----------------+------+---------+-------------+------------------+-----------------+-------------+-----+------+------------+------------+--------------+--------------+------+\n|age| workclass|fnlwgt|education|education_num| marital_status| occupation| relationship| race| sex|capital_gain|capital_loss|hours_per_week|native_country|income|\n+---+----------------+------+---------+-------------+------------------+-----------------+-------------+-----+------+------------+------------+--------------+--------------+------+\n| 39| State-gov| 77516|Bachelors| 13| Never-married| Adm-clerical|Not-in-family|White| Male| 2174| 0| 40| United-States| <=50K|\n| 50|Self-emp-not-inc| 83311|Bachelors| 13|Married-civ-spouse| Exec-managerial| Husband|White| Male| 0| 0| 13| United-States| <=50K|\n| 38| Private|215646| HS-grad| 9| Divorced|Handlers-cleaners|Not-in-family|White| Male| 0| 0| 40| United-States| <=50K|\n| 53| Private|234721| 11th| 7|Married-civ-spouse|Handlers-cleaners| Husband|Black| Male| 0| 0| 40| United-States| <=50K|\n| 28| Private|338409|Bachelors| 13|Married-civ-spouse| Prof-specialty| Wife|Black|Female| 0| 0| 40| Cuba| <=50K|\n+---+----------------+------+---------+-------------+------------------+-----------------+-------------+-----+------+------------+------------+--------------+--------------+------+\nonly showing top 5 rows\n\nMSSQL Spark Connector read done", + "output_type": "stream" } ], - "execution_count": 7 + "execution_count": 13 }, { "cell_type": "markdown", @@ -436,7 +363,7 @@ "#Write from Spark to datapools using Spark Connector\r\n", "print(\"MSSQL Spark Connector write(overwrite) start \")\r\n", "\r\n", - "servername = \"jdbc:sqlserver://master-p-svc:1433\"\r\n", + "servername = \"jdbc:sqlserver://mssql.azdata.local:31433\"\r\n", "dbname = \"spark_mssql_db\"\r\n", "security_spec = \";integratedSecurity=true;authenticationScheme=JavaKerberos;\"\r\n", "url = servername + \";\" + \"databaseName=\" + dbname + security_spec\r\n", @@ -465,8 +392,26 @@ "metadata": { "azdata_cell_guid": "9cbe5af3-6ddb-4f19-8423-d10cbb7d48a7" }, - "outputs": [], - "execution_count": null + "outputs": [ + { + "data": { + "text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…", + "application/vnd.jupyter.widget-view+json": { + "version_major": 2, + "version_minor": 0, + "model_id": "17795c5c901e469da197644ce432836b" + } + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "text": "MSSQL Spark Connector write(overwrite) start \nMSSQL Connector write(overwrite) done", + "output_type": "stream" + } + ], + "execution_count": 12 }, { "cell_type": "code", @@ -489,8 +434,26 @@ "metadata": { "azdata_cell_guid": "5550bfce-4852-4fb8-9caa-1120c05dafb7" }, - "outputs": [], - "execution_count": null + "outputs": [ + { + "data": { + "text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…", + "application/vnd.jupyter.widget-view+json": { + "version_major": 2, + "version_minor": 0, + "model_id": "19c6e7b6076b42718229d4ae61ccd61c" + } + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "text": "MSSQL Spark Connector read data pool external table start \n+---+----------------+------+---------+-------------+------------------+-----------------+-------------+-----+------+------------+------------+--------------+--------------+------+\n|age| workclass|fnlwgt|education|education_num| marital_status| occupation| relationship| race| sex|capital_gain|capital_loss|hours_per_week|native_country|income|\n+---+----------------+------+---------+-------------+------------------+-----------------+-------------+-----+------+------------+------------+--------------+--------------+------+\n| 39| State-gov| 77516|Bachelors| 13| Never-married| Adm-clerical|Not-in-family|White| Male| 2174| 0| 40| United-States| <=50K|\n| 50|Self-emp-not-inc| 83311|Bachelors| 13|Married-civ-spouse| Exec-managerial| Husband|White| Male| 0| 0| 13| United-States| <=50K|\n| 38| Private|215646| HS-grad| 9| Divorced|Handlers-cleaners|Not-in-family|White| Male| 0| 0| 40| United-States| <=50K|\n| 53| Private|234721| 11th| 7|Married-civ-spouse|Handlers-cleaners| Husband|Black| Male| 0| 0| 40| United-States| <=50K|\n| 28| Private|338409|Bachelors| 13|Married-civ-spouse| Prof-specialty| Wife|Black|Female| 0| 0| 40| Cuba| <=50K|\n+---+----------------+------+---------+-------------+------------------+-----------------+-------------+-----+------+------------+------------+--------------+--------------+------+\nonly showing top 5 rows\n\nMSSQL Connector read from data pool external table succeeded", + "output_type": "stream" + } + ], + "execution_count": 14 } ] } \ No newline at end of file diff --git a/samples/features/sql-big-data-cluster/spark/data-virtualization/mssql_spark_connector_user_creation.ipynb b/samples/features/sql-big-data-cluster/spark/data-virtualization/mssql_spark_connector_user_creation.ipynb index 8e58681964..9d038ff346 100644 --- a/samples/features/sql-big-data-cluster/spark/data-virtualization/mssql_spark_connector_user_creation.ipynb +++ b/samples/features/sql-big-data-cluster/spark/data-virtualization/mssql_spark_connector_user_creation.ipynb @@ -27,14 +27,32 @@ "azdata_cell_guid": "4d6539fc-aa99-4cd8-a9de-357e2d6424e2" } }, + { + "cell_type": "markdown", + "source": [ + "## For non-AD cluster, run the following cells to prepare user and table:" + ], + "metadata": { + "azdata_cell_guid": "9642f940-770f-4bfd-85ee-e0159ccb5689" + } + }, { "cell_type": "code", "source": [ - "-- Start with creating a testdatabase. \r\n", - "Create DATABASE connector_test_db" + "-- Create a database named \"connector_test_db\"\r\n", + "IF NOT EXISTS (SELECT * FROM sys.databases WHERE name = N'connector_test_db')\r\n", + "BEGIN\r\n", + " Create DATABASE connector_test_db\r\n", + "END\r\n", + "\r\n", + "-- Create a login\r\n", + "IF NOT EXISTS (select name from sys.server_principals where name='connector_user')\r\n", + "BEGIN\r\n", + " CREATE LOGIN connector_user WITH PASSWORD ='password123!#' \r\n", + "END" ], "metadata": { - "azdata_cell_guid": "8fdf2f56-f89d-4bf9-81be-1dfb92631251" + "azdata_cell_guid": "84932473-88cd-433e-baf7-eeee178021aa" }, "outputs": [ { @@ -47,37 +65,44 @@ { "output_type": "display_data", "data": { - "text/html": "Total execution time: 00:00:01.345" + "text/html": "Total execution time: 00:00:00.335" }, "metadata": {} } ], - "execution_count": 1 + "execution_count": 6 }, { "cell_type": "code", "source": [ - "Use connector_test_db;\n", - "CREATE LOGIN connector_user WITH PASSWORD ='password123!#' \n", - "CREATE USER connector_user FROM LOGIN connector_user\n", - "\n", - "-- to view data pool node configuration\n", - "grant VIEW DATABASE STATE to connector_user\n", - "\n", - "-- To create external tables in data pools\n", - "grant alter any external data source to connector_user;\n", - "\n", - "-- To create external table\n", - "grant create table to connector_user;\n", - "grant alter any schema to connector_user;\n", - "\n", - "\n", - "\n", - "ALTER ROLE [db_datareader] ADD MEMBER connector_user\n", - "ALTER ROLE [db_datawriter] ADD MEMBER connector_user" + "Use connector_test_db;\r\n", + "\r\n", + "-- Create a database user\r\n", + "CREATE USER connector_user FROM LOGIN connector_user\r\n", + "\r\n", + "-- To view data pool node configuration\r\n", + "grant VIEW DATABASE STATE to connector_user;\r\n", + "\r\n", + "-- To create external tables in data pools\r\n", + "grant alter any external data source to connector_user;\r\n", + "\r\n", + "-- To create external table\r\n", + "grant create table to connector_user;\r\n", + "grant alter any schema to connector_user;\r\n", + "\r\n", + "ALTER ROLE [db_datareader] ADD MEMBER connector_user;\r\n", + "ALTER ROLE [db_datawriter] ADD MEMBER connector_user;\r\n", + "\r\n", + "-- Create external table in data pool\r\n", + "CREATE EXTERNAL DATA SOURCE connector_ds WITH (LOCATION = 'sqldatapool://controller-svc/default');\r\n", + "EXECUTE('USE connector_test_db; CREATE EXTERNAL TABLE [dummy3] ([number] int, [word] nvarchar(2048)) WITH (DATA_SOURCE = connector_ds, DISTRIBUTION = ROUND_ROBIN)')\r\n", + "\r\n", + "-- Create a login in data pools and Provide right permissions to this user\r\n", + "EXECUTE( ' Use connector_test_db; CREATE LOGIN connector_user WITH PASSWORD = ''password123!#'' ; ' ) AT DATA_SOURCE connector_ds;\r\n", + "EXECUTE( ' Use connector_test_db; CREATE USER connector_user ; ALTER ROLE [db_datareader] ADD MEMBER connector_user; ALTER ROLE [db_datawriter] ADD MEMBER connector_user ;') AT DATA_SOURCE connector_ds;" ], "metadata": { - "azdata_cell_guid": "4dd976a3-1ed7-43d3-ad29-657fe1046fb3" + "azdata_cell_guid": "8fdf2f56-f89d-4bf9-81be-1dfb92631251" }, "outputs": [ { @@ -90,25 +115,39 @@ { "output_type": "display_data", "data": { - "text/html": "Total execution time: 00:00:00.087" + "text/html": "Total execution time: 00:00:10.334" }, "metadata": {} } ], - "execution_count": 2 + "execution_count": 1 + }, + { + "cell_type": "markdown", + "source": [ + "## For AD cluster, run the following cells to prepare user and table:" + ], + "metadata": { + "azdata_cell_guid": "11ea89a5-5f60-495d-864e-a80ef7a10510" + } }, { "cell_type": "code", "source": [ - "-- Add this user to a data pool user.\r\n", - "Use connector_test_db;\r\n", - "CREATE EXTERNAL DATA SOURCE connector_ds WITH (LOCATION = 'sqldatapool://controller-svc/default');\r\n", - "EXECUTE('USE connector_test_db; CREATE EXTERNAL TABLE [dummy3] ([number] int, [word] nvarchar(2048)) WITH (DATA_SOURCE = connector_ds, DISTRIBUTION = ROUND_ROBIN)')\r\n", + "-- Create a database named \"spark_mssql_db\"\r\n", + "IF NOT EXISTS (SELECT * FROM sys.databases WHERE name = N'spark_mssql_db')\r\n", + "BEGIN\r\n", + " CREATE DATABASE spark_mssql_db\r\n", + "END\r\n", "\r\n", - "" + "-- Create a login from a domain user\r\n", + "IF NOT EXISTS (select name from sys.server_principals where name='AZDATA\\testusera1')\r\n", + "BEGIN\r\n", + " CREATE LOGIN [AZDATA\\testusera1] FROM WINDOWS\r\n", + "END" ], "metadata": { - "azdata_cell_guid": "0b06540d-0787-493d-aba6-83420200da14" + "azdata_cell_guid": "819012ba-db8f-4c25-9606-1801f46d60e4" }, "outputs": [ { @@ -121,22 +160,45 @@ { "output_type": "display_data", "data": { - "text/html": "Total execution time: 00:00:04.496" + "text/html": "Total execution time: 00:00:10.921" }, "metadata": {} } ], - "execution_count": 3 + "execution_count": 18 }, { "cell_type": "code", "source": [ + "Use spark_mssql_db;\r\n", + "\r\n", + "-- Create a database user\r\n", + "CREATE USER [AZDATA\\testusera1];\r\n", + "\r\n", + "-- To view data pool node configuration\r\n", + "GRANT VIEW DATABASE STATE to [AZDATA\\testusera1];\r\n", + "\r\n", + "-- To create external tables in data pools\r\n", + "grant alter any external data source to [AZDATA\\testusera1];\r\n", + "\r\n", + "-- To create external table\r\n", + "grant create table to [AZDATA\\testusera1];\r\n", + "grant alter any schema to [AZDATA\\testusera1];\r\n", + "\r\n", + "ALTER ROLE [db_datareader] ADD MEMBER [AZDATA\\testusera1];\r\n", + "ALTER ROLE [db_datawriter] ADD MEMBER [AZDATA\\testusera1];\r\n", + "\r\n", + "-- Create external table in data pool\r\n", + "CREATE EXTERNAL DATA SOURCE connector_ds WITH (LOCATION = 'sqldatapool://controller-svc/default');\r\n", + "EXECUTE('USE spark_mssql_db; CREATE EXTERNAL TABLE [dummy3] ([number] int, [word] nvarchar(2048)) WITH (DATA_SOURCE = connector_ds, DISTRIBUTION = ROUND_ROBIN)')\r\n", + "\r\n", "-- Create a login in data pools and Provide right permissions to this user\r\n", - "EXECUTE( ' Use connector_test_db; CREATE LOGIN connector_user WITH PASSWORD = ''password123!#'' ; ' ) AT DATA_SOURCE connector_ds;\r\n", - "EXECUTE( ' Use connector_test_db; CREATE USER connector_user ; ALTER ROLE [db_datareader] ADD MEMBER connector_user; ALTER ROLE [db_datawriter] ADD MEMBER connector_user ;') AT DATA_SOURCE connector_ds;" + "EXECUTE( ' USE spark_mssql_db; CREATE LOGIN [AZDATA\\testusera1] FROM WINDOWS ' ) AT DATA_SOURCE connector_ds;\r\n", + "\r\n", + "EXECUTE( ' USE spark_mssql_db; CREATE USER [AZDATA\\testusera1] ; ALTER ROLE [db_datareader] ADD MEMBER [AZDATA\\testusera1]; ALTER ROLE [db_datawriter] ADD MEMBER [AZDATA\\testusera1] ;') AT DATA_SOURCE connector_ds;" ], "metadata": { - "azdata_cell_guid": "5079dfe9-22e3-4a8f-919b-c6219f5a901a" + "azdata_cell_guid": "2c853db3-e081-4229-a66f-5c248fb8eb0d" }, "outputs": [ { @@ -149,12 +211,12 @@ { "output_type": "display_data", "data": { - "text/html": "Total execution time: 00:00:01.174" + "text/html": "Total execution time: 00:00:10.628" }, "metadata": {} } ], - "execution_count": 5 + "execution_count": 19 } ] } \ No newline at end of file