Updating to New DSVM and Setup Script (#11)

* checking in

* used correct branch

* fixed jupyrt notebook

* updated set up for Filestream access

* Updated Setup and SQL Configuration to work with more solutions

* updated positional parameters in ConfigureSQL.ps1

* fixed setup

* refixed setup

* Testing to see if Enabling file stream works

* testing File Stream Access

* testing file stream

* done testing filestream

* changed to securstring password

* fixed

* changed to master branch on git

Data/Product.csv

@@ -1,7 +1,7 @@
 Product_Id,Product,Category,Term,No_of_people_covered,Premium,Payment_frequency,Net_Amt_Insured,Amt_on_Maturity,Amt_on_Maturity_Bin
-1,Protect Your Future,Long Term Care,10,4,1000,Monthly,1e+05,180000,<200000
-2,Live Free,Life,15,2,1500,Monthly,2e+05,405000,<400000
+1,Protect Your Future,Long Term Care,10,4,1000,Monthly,100000,180000,<200000
+2,Live Free,Life,15,2,1500,Monthly,200000,405000,<400000
 3,Secured Happiness,Health,20,1,2000,Monthly,150000,720000,<400000
-4,Making Tomorrow Better,Disability,30,4,700,Quarterly,1e+05,126000,<200000
-5,Secured Life,Health,24,2,900,Quarterly,2e+05,129600,<200000
+4,Making Tomorrow Better,Disability,30,4,700,Quarterly,100000,126000,<200000
+5,Secured Life,Health,24,2,900,Quarterly,200000,129600,<200000
 6,Live Happy,Life,16,5,2000,Yearly,150000,48000,<200000

R/Campaign Optimization R Notebook.ipynb

@@ -240,11 +240,10 @@
    ],
    "source": [
     "# Load the 4 raw data sets in-memory from CSV files. Use the path in read.csv if data not loaded into Jupyter folder.\n",
-    "# path = \"C:/Users/dsvmuser/Desktop/Campaign/Data/\"\n",
-    "table_Campaign_Detail <- read.csv( \"Campaign_Detail.csv\")\n",
-    "table_Lead_Demography <- read.csv(\"Lead_Demography.csv\")\n",
-    "table_Market_Touchdown <- read.csv(\"Market_Touchdown.csv\")\n",
-    "table_Product <- read.csv(\"Product.csv\")\n",
+    "table_Campaign_Detail <- read.csv( \"../Data/Campaign_Detail.csv\")\n",
+    "table_Lead_Demography <- read.csv(\"../Data/Lead_Demography.csv\")\n",
+    "table_Market_Touchdown <- read.csv(\"../Data/Market_Touchdown.csv\")\n",
+    "table_Product <- read.csv(\"../Data/Product.csv\")\n",
     "\n",
     "print(\"Data read in-memory\")"
    ]

Resources/ActionScripts/ConfigureSQL.ps1

@@ -0,0 +1,255 @@
+[CmdletBinding()]
+param(
+[parameter(Mandatory=$true, Position=1)]
+[string]$ServerName,
+
+[parameter(Mandatory=$true, Position=2)]
+[string]$SolutionName,
+
+[parameter(Mandatory=$true, Position=3)]
+[string]$InstallPy,
+
+[parameter(Mandatory=$true, Position=4)]
+[string]$InstallR,
+
+[parameter(Mandatory=$true, Position=5)]
+[string]$Prompt
+)
+
+
+
+
+
+
+$Prompt = 'N'
+
+
+
+
+
+
+###$ServerName = if ($Prompt -eq 'Y') {Read-Host  -Prompt "Enter SQL Server Name Or SQL InstanceName you are installing on"} else {$si}
+
+
+WRITE-HOST " ServerName set to $ServerName"
+
+$db = if ($Prompt -eq 'Y') {Read-Host  -Prompt "Enter Desired Database Base Name"} else {$SolutionName} 
+
+$dataList = ("Campaign_Detail", "Lead_Demography", "Market_Touchdown", "Product")
+
+
+
+##########################################################################
+
+# Create Database and BaseTables 
+
+#########################################################################
+
+####################################################################
+# Check to see If SQL Version is at least SQL 2017 and Not SQL Express 
+####################################################################
+
+
+$query = 
+"select 
+        case 
+            when 
+                cast(left(cast(serverproperty('productversion') as varchar), 4) as numeric(4,2)) >= 14 
+                and CAST(SERVERPROPERTY ('edition') as varchar) Not like 'Express%' 
+            then 'Yes'
+        else 'No' end as 'isSQL17'"
+
+$isCompatible = Invoke-Sqlcmd -ServerInstance $ServerName -Database Master -Query $query
+$isCompatible = $isCompatible.Item(0)
+if ($isCompatible -eq 'Yes' -and $InstallPy -eq 'Yes') {
+    Write-Host " This Version of SQL is Compatible with SQL Py "
+
+    ## Create Py Database
+    Write-Host "  Creating SQL Database for Py "
+
+
+    Write-Host -ForeGroundColor 'cyan' (" Using $ServerName SQL Instance") 
+
+    ## Create PY Server DB
+    $dbName = $db + "_Py"
+    $SqlParameters = @("dbName=$dbName")
+
+    $CreateSQLDB = "$ScriptPath\CreateDatabase.sql"
+
+    $CreateSQLObjects = "$ScriptPath\CreateSQLObjectsPy.sql"
+    Write-Host -ForeGroundColor 'cyan' (" Calling Script to create the  $dbName database") 
+    invoke-sqlcmd -inputfile $CreateSQLDB -serverinstance $ServerName -database master -Variable $SqlParameters
+
+
+    Write-Host -ForeGroundColor 'cyan' (" SQLServerDB $dbName Created")
+    invoke-sqlcmd "USE $dbName;" 
+
+    Write-Host -ForeGroundColor 'cyan' (" Calling Script to create the objects in the $dbName database")
+    invoke-sqlcmd -inputfile $CreateSQLObjects -serverinstance $ServerName -database $dbName
+
+
+    Write-Host -ForeGroundColor 'cyan' (" SQLServerObjects Created in $dbName Database")
+$OdbcName = "obdc" + $dbname
+ ## Create ODBC Connection for PowerBI to Use 
+Add-OdbcDsn -Name $OdbcName -DriverName "ODBC Driver 13 for SQL Server" -DsnType 'System' -Platform '64-bit' -SetPropertyValue @("Server=$ServerName", "Trusted_Connection=Yes", "Database=$dbName") -ErrorAction SilentlyContinue -PassThru
+
+
+
+
+}
+else 
+{
+    if ($isCompatible -eq 'Yes' -and $InstallPy -eq 'Yes') {"This Version of SQL is not compatible with Py , Py Code and DB's will not be Created "}
+    else {" There is not a py version of this solution"}
+}
+
+
+
+
+If ($InstallR -eq 'Yes')
+{
+Write-Host "  Creating SQL Database for R "
+
+
+Write-Host -ForeGroundColor 'cyan' (" Using $ServerName SQL Instance") 
+
+$dbName = $db + "_R"
+
+
+## Create RServer DB 
+$SqlParameters = @("dbName=$dbName")
+
+$CreateSQLDB = "$ScriptPath\CreateDatabase.sql"
+
+$CreateSQLObjects = "$ScriptPath\CreateSQLObjectsR.sql"
+Write-Host -ForeGroundColor 'cyan' (" Calling Script to create the  $dbName database") 
+invoke-sqlcmd -inputfile $CreateSQLDB -serverinstance $ServerName -database master -Variable $SqlParameters
+
+
+Write-Host -ForeGroundColor 'cyan' (" SQLServerDB $dbName Created")
+invoke-sqlcmd "USE $dbName;" 
+
+Write-Host -ForeGroundColor 'cyan' (" Calling Script to create the objects in the $dbName database")
+invoke-sqlcmd -inputfile $CreateSQLObjects -serverinstance $ServerName -database $dbName
+
+
+Write-Host -ForeGroundColor 'cyan' (" SQLServerObjects Created in $dbName Database")
+
+
+###Configure Database for R 
+Write-Host "  
+Configuring $SolutionName Solution for R
+"
+
+$dbName = $db + "_R" 
+
+## Create ODBC Connection for PowerBI to Use 
+$OdbcName = "obdc" + $dbname
+## Create ODBC Connection for PowerBI to Use 
+Add-OdbcDsn -Name $OdbcName -DriverName "ODBC Driver 13 for SQL Server" -DsnType 'System' -Platform '64-bit' -SetPropertyValue @("Server=$ServerName", "Trusted_Connection=Yes", "Database=$dbName") -ErrorAction SilentlyContinue -PassThru
+
+
+##########################################################################
+# Deployment Pipeline
+##########################################################################
+
+$RStart = Get-Date
+try
+{
+
+Write-Host -ForeGroundColor 'cyan' (" Import CSV File(s). This Should take about 30 Seconds Per File")
+ ##Move this to top 
+
+
+# upload csv files into SQL tables
+foreach ($dataFile in $dataList)
+{
+$destination = $SolutionData + $dataFile + ".csv" 
+$tableName = $DBName + ".dbo." + $dataFile
+$tableSchema = $dataPath + "\" + $dataFile + ".xml"
+$dataSet = Import-Csv $destination
+Write-Host -ForegroundColor 'cyan' ("         Loading $dataFile.csv into SQL Table") 
+Write-SqlTableData -InputData $dataSet  -DatabaseName $dbName -Force -Passthru -SchemaName dbo -ServerInstance $ServerName -TableName $dataFile
+
+
+Write-Host -ForeGroundColor 'cyan' (" $datafile table loaded from CSV File(s).")
+}
+}
+catch
+{
+Write-Host -ForegroundColor DarkYellow "Exception in populating database tables:"
+Write-Host -ForegroundColor Red $Error[0].Exception 
+throw
+}
+Write-Host -ForeGroundColor 'cyan' (" Finished loading .csv File(s).")
+
+Write-Host -ForeGroundColor 'Cyan' (" Training Model and Scoring Data...")
+
+
+
+$query = "EXEC Initial_Run_Once_R"
+#SqlServer\Invoke-Sqlcmd -ServerInstance $ServerName -Database $dbName -Query $query -ConnectionTimeout  0 -QueryTimeout 0
+SqlServer\Invoke-Sqlcmd -ServerInstance LocalHost -Database $dbName -Query $query -ConnectionTimeout  0 -QueryTimeout 0
+
+$Rend = Get-Date
+
+$Duration = New-TimeSpan -Start $RStart -End $Rend 
+Write-Host -ForegroundColor 'green'(" R Server Configured in $Duration")
+}
+ELSE 
+{Write-Host -ForegroundColor 'Green' "There is not a R Version for this Solution so R will not be Installed"}
+
+
+###Conifgure Database for Py 
+if ($isCompatible -eq 'Yes'-and $InstallPy -eq 'Yes')
+{
+$PyStart = get-date
+Write-Host "  
+Configuring $SolutionName Solution for Py
+# "
+$dbname = $db + "_Py"
+
+##########################################################################
+# Deployment Pipeline Py
+##########################################################################
+
+
+try
+{
+
+Write-Host -ForeGroundColor 'cyan' (" Import CSV File(s). This Should take about 30 Seconds Per File")
+#$dataList = "LengthOfStay"
+
+
+# upload csv files into SQL tables
+foreach ($dataFile in $dataList)
+{
+    $destination = $SolutionData + $dataFile + ".csv" 
+    $tableName = $DBName + ".dbo." + $dataFile
+    $tableSchema = $dataPath + "\" + $dataFile + ".xml"
+    $dataSet = Import-Csv $destination
+ Write-Host -ForegroundColor 'cyan' ("         Loading $dataFile.csv into SQL Table") 
+    Write-SqlTableData -InputData $dataSet  -DatabaseName $dbName -Force -Passthru -SchemaName dbo -ServerInstance $ServerName -TableName $dataFile
+
+
+ Write-Host -ForeGroundColor 'cyan' (" $datafile table loaded from CSV File(s).")
+}
+}
+catch
+{
+Write-Host -ForegroundColor DarkYellow "Exception in populating database tables:"
+Write-Host -ForegroundColor Red $Error[0].Exception 
+throw
+}
+Write-Host -ForeGroundColor 'cyan' (" Finished loading .csv File(s).")
+
+Write-Host -ForeGroundColor 'Cyan' (" Training Model and Scoring Data...")
+$query = "EXEC Inital_Run_Once_Py"
+SqlServer\Invoke-Sqlcmd -ServerInstance LocalHost -Database $dbName -Query $query -ConnectionTimeout  0 -QueryTimeout 0
+
+$Pyend = Get-Date
+
+$Duration = New-TimeSpan -Start $PyStart -End $Pyend 
+Write-Host -ForegroundColor 'green'(" Py Server Configured in $Duration")
+
+}