Merge pull request #263 from pbi-tools/release/rc.3-tmdl

1.0.0-rc.3 Release

RELEASE_NOTES.md

@@ -1,9 +1,10 @@
 # Release Notes
 
-## 1.0.0-rc.3 - 2023-04-01
-
-<version:1.0.0-rc.3+preview.2>
+## 1.0.0-rc.3 - 2023-04-11
 
+- #262 TMDL Serialization Support
+  - New model serialization modes: `Tmdl` (default), `Legacy` (PbixProj)
+  - New (optional) environment variable: `PBITOOLS_DefaultModelSerialization`
 - New CLI action: extract-pbidesktop
   - Extracts binaries from a PBIDesktopSetup.exe|.msi installer bundle (silent/x-copy install). (Implemented using an embedded tool: wix-extract.exe)
   - Arguments:

docs/usage-core.md

@@ -2,7 +2,7 @@
 
     pbi-tools.core <action> -options
 
-_pbi-tools (Core), 1.0.0-rc.3+preview.2 - https://pbi.tools/_
+_pbi-tools (Core), 1.0.0-rc.3 - https://pbi.tools/_
 
 ### Actions
 
@@ -29,7 +29,7 @@ Performs an offline conversion of PbixProj or Tabular model sources into another
 | --- | --- | --- | --- |
 | source* |  |  | The source(s) to convert. Can be a PbixProj folder, a Model/TE folder, or a TMSL json file. |
 | outPath |  |  | The (optional) destination. Can be a folder or a file, depending on the conversion mode. Must be a folder if the source is a TMSL json file. |
-| modelSerialization |  |  | The model serialization mode. <br> `Default`  - Serializes the tabular model into the default PbixProj folder structure and performs various transformations to optimize file contents for source control. <br> `Raw`  - Serializes the tabular model into a single JSON file containing the full TMSL payload from the PBIX model. No transformations are applied. |
+| modelSerialization |  |  | The model serialization mode. <br> `Default`  - The default serialization format, effective if no option is specified. The default is TMDL. <br> `Raw`  - Serializes the tabular model into a single JSON file containing the full TMSL payload from the PBIX model. No transformations are applied. <br> `Legacy`  - Serializes the tabular model into the default PbixProj folder structure and performs various transformations to optimize file contents for source control. <br> `Tmdl`  - Serializes the tabular model into TMDL format. |
 | mashupSerialization |  |  | The mashup serialization mode. <br> `Default`  - Similar to 'Raw' mode, with the exception that QueryGroups are extracted into a separate file for readability. <br> `Raw`  - Serializes all Mashup parts with no transformations applied. <br> `Expanded`  - Serializes the Mashup metadata part into a Json document, and embedded M queries into separate files. This mode is not supported for compilation. |
 | settingsFile |  |  | An external .pbixproj.json file containing serialization settings. Serialization modes specified as command-line arguments take precedence. |
 | updateSettings | `False` | X | If set, updates the effective PbixProj settings file used for this conversion. |

docs/usage.md

@@ -2,7 +2,7 @@
 
     pbi-tools <action> -options
 
-_pbi-tools (Desktop), 1.0.0-rc.3+preview.2 - https://pbi.tools/_
+_pbi-tools (Desktop), 1.0.0-rc.3 - https://pbi.tools/_
 
 ### Actions
 
@@ -43,7 +43,7 @@ Performs an offline conversion of PbixProj or Tabular model sources into another
 | --- | --- | --- | --- |
 | source* |  |  | The source(s) to convert. Can be a PbixProj folder, a Model/TE folder, or a TMSL json file. |
 | outPath |  |  | The (optional) destination. Can be a folder or a file, depending on the conversion mode. Must be a folder if the source is a TMSL json file. |
-| modelSerialization |  |  | The model serialization mode. <br> `Default`  - Serializes the tabular model into the default PbixProj folder structure and performs various transformations to optimize file contents for source control. <br> `Raw`  - Serializes the tabular model into a single JSON file containing the full TMSL payload from the PBIX model. No transformations are applied. |
+| modelSerialization |  |  | The model serialization mode. <br> `Default`  - The default serialization format, effective if no option is specified. The default is TMDL. <br> `Raw`  - Serializes the tabular model into a single JSON file containing the full TMSL payload from the PBIX model. No transformations are applied. <br> `Legacy`  - Serializes the tabular model into the default PbixProj folder structure and performs various transformations to optimize file contents for source control. <br> `Tmdl`  - Serializes the tabular model into TMDL format. |
 | mashupSerialization |  |  | The mashup serialization mode. <br> `Default`  - Similar to 'Raw' mode, with the exception that QueryGroups are extracted into a separate file for readability. <br> `Raw`  - Serializes all Mashup parts with no transformations applied. <br> `Expanded`  - Serializes the Mashup metadata part into a Json document, and embedded M queries into separate files. This mode is not supported for compilation. |
 | settingsFile |  |  | An external .pbixproj.json file containing serialization settings. Serialization modes specified as command-line arguments take precedence. |
 | updateSettings | `False` | X | If set, updates the effective PbixProj settings file used for this conversion. |
@@ -102,7 +102,7 @@ Extracts the contents of a PBIX/PBIT file into a folder structure suitable for s
 | pbiPort |  |  | The port number from a running Power BI Desktop instance (look up via 'pbi-tools info'). When specified, the model will not be read from the PBIX file, and will instead be retrieved from the PBI instance. Only supported for V3 PBIX files. |
 | extractFolder |  |  | The folder to extract the PBIX file to. Only needed to override the default location. Can be relative to current working directory. |
 | mode | `Auto` |  | The extraction mode. <br> `Auto`  - Attempts extraction using the V3 model, and falls back to Legacy mode in case the PBIX file does not have V3 format. <br> `V3`  - Extracts V3 PBIX files only. Fails if the file provided has a legacy format. <br> `Legacy`  - Extracts legacy PBIX files only. Fails if the file provided has the V3 format. |
-| modelSerialization |  |  | The model serialization mode. <br> `Default`  - Serializes the tabular model into the default PbixProj folder structure and performs various transformations to optimize file contents for source control. <br> `Raw`  - Serializes the tabular model into a single JSON file containing the full TMSL payload from the PBIX model. No transformations are applied. |
+| modelSerialization |  |  | The model serialization mode. <br> `Default`  - The default serialization format, effective if no option is specified. The default is TMDL. <br> `Raw`  - Serializes the tabular model into a single JSON file containing the full TMSL payload from the PBIX model. No transformations are applied. <br> `Legacy`  - Serializes the tabular model into the default PbixProj folder structure and performs various transformations to optimize file contents for source control. <br> `Tmdl`  - Serializes the tabular model into TMDL format. |
 | mashupSerialization |  |  | The mashup serialization mode. <br> `Default`  - Similar to 'Raw' mode, with the exception that QueryGroups are extracted into a separate file for readability. <br> `Raw`  - Serializes all Mashup parts with no transformations applied. <br> `Expanded`  - Serializes the Mashup metadata part into a Json document, and embedded M queries into separate files. This mode is not supported for compilation. |
 | watch |  | X | Enables watch mode. Monitors the PBIX file open in a Power BI Desktop session, and extracts sources each time the file is saved. |
 

paket.dependencies

@@ -24,8 +24,10 @@ group Main
   nuget CommandLineParser
   nuget WildcardMatch
   nuget Microsoft.AnalysisServices.retail.amd64 framework: net472
+  nuget Microsoft.AnalysisServices.Tabular.Tmdl.retail.amd64 prerelease framework: net472
   nuget Microsoft.AnalysisServices.AdomdClient.retail.amd64 framework: net472
   nuget Microsoft.AnalysisServices.NetCore.retail.amd64 framework: net6.0,net7.0
+  nuget Microsoft.AnalysisServices.Tabular.Tmdl.NetCore.retail.amd64 prerelease framework: net6.0,net7.0
   nuget Microsoft.AnalysisServices.AdomdClient.NetCore.retail.amd64 framework: net6.0,net7.0
   nuget System.IO.Packaging
   nuget Microsoft.Identity.Client

paket.lock

@@ -3,7 +3,7 @@ CONTENT: NONE
 RESTRICTION: || (== net472) (== net6.0) (== net7.0)
 NUGET
   remote: https://api.nuget.org/v3/index.json
-    Azure.Core (1.30) - restriction: || (&& (== net472) (>= net6.0)) (== net6.0) (== net7.0)
+    Azure.Core (1.31) - restriction: || (&& (== net472) (>= net6.0)) (== net6.0) (== net7.0)
       Microsoft.Bcl.AsyncInterfaces (>= 1.1.1)
       System.Diagnostics.DiagnosticSource (>= 4.6)
       System.Memory.Data (>= 1.0.2)
@@ -40,17 +40,22 @@ NUGET
       Microsoft.Data.SqlClient (>= 5.0.1) - restriction: || (&& (== net472) (>= net6.0)) (== net6.0) (== net7.0)
     Fody (6.6.4)
     HtmlAgilityPack (1.11.46)
-    LibGit2Sharp (0.26.2)
-      LibGit2Sharp.NativeBinaries (2.0.306)
-    LibGit2Sharp.NativeBinaries (2.0.306)
-    Microsoft.AnalysisServices.AdomdClient.NetCore.retail.amd64 (19.60) - restriction: || (== net6.0) (== net7.0)
+    LibGit2Sharp (0.27)
+      LibGit2Sharp.NativeBinaries (2.0.319)
+    LibGit2Sharp.NativeBinaries (2.0.319)
+    Microsoft.AnalysisServices.AdomdClient.NetCore.retail.amd64 (19.61.1.4) - restriction: || (== net6.0) (== net7.0)
       Microsoft.Identity.Client (>= 4.43)
-    Microsoft.AnalysisServices.AdomdClient.retail.amd64 (19.60) - restriction: == net472
+    Microsoft.AnalysisServices.AdomdClient.retail.amd64 (19.61.1.4) - restriction: == net472
       Microsoft.Identity.Client (>= 4.43)
-    Microsoft.AnalysisServices.NetCore.retail.amd64 (19.60) - restriction: || (== net6.0) (== net7.0)
+    Microsoft.AnalysisServices.NetCore.retail.amd64 (19.61.1.4) - restriction: || (== net6.0) (== net7.0)
       Microsoft.Identity.Client (>= 4.43)
-    Microsoft.AnalysisServices.retail.amd64 (19.60) - restriction: == net472
+    Microsoft.AnalysisServices.retail.amd64 (19.61.1.4) - restriction: == net472
       Microsoft.Identity.Client (>= 4.43)
+    Microsoft.AnalysisServices.Tabular.Tmdl.NetCore.retail.amd64 (19.61.1.4-tmdlpreview) - restriction: || (== net6.0) (== net7.0)
+      Microsoft.AnalysisServices.NetCore.retail.amd64 (>= 19.61.1.4)
+    Microsoft.AnalysisServices.Tabular.Tmdl.retail.amd64 (19.61.1.4-tmdlpreview) - restriction: == net472
+      Microsoft.AnalysisServices.retail.amd64 (>= 19.61.1.4)
+      System.ValueTuple (>= 4.5)
     Microsoft.Azure.Services.AppAuthentication (1.6.2)
       Microsoft.IdentityModel.Clients.ActiveDirectory (>= 5.2.9)
       System.Diagnostics.Process (>= 4.3) - restriction: || (&& (== net472) (< net452)) (== net6.0) (== net7.0)
@@ -73,13 +78,13 @@ NUGET
       System.Text.Encoding.CodePages (>= 6.0) - restriction: || (&& (== net472) (< net462)) (&& (== net472) (>= netstandard2.1)) (== net6.0) (== net7.0)
       System.Text.Encodings.Web (>= 6.0)
     Microsoft.Data.SqlClient.SNI.runtime (5.1) - restriction: || (&& (== net472) (>= net6.0)) (== net6.0) (== net7.0)
-    Microsoft.Identity.Client (4.51)
+    Microsoft.Identity.Client (4.52)
       Microsoft.IdentityModel.Abstractions (>= 6.22)
-    Microsoft.Identity.Client.Extensions.Msal (2.27) - restriction: || (&& (== net472) (>= net6.0)) (== net6.0) (== net7.0)
-      Microsoft.Identity.Client (>= 4.51)
+    Microsoft.Identity.Client.Extensions.Msal (2.28) - restriction: || (&& (== net472) (>= net6.0)) (== net6.0) (== net7.0)
+      Microsoft.Identity.Client (>= 4.52)
       System.IO.FileSystem.AccessControl (>= 5.0) - restriction: || (&& (== net472) (< net45)) (&& (== net472) (>= netcoreapp3.1)) (== net6.0) (== net7.0)
       System.Security.Cryptography.ProtectedData (>= 4.5) - restriction: || (&& (== net472) (< net45)) (&& (== net472) (>= netcoreapp3.1)) (== net6.0) (== net7.0)
-    Microsoft.IdentityModel.Abstractions (6.27)
+    Microsoft.IdentityModel.Abstractions (6.28.1)
     Microsoft.IdentityModel.Clients.ActiveDirectory (5.3)
       Microsoft.CSharp (>= 4.3) - restriction: || (&& (== net472) (< net45)) (== net6.0) (== net7.0)
       NETStandard.Library (>= 1.6.1) - restriction: || (&& (== net472) (< net45)) (== net6.0) (== net7.0)
@@ -94,22 +99,22 @@ NUGET
       System.Security.SecureString (>= 4.3) - restriction: || (&& (== net472) (< net45)) (== net6.0) (== net7.0)
       System.Xml.XDocument (>= 4.3) - restriction: || (&& (== net472) (< net45)) (== net6.0) (== net7.0)
       System.Xml.XmlDocument (>= 4.3) - restriction: || (&& (== net472) (< net45)) (== net6.0) (== net7.0)
-    Microsoft.IdentityModel.JsonWebTokens (6.27) - restriction: || (&& (== net472) (>= net6.0)) (== net6.0) (== net7.0)
-      Microsoft.IdentityModel.Tokens (>= 6.27)
+    Microsoft.IdentityModel.JsonWebTokens (6.28.1) - restriction: || (&& (== net472) (>= net6.0)) (== net6.0) (== net7.0)
+      Microsoft.IdentityModel.Tokens (>= 6.28.1)
       System.Text.Encoding (>= 4.3)
       System.Text.Encodings.Web (>= 4.7.2)
       System.Text.Json (>= 4.7.2)
-    Microsoft.IdentityModel.Logging (6.27) - restriction: || (&& (== net472) (>= net6.0)) (== net6.0) (== net7.0)
-      Microsoft.IdentityModel.Abstractions (>= 6.27)
-    Microsoft.IdentityModel.Protocols (6.27) - restriction: || (&& (== net472) (>= net6.0)) (== net6.0) (== net7.0)
-      Microsoft.IdentityModel.Logging (>= 6.27)
-      Microsoft.IdentityModel.Tokens (>= 6.27)
-    Microsoft.IdentityModel.Protocols.OpenIdConnect (6.27) - restriction: || (&& (== net472) (>= net6.0)) (== net6.0) (== net7.0)
-      Microsoft.IdentityModel.Protocols (>= 6.27)
-      System.IdentityModel.Tokens.Jwt (>= 6.27)
-    Microsoft.IdentityModel.Tokens (6.27) - restriction: || (&& (== net472) (>= net6.0)) (== net6.0) (== net7.0)
+    Microsoft.IdentityModel.Logging (6.28.1) - restriction: || (&& (== net472) (>= net6.0)) (== net6.0) (== net7.0)
+      Microsoft.IdentityModel.Abstractions (>= 6.28.1)
+    Microsoft.IdentityModel.Protocols (6.28.1) - restriction: || (&& (== net472) (>= net6.0)) (== net6.0) (== net7.0)
+      Microsoft.IdentityModel.Logging (>= 6.28.1)
+      Microsoft.IdentityModel.Tokens (>= 6.28.1)
+    Microsoft.IdentityModel.Protocols.OpenIdConnect (6.28.1) - restriction: || (&& (== net472) (>= net6.0)) (== net6.0) (== net7.0)
+      Microsoft.IdentityModel.Protocols (>= 6.28.1)
+      System.IdentityModel.Tokens.Jwt (>= 6.28.1)
+    Microsoft.IdentityModel.Tokens (6.28.1) - restriction: || (&& (== net472) (>= net6.0)) (== net6.0) (== net7.0)
       Microsoft.CSharp (>= 4.5) - restriction: || (&& (== net472) (< net45)) (&& (== net472) (>= net6.0)) (== net6.0) (== net7.0)
-      Microsoft.IdentityModel.Logging (>= 6.27)
+      Microsoft.IdentityModel.Logging (>= 6.28.1)
       System.Security.Cryptography.Cng (>= 4.5) - restriction: || (&& (== net472) (< net45)) (&& (== net472) (>= net6.0)) (== net6.0) (== net7.0)
     Microsoft.NETCore.Platforms (7.0.1)
     Microsoft.NETCore.Targets (5.0)
@@ -331,9 +336,9 @@ NUGET
       System.Runtime (>= 4.3) - restriction: || (&& (== net472) (< net46)) (== net6.0) (== net7.0)
       System.Runtime.Extensions (>= 4.3) - restriction: || (&& (== net472) (< net46)) (== net6.0) (== net7.0)
       System.Runtime.InteropServices (>= 4.3) - restriction: || (&& (== net472) (< net46)) (== net6.0) (== net7.0)
-    System.IdentityModel.Tokens.Jwt (6.27) - restriction: || (&& (== net472) (>= net6.0)) (== net6.0) (== net7.0)
-      Microsoft.IdentityModel.JsonWebTokens (>= 6.27)
-      Microsoft.IdentityModel.Tokens (>= 6.27)
+    System.IdentityModel.Tokens.Jwt (6.28.1) - restriction: || (&& (== net472) (>= net6.0)) (== net6.0) (== net7.0)
+      Microsoft.IdentityModel.JsonWebTokens (>= 6.28.1)
+      Microsoft.IdentityModel.Tokens (>= 6.28.1)
     System.IO (4.3)
       Microsoft.NETCore.Platforms (>= 1.1) - restriction: || (&& (== net472) (< net45)) (== net6.0) (== net7.0)
       Microsoft.NETCore.Targets (>= 1.1) - restriction: || (&& (== net472) (< net45)) (== net6.0) (== net7.0)
@@ -902,7 +907,7 @@ NUGET
     NuGet.Protocol (6.5)
       NuGet.Packaging (>= 6.5)
     NuGet.Versioning (6.5)
-    Octokit (5.0.2)
+    Octokit (5.0.3)
     System.CodeDom (7.0)
     System.Collections.Immutable (7.0)
       System.Runtime.CompilerServices.Unsafe (>= 6.0)

src/PBI-Tools.NET7/paket.references

@@ -7,6 +7,7 @@ HtmlAgilityPack
 CsvHelper
 WildcardMatch
 Microsoft.AnalysisServices.NetCore.retail.amd64
+Microsoft.AnalysisServices.Tabular.Tmdl.NetCore.retail.amd64
 Microsoft.AnalysisServices.AdomdClient.NetCore.retail.amd64
 System.IO.Packaging
 Microsoft.Identity.Client

src/PBI-Tools.NETCore/paket.references

@@ -7,6 +7,7 @@ HtmlAgilityPack
 CsvHelper
 WildcardMatch
 Microsoft.AnalysisServices.NetCore.retail.amd64
+Microsoft.AnalysisServices.Tabular.Tmdl.NetCore.retail.amd64
 Microsoft.AnalysisServices.AdomdClient.NetCore.retail.amd64
 System.IO.Packaging
 Microsoft.Identity.Client

src/PBI-Tools/Configuration/AppSettings.cs

@@ -11,6 +11,8 @@
 
 namespace PbiTools.Configuration
 {
+    using ProjectSystem;
+
     public class AppSettings
     {
         public const string EnvPrefix = "PBITOOLS_";
@@ -30,6 +32,7 @@ public static class Environment
             public static readonly string Debug = $"{EnvPrefix}{nameof(Debug)}";
             public static readonly string UICulture = $"{EnvPrefix}{nameof(UICulture)}";
             public static readonly string EffectiveDate = $"{EnvPrefix}{nameof(EffectiveDate)}";
+            public static readonly string DefaultModelSerialization = $"{EnvPrefix}{nameof(DefaultModelSerialization)}";
         }
 
         public static string GetEnvironmentSetting(string name) => System.Environment.GetEnvironmentVariable(name)
@@ -48,6 +51,12 @@ public static bool GetBooleanSetting(string name) => GetEnvironmentSetting(name)
                 _ => false
             };
 
+        public static ModelSerializationMode? DefaultModelSerializationMode => GetEnvironmentSetting(Environment.DefaultModelSerialization) switch
+        {
+            var s when Enum.TryParse<ModelSerializationMode>(s, out var result) => result,
+            _ => null
+        };
+
         public AppSettings()
         {
             // The Console log level can optionally be configured vai an environment variable:

src/PBI-Tools/FileSystem/ProjectFile.cs

@@ -41,6 +41,7 @@ public interface IProjectFile
         /// Provides a <see cref="TextWriter"/> to write the file to.
         /// </summary>
         void WriteText(Action<TextWriter> onTextWriterAvailable);
+
     }
 
 
@@ -97,6 +98,11 @@ public void WriteText(Action<TextWriter> onTextWriterAvailable)
                 callback(writer);
             }
 
+            this.MarkWritten();
+        }
+
+        internal void MarkWritten()
+        {
             _root.FileWritten(Path); // keeps track of files added or updated
         }
     }