- Philips: added support for CM\_*.xml variant that uses a <ECSM> roo…

…t element around the <ChannelMap> - Philips: ability to read/write broken CM\_*.xml files that contain channel names with an unescaped & character - Philips: enabled write mode for Repair\\FLASH\_\*/\*.db file format (one variant was confirmed to work) Favorite lists for this format are disabled for now (TV ignored them). - Panasonic: importing a modified svl.bin file caused the TV to use case-sensitive sorting when using the function to list the names sorted alphabetically. This is now fixed.
PredatH0r · Sep 22, 2021 · 063ed16 · 063ed16
1 parent 73d8f0c
commit 063ed16
Show file tree

Hide file tree

Showing 9 changed files with 4,231 additions and 31 deletions.
diff --git a/source/ChanSort.Loader.Panasonic/DbChannel.cs b/source/ChanSort.Loader.Panasonic/DbChannel.cs
@@ -133,9 +133,6 @@ private void ReadNamesWithEncodingDetection(IDataReader r, IDictionary<string, i
     {
       byte[] buffer = new byte[100];
       int len = (int)r.GetBytes(field["sname"], 0, buffer, 0, buffer.Length);
-      int end = Array.IndexOf<byte>(buffer, 0, 0, len);
-      if (end >= 0)
-        len = end;
       this.RawName = new byte[len];
       Array.Copy(buffer, 0, this.RawName, 0, len);
       this.ChangeEncoding(encoding);      
@@ -152,22 +149,25 @@ public override void ChangeEncoding(Encoding encoding)
       // it can be code page encoded without any clue to what the code page is
       // it can have DVB-control characters inside an UTF-8 stream
 
-      if (RawName.Length == 0)
+      int len = Array.IndexOf<byte>(this.RawName, 0, 0, this.RawName.Length);
+      if (len < 0)
+        len = this.RawName.Length;
+      if (len == 0)
         return;
 
-      int startOffset;
-      int bytesPerChar;
-      if (!GetRecommendedEncoding(ref encoding, out startOffset, out bytesPerChar)) 
+      if (!GetRecommendedEncoding(ref encoding, out var startOffset, out var bytesPerChar)) 
         return;
 
       // single byte code pages might have UTF-8 code mixed in, so we have to parse it manually
       StringBuilder sb = new StringBuilder();
       this.NonAscii = false;
       this.ValidUtf8 = true;
-      for (int i = startOffset; i < this.RawName.Length; i+=bytesPerChar)
+      for (int i = startOffset; i < len; i+=bytesPerChar)
       {
         byte c = this.RawName[i];
-        byte c2 = i + 1 < this.RawName.Length ? this.RawName[i + 1] : (byte)0;
+        byte c2 = i + 1 < len ? this.RawName[i + 1] : (byte)0;
+        byte c3 = i + 2 < len ? this.RawName[i + 2] : (byte)0;
+        byte c4 = i + 4 < len ? this.RawName[i + 3] : (byte)0;
         if (c >= 0x80)
           NonAscii = true;
 
@@ -178,10 +178,28 @@ public override void ChangeEncoding(Encoding encoding)
           ValidUtf8 = false;
           sb.Append((char) c);
         }
-        else if (bytesPerChar == 1 && c >= 0xC0 && c <= 0xDF && c2 >= 0x80 && c2 <= 0xBF) // 2 byte UTF-8
+        else if (bytesPerChar == 1)
         {
-          sb.Append((char)(((c & 0x1F) << 6) | (c2 & 0x3F)));
-          ++i;
+          if (c >= 0xC0 && c <= 0xDF && c2 >= 0x80 && c2 <= 0xBF) // 2 byte UTF-8
+          {
+            sb.Append((char)(((c & 0x1F) << 6) | (c2 & 0x3F)));
+            ++i;
+          }
+          else if (c >= 0xE0 && c <= 0xEF && (c2 & 0xC0) == 0x80 && (c3 & 0xC0) == 0x80) // 3 byte UTF-8
+          {
+            sb.Append((char)(((c & 0x0F) << 12) | ((c2 & 0x3F) << 6) | (c3 & 0x3F)));
+            i += 2;
+          }
+          else if (c >= 0xF0 && c <= 0xF7 && (c2 & 0xC0) == 0x80 && (c3 & 0xC0) == 0x80 && (c4 & 0xC0) == 0x80) // 4 byte UTF-8
+          {
+            sb.Append((char)(((c & 0x07) << 18) | ((c2 & 0x3F) << 12) | ((c3 & 0x3F) << 6) | (c4 & 0x3F)));
+            i += 3;
+          }
+          else
+          {
+            ValidUtf8 = false;
+            sb.Append(encoding.GetString(this.RawName, i, bytesPerChar));
+          }
         }
         else
         {
@@ -190,8 +208,7 @@ public override void ChangeEncoding(Encoding encoding)
         }
       }
 
-      string longName, shortName;
-      this.GetChannelNames(sb.ToString(), out longName, out shortName);
+      this.GetChannelNames(sb.ToString(), out var longName, out var shortName);
       this.Name = longName;
       this.ShortName = shortName;
     }

diff --git a/source/ChanSort.Loader.Panasonic/Serializer.cs b/source/ChanSort.Loader.Panasonic/Serializer.cs
@@ -20,7 +20,7 @@ class Serializer : SerializerBase
 
     private string workFile;
     private CypherMode cypherMode;
-    private byte[] fileHeader = new byte[0];
+    private byte[] fileHeader = Array.Empty<byte>();
     private int dbSizeOffset;
     private bool littleEndianByteOrder;
     private string charEncoding;
@@ -329,11 +329,14 @@ public override void Save(string tvOutputFile)
     #region WriteChannels()
     private void WriteChannels(SqliteCommand cmd, ChannelList channelList)
     {
+      if (channelList.Channels.Count == 0)
+        return;
+
       cmd.CommandText = "update SVL set major_channel=@progNr, sname=@sname, profile1index=@fav1, profile2index=@fav2, profile3index=@fav3, profile4index=@fav4, child_lock=@lock, skip=@skip where rowid=@rowid";
       cmd.Parameters.Clear();
       cmd.Parameters.Add("@rowid", SqliteType.Integer);
       cmd.Parameters.Add("@progNr", SqliteType.Integer);
-      cmd.Parameters.Add("@sname", SqliteType.Blob);
+      cmd.Parameters.Add("@sname", this.implicitUtf8 ? SqliteType.Text : SqliteType.Blob); // must use "TEXT" when possible to preserve collation / case-insensitive sorting for the TV
       cmd.Parameters.Add("@fav1", SqliteType.Integer);
       cmd.Parameters.Add("@fav2", SqliteType.Integer);
       cmd.Parameters.Add("@fav3", SqliteType.Integer);
@@ -351,7 +354,7 @@ private void WriteChannels(SqliteCommand cmd, ChannelList channelList)
         channel.UpdateRawData(this.explicitUtf8, this.implicitUtf8);
         cmd.Parameters["@rowid"].Value = channel.RecordIndex;
         cmd.Parameters["@progNr"].Value = channel.NewProgramNr;
-        cmd.Parameters["@sname"].Value = channel.RawName;
+        cmd.Parameters["@sname"].Value = this.implicitUtf8 ? channel.Name : channel.RawName; // must use a string when possible to preserve collation / case-insensitive sorting for the TV
         for (int fav = 0; fav < 4; fav++)
           cmd.Parameters["@fav" + (fav + 1)].Value = Math.Max(0, channel.GetPosition(fav+1));
         cmd.Parameters["@lock"].Value = channel.Lock;

diff --git a/source/ChanSort.Loader.Philips/ChanSort.Loader.Philips.ini b/source/ChanSort.Loader.Philips/ChanSort.Loader.Philips.ini
@@ -19,7 +19,7 @@
 
 [flash_db]
 reorderRecordsByChannelNumber=true
-allowEdit=false
+allowEdit=true
 
 [mgr_chan_s_fta.db]
 lenHeader=64

diff --git a/source/ChanSort.Loader.Philips/DbSerializer.cs b/source/ChanSort.Loader.Philips/DbSerializer.cs
@@ -50,8 +50,8 @@ class DbSerializer : SerializerBase
     #region ctor()
     public DbSerializer(string inputFile) : base(inputFile)
     {
-      this.Features.MaxFavoriteLists = 1;
-      this.Features.FavoritesMode = FavoritesMode.OrderedPerSource;
+      this.Features.MaxFavoriteLists = 0; //1;
+      this.Features.FavoritesMode = FavoritesMode.None; // FavoritesMode.OrderedPerSource; // doesn't work yet, must be hidden somewhere inside the FLASH files too
       this.Features.DeleteMode = DeleteMode.NotSupported;
       this.Features.CanHaveGaps = true; // the mgr_chan_s_pkg can have gaps
 

diff --git a/source/ChanSort.Loader.Philips/XmlSerializer.cs b/source/ChanSort.Loader.Philips/XmlSerializer.cs
@@ -20,12 +20,19 @@ namespace ChanSort.Loader.Philips
     The .BIN file itself is compressed with some unknown cl_Zip compression / archive format and can't be edited with ChanSort.
     Deleting a channel is not possible by modifiying the .xml file. Omitting a channel only results in duplicate numbers with the TV still showing the missing channels at their old numbers.
     The channel nodes in the .XML must be kept in the original order with "oldpresetnumber" keeping the original value and only "presetnumber" being updated.
+    There are (at least) two versions of this format. One starts with <ChannelMap> as the root node, one has a <ECSM> root node wrapping <ChannelMap> and other elements.
 
     <Channel>
     <Setup oldpresetnumber="1" presetnumber="1" name="Das Erste" ></Setup>
     <Broadcast medium="dvbc" frequency="410000" system="west" serviceID="1" ONID="41985" TSID="1101" modulation="256" symbolrate="6901000" bandwidth="Unknown"></Broadcast>
     </Channel>
   
+    <Channel>
+    <Setup presetnumber="1" name="Das Erste HD" ></Setup>
+    <Broadcast medium="dvbs" satellitename="Astra1F-1L 19.2E
+    @" frequency="11494" system="west" serviceID="2604" ONID="1" TSID="1019" modulation="8-VSB" symbolrate="22000"></Broadcast>
+    </Channel>
+
   
     Newer channel lists from Philips contain multiple XML files with a different internal structure, which also varies based on the version number in the ChannelMap_xxx folder name.
     The official Philips Channel Editor 6.61.22 supports the binary file format 1.1 and 1.2 as well as the XML file format "ChannelMap_100" (but not 45, 105 nor 110).
@@ -238,7 +245,8 @@ private void LoadFile(string fileName)
         fileData.doc = new XmlDocument();
         fileData.content = File.ReadAllBytes(fileName);
         fileData.textContent = Encoding.UTF8.GetString(fileData.content);
-        fileData.newline = fileData.textContent.Contains("\r\n") ? "\r\n" : "\n";
+        var idx = fileData.textContent.IndexOf('\n');
+        fileData.newline = idx < 0 ? "" : idx > 0 && fileData.textContent[idx-1] == '\r' ? "\r\n" : "\n"; // there are Repair\*.xml files with <ECSM> root that use \n normally but contain a \n\r\n near the end
 
         // indentation can be 0, 2 or 4 spaces
         var idx1 = fileData.textContent.IndexOf("<Channel>");
@@ -255,7 +263,13 @@ private void LoadFile(string fileName)
           ValidationFlags = XmlSchemaValidationFlags.None,
           DtdProcessing = DtdProcessing.Ignore
         };
-        using (var reader = XmlReader.Create(new StringReader(fileData.textContent), settings))
+
+        fileData.formatVersion = Path.GetFileName(fileName).ToLowerInvariant().StartsWith("cm_") ? FormatVersion.RepairXml : FormatVersion.ChannelMapXml; // first guess, will be set based on file content later
+
+        var xml = fileData.textContent;
+        if (fileData.formatVersion == FormatVersion.RepairXml)
+          xml = xml.Replace("&", "&amp;"); // Philips exports broken XML with unescaped & instead of &amp;
+        using (var reader = XmlReader.Create(new StringReader(xml), settings))
         {
           fileData.doc.Load(reader);
         }
@@ -268,6 +282,8 @@ private void LoadFile(string fileName)
       var root = fileData.doc.FirstChild;
       if (root is XmlDeclaration)
         root = root.NextSibling;
+      if (root?.LocalName == "ECSM")
+        root = root.FirstChild;
       if (fail || root == null || (root.LocalName != "ChannelMap" && root.LocalName != "FavoriteListMAP"))
         throw new FileLoadException("\"" + fileName + "\" is not a supported Philips XML file");
 
@@ -313,21 +329,22 @@ private ChannelList DetectFormatAndFeatures(FileData file, XmlNode node)
 
       if (setupNode.HasAttribute("name"))
       {
-        file.formatVersion = 1;
+        file.formatVersion = FormatVersion.RepairXml;
         this.iniMapSection = ini.GetSection("Repair_xml");
         this.Features.FavoritesMode = FavoritesMode.None;
         foreach (var list in this.DataRoot.ChannelLists)
         {
           list.VisibleColumnFieldNames.Remove(nameof(ChannelInfo.Favorites));
           list.VisibleColumnFieldNames.Remove(nameof(ChannelInfo.Lock));
           list.VisibleColumnFieldNames.Remove(nameof(ChannelInfo.Hidden));
-          list.VisibleColumnFieldNames.Remove(nameof(ChannelInfo.ServiceTypeName));
+          list.VisibleColumnFieldNames.Remove(nameof(ChannelInfo.ServiceType));
+          list.VisibleColumnFieldNames.Add("-" + nameof(ChannelInfo.ServiceTypeName));
           list.VisibleColumnFieldNames.Remove(nameof(ChannelInfo.Encrypted));
         }
       }
       else if (setupNode.HasAttribute("ChannelName"))
       {
-        file.formatVersion = 2;
+        file.formatVersion = FormatVersion.ChannelMapXml;
         this.Features.FavoritesMode = FavoritesMode.Flags;
         this.Features.MaxFavoriteLists = 1;
 
@@ -417,9 +434,9 @@ private void ReadChannel(FileData file, ChannelList curList, XmlNode node, int r
       var chan = new Channel(curList.SignalSource & SignalSource.MaskAdInput, rowId, uniqueId, setupNode);
       chan.OldProgramNr = -1;
       chan.IsDeleted = false;
-      if (file.formatVersion == 1)
+      if (file.formatVersion == FormatVersion.RepairXml)
         this.ParseRepairXml(data, chan);
-      else if (file.formatVersion == 2)
+      else if (file.formatVersion == FormatVersion.ChannelMapXml)
         this.ParseChannelMapXml(data, chan);
 
       if ((chan.SignalSource & SignalSource.MaskAdInput) == SignalSource.DvbT)
@@ -447,7 +464,10 @@ private void ParseRepairXml(Dictionary<string, string> data, Channel chan)
       chan.OriginalNetworkId = ParseInt(data.TryGet("ONID"));
       chan.TransportStreamId = ParseInt(data.TryGet("TSID"));
       chan.ServiceType = ParseInt(data.TryGet("serviceType"));
-      chan.SymbolRate = ParseInt(data.TryGet("symbolrate")) / 1000;
+      chan.SymbolRate = ParseInt(data.TryGet("symbolrate"));
+      if (chan.SymbolRate > 100000) // DVB-C/T specify it in Sym/s, DVB-S in kSym/sec
+        chan.SymbolRate /= 1000;
+      chan.Satellite = data.TryGet("satellitename")?.TrimEnd('@', '\n', '\r'); // the satellitename can have a "\n@" at the end
     }
     #endregion
 
@@ -583,7 +603,7 @@ public override void Save(string tvOutputFile)
 
       foreach (var file in this.fileDataList)
       {
-        if (reorderNodes && (file.formatVersion == 1 || Path.GetFileName(file.path).ToLowerInvariant().StartsWith("dvb")))
+        if (reorderNodes && (file.formatVersion == FormatVersion.RepairXml || Path.GetFileName(file.path).ToLowerInvariant().StartsWith("dvb")))
           this.ReorderNodes(file);
         var satelliteListcopy = this.iniMapSection?.GetString("satelliteListcopy") ?? "";
         var nodeList = file.doc.GetElementsByTagName("SatelliteListcopy");
@@ -765,7 +785,7 @@ private string EncodeName(string name, int maxBytes, bool padBytes, bool upperCa
     #region ReorderNodes()
     private void ReorderNodes(FileData file)
     {
-      var progNrAttrib = file.formatVersion == 1 ? "presetnumber" : "ChannelNumber";
+      var progNrAttrib = file.formatVersion == FormatVersion.RepairXml ? "presetnumber" : "ChannelNumber";
 
       var nodes = file.doc.DocumentElement.GetElementsByTagName("Channel");
       var list = new List<XmlElement>();
@@ -810,6 +830,8 @@ private void SaveFile(FileData file)
       // append trailing newline, if the original file had one
       if (file.textContent.EndsWith(file.newline) && !xml.EndsWith(file.newline))
         xml += file.newline;
+      if (file.formatVersion == FormatVersion.RepairXml)
+        xml = xml.Replace("&amp;", "&"); // Philips uses broken XML with unescaped & instead of &amp;
 
       var enc = new UTF8Encoding(false, false);
       File.WriteAllText(file.path, xml, enc);
@@ -824,6 +846,14 @@ public override string GetFileInformation()
     #endregion
 
 
+    #region enum FormatVersion
+
+    private enum FormatVersion
+    {
+      RepairXml = 1, ChannelMapXml = 2
+    }
+    #endregion
+
     #region class FileData
     private class FileData
     {
@@ -833,7 +863,7 @@ private class FileData
       public string textContent;
       public string newline;
       public string indent;
-      public int formatVersion;
+      public FormatVersion formatVersion;
     }
     #endregion
   }

diff --git a/source/Information/FileStructures_for_HHD_Hex_Editor_Neo/philips_mgr_476.h b/source/Information/FileStructures_for_HHD_Hex_Editor_Neo/philips_mgr_476.h
@@ -216,4 +216,37 @@ public struct Philips_FLASH_DTVINFO_S_PKG
     //s_unknown unknownTable[numUnknown];
 	BYTE filler[0x10000-current_offset];
     s_channelBlock channelBlocks[*];
+};
+
+#pragma byte_order(BigEndian)		
+
+public struct Philips_FLASH_SDTSECTS_S_PKG
+{
+	BYTE u1[4];
+	BYTE allocationBitmap[64];
+	struct
+	{
+		var off0 = current_offset;
+		DWORD ffff;
+		BYTE u1[3];
+		WORD tsid;
+		BYTE u2[6];
+		struct Channel
+		{
+			WORD serviceId;
+			BYTE u1[6];
+			BYTE lenProviderName;
+			char provider[lenProviderName];
+			BYTE lenChannelName;
+			char channelName[lenChannelName];		
+		};		
+		Channel chan0;
+		BYTE u3[51];		
+		Channel chan1;
+		BYTE u4[60];
+		Channel chan2;
+		BYTE u5[18];
+		Channel chans[16];
+		BYTE filler[0x44C - current_offset];
+	} transponder[*];
 };
diff --git a/source/Test.Loader.Philips/Test.Loader.Philips.csproj b/source/Test.Loader.Philips/Test.Loader.Philips.csproj
@@ -260,6 +260,7 @@
     <Content Include="TestFiles\ChannelMap_100\ChannelList\s2channellib\DVBSall.xml">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
+    <Content Include="TestFiles\Repair\CM_T923E_LA_CK.xml" />
   </ItemGroup>
   <ItemGroup>
     <None Include="app.config" />