extract AppleDict meta-info (langs, title, author) #418

soshial · 2023-01-28T12:08:00Z

Done in this PR:

Simplify logic of finding filepath of Body.data
This is data that we can extract from Info.plist file:

Oxford Dictionary of English

{
   "DCSDictionaryCSS": "DefaultStyle.css",
   "CFBundleName": "English",
   "DCSDictionaryUseSystemAppearance": true,
   "IDXDictionaryIndexes": [
      {
         "IDXIndexSupportDataID": false,
         "IDXIndexWritable": false,
         "IDXIndexPath": "KeyText.index",
         "IDXIndexAccessMethod": "com.apple.TrieAccessMethod",
         "IDXIndexName": "DCSKeywordIndex",
         "IDXIndexDataSizeLength": 2,
         "IDXIndexBigEndian": false,
         "TrieIndexCompressionType": 1,
         "IDXIndexDataFields": {
            "IDXFixedDataFields": [
               {
                  "IDXDataFieldName": "DCSPrivateFlag",
                  "IDXDataSize": 2
               }
            ],
            "IDXExternalDataFields": [
               {
                  "IDXDataFieldName": "DCSExternalBodyID",
                  "IDXDataSize": 8,
                  "IDXIndexName": "DCSBodyDataIndex"
               }
            ],
            "IDXVariableDataFields": [
               {
                  "IDXDataFieldName": "DCSKeyword",
                  "IDXDataSizeLength": 2
               },
               {
                  "IDXDataFieldName": "DCSHeadword",
                  "IDXDataSizeLength": 2
               },
               {
                  "IDXDataFieldName": "DCSEntryTitle",
                  "IDXDataSizeLength": 2
               },
               {
                  "IDXDataFieldName": "DCSAnchor",
                  "IDXDataSizeLength": 2
               },
               {
                  "IDXDataFieldName": "DCSYomiWord",
                  "IDXDataSizeLength": 2
               }
            ]
         },
         "IDXIndexKeyMatchingMethods": [
            "IDXExactMatch",
            "IDXPrefixMatch",
            "IDXCommonPrefixMatch",
            "IDXWildcardMatch",
            "IDXAllMatch"
         ],
         "TrieAuxiliaryDataOptions": {
            "HeapDataCompressionType": 3,
            "IDXIndexPath": "KeyText.data"
         }
      },
      {
         "IDXIndexSupportDataID": false,
         "IDXIndexWritable": false,
         "IDXIndexPath": "EntryID.index",
         "IDXIndexAccessMethod": "com.apple.TrieAccessMethod",
         "IDXIndexName": "DCSReferenceIndex",
         "IDXIndexDataSizeLength": 2,
         "IDXIndexBigEndian": false,
         "TrieIndexCompressionType": 1,
         "IDXIndexDataFields": {
            "IDXExternalDataFields": [
               {
                  "IDXDataFieldName": "DCSExternalBodyID",
                  "IDXDataSize": 8,
                  "IDXIndexName": "DCSBodyDataIndex"
               }
            ]
         },
         "IDXIndexKeyMatchingMethods": [
            "IDXExactMatch"
         ],
         "TrieAuxiliaryDataOptions": {
            "IDXIndexPath": "EntryID.data"
         }
      },
      {
         "IDXIndexPath": "Body.data",
         "IDXIndexAccessMethod": "com.apple.HeapAccessMethod",
         "IDXIndexName": "DCSBodyDataIndex",
         "IDXIndexBigEndian": false,
         "IDXIndexDataFields": {
            "IDXVariableDataFields": [
               {
                  "IDXDataFieldName": "DCSBodyData",
                  "IDXDataSizeLength": 4
               }
            ]
         },
         "IDXIndexWritable": false,
         "HeapDataCompressionType": 2,
         "IDXIndexSupportDataID": true
      }
   ],
   "DCSDictionaryManufacturerName": "Apple Inc.",
   "CFBundleDevelopmentRegion": "en",
   "DCSDictionaryFrontMatterReferenceID": "fbm_index",
   "CFBundleShortVersionString": "2.6",
   "DCSElementXPath": {
      "pos": "//span[@class=\"sg\"]/descendant::span[@class=\"pos\"][1]",
      "definitions": "//span[contains(@class,\"x_xd0\")][1]/descendant::span[contains(@class,\"x_xd1\") and not(contains(@class,\"t_subsense\"))]/descendant::span[@class=\"df\" or @class=\"xrg\"][1][not(@d:prtl)]",
      "pronunciation": "(//span[@class=\"pr\" or @class=\"prx\"]/descendant::span[contains(@class,\"ph\")])[1]"
   },
   "DCSDictionaryDetailedDisplayName": "British English",
   "CFBundleInfoDictionaryVersion": "6.0",
   "CFBundleIdentifier": "com.apple.dictionary.ODE",
   "DCSDictionaryPrimaryLanguage": "en_GB",
   "DCSDictionaryDefaultPrefs": {
      "version": "1",
      "pronunciation": "0"
   },
   "DCSDictionaryPreviewMarkupVersion": 1,
   "CFBundleDisplayName": "Oxford Dictionary of English",
   "DCSDictionaryCopyright": "Oxford Dictionary of English<br/>Copyright © 2010, 2022 by Oxford University Press. All rights reserved.",
   "IDXDictionaryVersion": 3,
   "DCSDictionaryNativeDisplayName": "Dictionary",
   "DCSBuildToolVersion": 3,
   "DCSDictionaryLanguages": [
      {
         "DCSDictionaryDescriptionLanguage": "en_GB",
         "DCSDictionaryIndexLanguage": "en_GB"
      }
   ]
}

Custom example (make up one's mind, make it)

{
   "DCSDictionaryCSS": "DefaultStyle.css",
   "CFBundleName": "English",
   "DCSDictionaryUseSystemAppearance": true,
   "IDXDictionaryIndexes": [
      {
         "IDXIndexSupportDataID": false,
         "IDXIndexWritable": false,
         "IDXIndexPath": "KeyText.index",
         "IDXIndexAccessMethod": "com.apple.TrieAccessMethod",
         "IDXIndexName": "DCSKeywordIndex",
         "IDXIndexDataSizeLength": 2,
         "IDXIndexBigEndian": false,
         "TrieIndexCompressionType": 1,
         "IDXIndexDataFields": {
            "IDXFixedDataFields": [
               {
                  "IDXDataFieldName": "DCSPrivateFlag",
                  "IDXDataSize": 2
               }
            ],
            "IDXExternalDataFields": [
               {
                  "IDXDataFieldName": "DCSExternalBodyID",
                  "IDXDataSize": 8,
                  "IDXIndexName": "DCSBodyDataIndex"
               }
            ],
            "IDXVariableDataFields": [
               {
                  "IDXDataFieldName": "DCSKeyword",
                  "IDXDataSizeLength": 2
               },
               {
                  "IDXDataFieldName": "DCSHeadword",
                  "IDXDataSizeLength": 2
               },
               {
                  "IDXDataFieldName": "DCSEntryTitle",
                  "IDXDataSizeLength": 2
               },
               {
                  "IDXDataFieldName": "DCSAnchor",
                  "IDXDataSizeLength": 2
               },
               {
                  "IDXDataFieldName": "DCSYomiWord",
                  "IDXDataSizeLength": 2
               }
            ]
         },
         "IDXIndexKeyMatchingMethods": [
            "IDXExactMatch",
            "IDXPrefixMatch",
            "IDXCommonPrefixMatch",
            "IDXWildcardMatch",
            "IDXAllMatch"
         ],
         "TrieAuxiliaryDataOptions": {
            "HeapDataCompressionType": 3,
            "IDXIndexPath": "KeyText.data"
         }
      },
      {
         "IDXIndexSupportDataID": false,
         "IDXIndexWritable": false,
         "IDXIndexPath": "EntryID.index",
         "IDXIndexAccessMethod": "com.apple.TrieAccessMethod",
         "IDXIndexName": "DCSReferenceIndex",
         "IDXIndexDataSizeLength": 2,
         "IDXIndexBigEndian": false,
         "TrieIndexCompressionType": 1,
         "IDXIndexDataFields": {
            "IDXExternalDataFields": [
               {
                  "IDXDataFieldName": "DCSExternalBodyID",
                  "IDXDataSize": 8,
                  "IDXIndexName": "DCSBodyDataIndex"
               }
            ]
         },
         "IDXIndexKeyMatchingMethods": [
            "IDXExactMatch"
         ],
         "TrieAuxiliaryDataOptions": {
            "IDXIndexPath": "EntryID.data"
         }
      },
      {
         "IDXIndexPath": "Body.data",
         "IDXIndexAccessMethod": "com.apple.HeapAccessMethod",
         "IDXIndexName": "DCSBodyDataIndex",
         "IDXIndexBigEndian": false,
         "IDXIndexDataFields": {
            "IDXVariableDataFields": [
               {
                  "IDXDataFieldName": "DCSBodyData",
                  "IDXDataSizeLength": 4
               }
            ]
         },
         "IDXIndexWritable": false,
         "HeapDataCompressionType": 2,
         "IDXIndexSupportDataID": true
      }
   ],
   "DCSDictionaryManufacturerName": "Apple Inc.",
   "CFBundleDevelopmentRegion": "en",
   "DCSDictionaryFrontMatterReferenceID": "fbm_index",
   "CFBundleShortVersionString": "2.6",
   "DCSElementXPath": {
      "pos": "//span[@class=\"sg\"]/descendant::span[@class=\"pos\"][1]",
      "definitions": "//span[contains(@class,\"x_xd0\")][1]/descendant::span[contains(@class,\"x_xd1\") and not(contains(@class,\"t_subsense\"))]/descendant::span[@class=\"df\" or @class=\"xrg\"][1][not(@d:prtl)]",
      "pronunciation": "(//span[@class=\"pr\" or @class=\"prx\"]/descendant::span[contains(@class,\"ph\")])[1]"
   },
   "DCSDictionaryDetailedDisplayName": "British English",
   "CFBundleInfoDictionaryVersion": "6.0",
   "CFBundleIdentifier": "com.apple.dictionary.ODE",
   "DCSDictionaryPrimaryLanguage": "en_GB",
   "DCSDictionaryDefaultPrefs": {
      "version": "1",
      "pronunciation": "0"
   },
   "DCSDictionaryPreviewMarkupVersion": 1,
   "CFBundleDisplayName": "Oxford Dictionary of English",
   "DCSDictionaryCopyright": "Oxford Dictionary of English<br/>Copyright © 2010, 2022 by Oxford University Press. All rights reserved.",
   "IDXDictionaryVersion": 3,
   "DCSDictionaryNativeDisplayName": "Dictionary",
   "DCSBuildToolVersion": 3,
   "DCSDictionaryLanguages": [
      {
         "DCSDictionaryDescriptionLanguage": "en_GB",
         "DCSDictionaryIndexLanguage": "en_GB"
      }
   ]
}

Custom PL-RU

{
   "CFBundleDevelopmentRegion": "zh-Hans",
   "CFBundleDisplayName": "Wielki Słownik Polsko-Rosyjski",
   "CFBundleIdentifier": "com.apple.dictionary.pol-rus",
   "CFBundleInfoDictionaryVersion": "6.0",
   "CFBundleName": "pol-rus",
   "CFBundleShortVersionString": "1.0",
   "DCSBuildToolVersion": 1,
   "DCSDictionaryCSS": "DefaultStyle.css",
   "DCSDictionaryCopyright": "Ver. 01d, based on Wielki Słownik Pol-Ros (Wiedza Powszechna) / coded and updated by Berk Bear",
   "DCSDictionaryManufacturerName": "Berk Bear",
   "IDXDictionaryIndexes": [
      {
         "IDXIndexAccessMethod": "com.apple.TrieAccessMethod",
         "IDXIndexBigEndian": false,
         "IDXIndexDataFields": {
            "IDXExternalDataFields": [
               {
                  "IDXDataFieldName": "DCSExternalBodyID",
                  "IDXDataSize": 4,
                  "IDXIndexName": "DCSBodyDataIndex"
               }
            ],
            "IDXFixedDataFields": [
               {
                  "IDXDataFieldName": "DCSPrivateFlag",
                  "IDXDataSize": 2
               }
            ],
            "IDXVariableDataFields": [
               {
                  "IDXDataFieldName": "DCSKeyword",
                  "IDXDataSizeLength": 2
               },
               {
                  "IDXDataFieldName": "DCSHeadword",
                  "IDXDataSizeLength": 2
               },
               {
                  "IDXDataFieldName": "DCSAnchor",
                  "IDXDataSizeLength": 2
               },
               {
                  "IDXDataFieldName": "DCSYomiWord",
                  "IDXDataSizeLength": 2
               }
            ]
         },
         "IDXIndexDataSizeLength": 2,
         "IDXIndexKeyMatchingMethods": [
            "IDXExactMatch",
            "IDXPrefixMatch",
            "IDXCommonPrefixMatch",
            "IDXWildcardMatch"
         ],
         "IDXIndexName": "DCSKeywordIndex",
         "IDXIndexPath": "KeyText.index",
         "IDXIndexSupportDataID": false,
         "IDXIndexWritable": false,
         "TrieAuxiliaryDataFile": "KeyText.data"
      },
      {
         "IDXIndexAccessMethod": "com.apple.TrieAccessMethod",
         "IDXIndexBigEndian": false,
         "IDXIndexDataFields": {
            "IDXExternalDataFields": [
               {
                  "IDXDataFieldName": "DCSExternalBodyID",
                  "IDXDataSize": 4,
                  "IDXIndexName": "DCSBodyDataIndex"
               }
            ]
         },
         "IDXIndexDataSizeLength": 2,
         "IDXIndexKeyMatchingMethods": [
            "IDXExactMatch"
         ],
         "IDXIndexName": "DCSReferenceIndex",
         "IDXIndexPath": "EntryID.index",
         "IDXIndexSupportDataID": false,
         "IDXIndexWritable": false,
         "TrieAuxiliaryDataFile": "EntryID.data"
      },
      {
         "HeapDataCompressionType": 1,
         "IDXIndexAccessMethod": "com.apple.HeapAccessMethod",
         "IDXIndexBigEndian": false,
         "IDXIndexDataFields": {
            "IDXVariableDataFields": [
               {
                  "IDXDataFieldName": "DCSBodyData",
                  "IDXDataSizeLength": 4
               }
            ]
         },
         "IDXIndexName": "DCSBodyDataIndex",
         "IDXIndexPath": "Body.data",
         "IDXIndexSupportDataID": true,
         "IDXIndexWritable": false
      }
   ],
   "IDXDictionaryVersion": 1
}

pyglossary/plugins/appledict/__init__.py

pyglossary/plugins/appledict_bin.py

ilius · 2023-01-28T14:55:58Z

Please run ./scripts/gen.sh and add the changes.

Have you tested it on several glossaries?

That's all.
Thank you.

soshial · 2023-01-28T15:20:01Z

I have several dicts that come bundled with macOS and also a couple of custom dicts (generated by DictionaryKit). And those seem to work okay.
I have some plans what to improve in the code of parsing binary AppleDict... I wish we had tests with files for binary AppleDict to check that my changes don't break anything.

soshial · 2023-01-28T15:42:46Z

I would also like to check out files of the exact dictionaries that caused adding

these lines (it's a bit hard to imagine that some dictionary might have chunk length encoded in less than 4 bytes)

pyglossary/pyglossary/plugins/appledict_bin.py

Lines 170 to 171 in 25473c0

if plus < 4:

bs = b"\x00" * (4 - plus) + bs

and these lines (why do we need to search for </d:entry> closing tag when we have it's length)

pyglossary/pyglossary/plugins/appledict_bin.py

Lines 210 to 215 in 25473c0

    
           if chunkSize == 0: 
        
           	endI = self._buf[pos:].find(b"</d:entry>") 
        
           	if endI == -1: 
        
           		chunkSize = len(self._buf) - pos 
        
           	else: 
        
           		chunkSize = endI + 10

ilius · 2023-01-28T16:02:40Z

Let's keep those out of this PR.

The problem with automated testing is that we need small glossaries. If you can create or find small glossaries for testing, open a new issue and we can work it out.

soshial · 2023-01-28T16:31:32Z

Please run ./scripts/gen.sh and add the changes.

What will this script do? I am a bit cautious.

The problem with automated testing is that we need small glossaries. If you can create or find small glossaries for testing, open a new issue and we can work it out.

I can send you some example dictionary files, but not many. Should I create a separate issue for that? Or maybe I may send to you via telegram?

ilius · 2023-01-28T17:18:11Z

Just run it and you'll see!

Open a new issue (list your files and sizes) and I will explain.

ilius · 2023-01-28T17:35:26Z

gen.sh updates documents and index.json when options or dependencies are changed.

soshial · 2023-01-28T19:51:54Z

Hmm, we could have squashed these commits into one. Commit history would be cleaner.

soshial · 2023-01-30T17:12:43Z

Thank you very much, @ilius, for fixing some issues that my code caused.

soshial added 2 commits January 28, 2023 14:06

extract AppleDict meta-info (langs, title, author)

16becbe

support plain-text AppleDict meta-info

7b4f305

ilius reviewed Jan 28, 2023

View reviewed changes

pyglossary/plugins/appledict/__init__.py Outdated Show resolved Hide resolved

pyglossary/plugins/appledict_bin.py Outdated Show resolved Hide resolved

ilius reviewed Jan 28, 2023

View reviewed changes

pyglossary/plugins/appledict_bin.py Outdated Show resolved Hide resolved

soshial added 2 commits January 28, 2023 19:47

AppleDict meta-info: PR comments

835daab

run bash script

5e821f7

ilius approved these changes Jan 28, 2023

View reviewed changes

ilius merged commit 13e8978 into ilius:master Jan 28, 2023

ilius added a commit that referenced this pull request Jan 29, 2023

AppleDictBin: fix regression in Reader.open, #418

e2dde37

ilius added a commit that referenced this pull request Jan 29, 2023

AppleDictBin: refactoring and style changes #418

443a37b

soshial deleted the appledict-meta-info branch February 16, 2023 12:18

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

extract AppleDict meta-info (langs, title, author) #418

extract AppleDict meta-info (langs, title, author) #418

soshial commented Jan 28, 2023 •

edited

Loading

ilius commented Jan 28, 2023

soshial commented Jan 28, 2023

soshial commented Jan 28, 2023 •

edited

Loading

ilius commented Jan 28, 2023

soshial commented Jan 28, 2023 •

edited

Loading

ilius commented Jan 28, 2023

ilius commented Jan 28, 2023

soshial commented Jan 28, 2023

soshial commented Jan 30, 2023

extract AppleDict meta-info (langs, title, author) #418

extract AppleDict meta-info (langs, title, author) #418

Conversation

soshial commented Jan 28, 2023 • edited Loading

Oxford Dictionary of English

Custom example (make up one's mind, make it)

Custom PL-RU

ilius commented Jan 28, 2023

soshial commented Jan 28, 2023

soshial commented Jan 28, 2023 • edited Loading

ilius commented Jan 28, 2023

soshial commented Jan 28, 2023 • edited Loading

ilius commented Jan 28, 2023

ilius commented Jan 28, 2023

soshial commented Jan 28, 2023

soshial commented Jan 30, 2023

soshial commented Jan 28, 2023 •

edited

Loading

soshial commented Jan 28, 2023 •

edited

Loading

soshial commented Jan 28, 2023 •

edited

Loading