[meta] improve startup time

[jyn514]

Partially addressed in ea2b5a7.

The main offenders are keyring and urllib3, which coincidentally are doing of all the hard work.

Times are generated with python -X importtime $(which grepme).

Import times

(bash) ~/.../python/grepme ▶️ python -X importtime /home/joshua/.local/bin/grepme
import time: self [us] | cumulative | imported package
import time:       161 |        161 | zipimport
import time:      3324 |       3324 | _frozen_importlib_external
import time:       111 |        111 |     _codecs
import time:       745 |        855 |   codecs
import time:       737 |        737 |   encodings.aliases
import time:      1081 |       2673 | encodings
import time:       383 |        383 | encodings.utf_8
import time:       232 |        232 | _signal
import time:       537 |        537 | encodings.latin_1
import time:        92 |         92 |     _abc
import time:       480 |        571 |   abc
import time:       561 |       1132 | io
import time:       236 |        236 |       _stat
import time:       330 |        565 |     stat
import time:       250 |        250 |       genericpath
import time:       392 |        641 |     posixpath
import time:      2074 |       2074 |     _collections_abc
import time:       955 |       4235 |   os
import time:       378 |        378 |   _sitebuiltins
import time:       141 |        141 |     _locale
import time:       268 |        409 |   _bootlocale
import time:       644 |        644 |   types
import time:       426 |        426 |       warnings
import time:       326 |        752 |     importlib
import time:       234 |        234 |       importlib.machinery
import time:       773 |       1007 |     importlib.abc
import time:       216 |        216 |           _operator
import time:      1556 |       1772 |         operator
import time:       271 |        271 |         keyword
import time:        86 |         86 |           _heapq
import time:       279 |        364 |         heapq
import time:       188 |        188 |         itertools
import time:       311 |        311 |         reprlib
import time:       130 |        130 |         _collections
import time:      1310 |       4342 |       collections
import time:       109 |        109 |         _functools
import time:       859 |        967 |       functools
import time:      1014 |       6322 |     contextlib
import time:       591 |       8670 |   importlib.util
import time:       331 |        331 |   sitecustomize
import time:       128 |        128 |   usercustomize
import time:      4039 |      18831 | site
import time:      1121 |       1121 |   enum
import time:       131 |        131 |     _sre
import time:       533 |        533 |       sre_constants
import time:       787 |       1319 |     sre_parse
import time:       522 |       1972 |   sre_compile
import time:       318 |        318 |   copyreg
import time:      1353 |       4763 | re
import time:       346 |        346 |       __future__
import time:       439 |        439 |         termios
import time:       444 |        883 |       getpass
import time:       303 |        303 |             collections.abc
import time:      4173 |       4476 |           configparser
import time:       233 |        233 |             time
import time:       301 |        301 |                   token
import time:      1495 |       1796 |                 tokenize
import time:       285 |       2080 |               linecache
import time:       641 |       2720 |             traceback
import time:       516 |        516 |               _weakrefset
import time:       758 |       1273 |             weakref
import time:        67 |         67 |               _string
import time:      1441 |       1507 |             string
import time:      1198 |       1198 |             threading
import time:       115 |        115 |             atexit
import time:      1967 |       9012 |           logging
import time:       458 |        458 |                 fnmatch
import time:       706 |       1163 |               glob
import time:       167 |        167 |                   errno
import time:       149 |        149 |                   zlib
import time:       445 |        445 |                     _compression
import time:       430 |        430 |                     _bz2
import time:       677 |       1551 |                   bz2
import time:       502 |        502 |                     _lzma
import time:       525 |       1026 |                   lzma
import time:       116 |        116 |                   pwd
import time:        76 |         76 |                   grp
import time:      1157 |       4240 |                 shutil
import time:       144 |        144 |                   _struct
import time:       337 |        480 |                 struct
import time:       131 |        131 |                 binascii
import time:      1670 |       6520 |               zipfile
import time:      1686 |       9369 |             entrypoints
import time:       410 |        410 |             keyring.credentials
import time:       407 |        407 |             keyring.errors
import time:       277 |        277 |             keyring.util
import time:       443 |        443 |             keyring.util.properties
import time:       899 |      11801 |           keyring.backend
import time:      1884 |       1884 |                 signal
import time:       136 |        136 |                 _posixsubprocess
import time:       108 |        108 |                 select
import time:       128 |        128 |                   math
import time:      1022 |       1150 |                 selectors
import time:      2249 |       5525 |               subprocess
import time:      6304 |      11829 |             platform
import time:     17043 |      28871 |           keyring.util.platform_
import time:       460 |        460 |           keyring.backends
import time:       523 |        523 |           keyring.backends.fail
import time:       538 |        538 |             dbus._compat
import time:      1537 |       1537 |             dbus.exceptions
import time:       230 |        230 |               _dbus_bindings
import time:       802 |       1032 |             dbus.types
import time:      2987 |       6092 |           dbus
import time:       420 |        420 |                   jeepney.auth
import time:      1419 |       1419 |                   jeepney.low_level
import time:       600 |        600 |                   jeepney.bus
import time:      3059 |       3059 |                     typing
import time:       657 |       3715 |                   jeepney.wrappers
import time:       512 |       6663 |                 jeepney
import time:       307 |       6970 |               jeepney.integrate
import time:       385 |        385 |                 _socket
import time:      3660 |       4045 |               socket
import time:       364 |        364 |               jeepney.routing
import time:       480 |        480 |               jeepney.bus_messages
import time:       745 |      12601 |             jeepney.integrate.blocking
import time:       234 |        234 |               secretstorage.defines
import time:      2363 |       2363 |                   _hashlib
import time:       160 |        160 |                     _blake2
import time:       154 |        154 |                     _sha3
import time:       554 |        867 |                   hashlib
import time:       503 |       3732 |                 hmac
import time:       340 |        340 |                     cryptography.__about__
import time:       414 |        753 |                   cryptography
import time:       437 |        437 |                         _opcode
import time:       556 |        992 |                       opcode
import time:       909 |       1901 |                     dis
import time:      3075 |       4975 |                   inspect
import time:       523 |       6251 |                 cryptography.utils
import time:       339 |      10321 |               secretstorage.dhcrypto
import time:       413 |        413 |               secretstorage.exceptions
import time:       286 |        286 |                       cryptography.hazmat
import time:       530 |        816 |                     cryptography.hazmat.primitives
import time:     11273 |      11273 |                       six
import time:       789 |        789 |                       cryptography.exceptions
import time:       270 |        270 |                         cryptography.hazmat.backends
import time:      1050 |       1319 |                       cryptography.hazmat.backends.interfaces
import time:      1514 |       1514 |                       cryptography.hazmat.primitives.ciphers.modes
import time:      1448 |      16341 |                     cryptography.hazmat.primitives.ciphers.base
import time:       431 |      17586 |                   cryptography.hazmat.primitives.ciphers
import time:       574 |        574 |                   cryptography.hazmat.primitives.ciphers.algorithms
import time:       608 |      18767 |                 secretstorage.util
import time:       487 |      19253 |               secretstorage.item
import time:      2491 |      32710 |             secretstorage.collection
import time:       443 |      45753 |           secretstorage
import time:       202 |        202 |             win32ctypes
import time:        68 |        270 |           win32ctypes.pywin32
import time:       205 |        205 |           pywintypes
import time:       899 |        899 |               _ctypes
import time:       518 |        518 |               ctypes._endian
import time:      1774 |       3191 |             ctypes
import time:       445 |       3636 |           keyring.backends._OS_X_API
import time:     55445 |     166539 |         keyring.core
import time:       383 |     166922 |       keyring
import time:       436 |     168585 |     grepme.login
import time:      1808 |       1808 |           locale
import time:      1902 |       3710 |         gettext
import time:      1421 |       5131 |       argparse
import time:       248 |        248 |         _datetime
import time:      1608 |       1855 |       datetime
import time:       340 |        340 |             _json
import time:       853 |       1192 |           json.scanner
import time:       908 |       2099 |         json.decoder
import time:      4758 |       4758 |         json.encoder
import time:       452 |       7307 |       json
import time:       637 |        637 |         certifi.core
import time:       671 |       1308 |       certifi
import time:        80 |         80 |               six.moves
import time:      1662 |       1662 |                 http
import time:       303 |        303 |                   email
import time:      1048 |       1048 |                     email.errors
import time:       476 |        476 |                         email.quoprimime
import time:       413 |        413 |                           base64
import time:       287 |        699 |                         email.base64mime
import time:       471 |        471 |                             quopri
import time:       274 |        744 |                           email.encoders
import time:       363 |       1106 |                         email.charset
import time:      1555 |       3834 |                       email.header
import time:        90 |         90 |                             _bisect
import time:       369 |        458 |                           bisect
import time:        85 |         85 |                           _random
import time:      1332 |       1874 |                         random
import time:       445 |        445 |                           urllib
import time:      1919 |       2363 |                         urllib.parse
import time:       890 |        890 |                           calendar
import time:       481 |       1371 |                         email._parseaddr
import time:      1206 |       6812 |                       email.utils
import time:       666 |      11312 |                     email._policybase
import time:      1414 |      13773 |                   email.feedparser
import time:       528 |      14604 |                 email.parser
import time:       380 |        380 |                   uu
import time:       542 |        542 |                   email._encoded_words
import time:       377 |        377 |                   email.iterators
import time:      1087 |       2385 |                 email.message
import time:      4597 |       4597 |                   _ssl
import time:      4793 |       9390 |                 ssl
import time:      1736 |      29774 |               http.client
import time:       189 |      30042 |             six.moves.http_client
import time:      2707 |      32749 |           urllib3.exceptions
import time:       611 |        611 |             urllib3.packages
import time:       765 |       1375 |           urllib3.packages.ssl_match_hostname
import time:       356 |        356 |             _queue
import time:       534 |        890 |           queue
import time:      6007 |       6007 |                     urllib3.util.selectors
import time:       724 |       6731 |                   urllib3.util.wait
import time:      1298 |       8028 |                 urllib3.util.connection
import time:       975 |        975 |                 urllib3.util.request
import time:       760 |        760 |                 urllib3.util.response
import time:      2343 |       2343 |                 urllib3.util.ssl_
import time:      1675 |       1675 |                 urllib3.util.timeout
import time:      2938 |       2938 |                 urllib3.util.retry
import time:        81 |         81 |                     six.moves.urllib
import time:        98 |        178 |                   six.moves.urllib.parse
import time:      2166 |       2343 |                 urllib3.util.url
import time:       779 |      19838 |               urllib3.util
import time:        56 |      19893 |             urllib3.util.ssl_
import time:      2901 |       2901 |             urllib3._collections
import time:      3210 |      26003 |           urllib3.connection
import time:       240 |        240 |                 _uuid
import time:       816 |       1055 |               uuid
import time:       226 |        226 |                   winreg
import time:       516 |        742 |                 mimetypes
import time:      1523 |       2264 |               urllib3.fields
import time:      1000 |       4319 |             urllib3.filepost
import time:      1360 |       5678 |           urllib3.request
import time:      4809 |       4809 |           urllib3.response
import time:      5912 |      77413 |         urllib3.connectionpool
import time:      3803 |       3803 |         urllib3.poolmanager
import time:      1315 |      82530 |       urllib3
import time:       249 |        249 |       grepme.constants
import time:       667 |      99044 |     grepme.lib
import time:       401 |     268028 |   grepme
import time:       282 |     268310 | grepme.__main__

[jyn514]

Note: these numbers are for a warm disk cache. With a cold cache the numbers are much worse, about 3 seconds:

cold cache

import time: self [us] | cumulative | imported package
import time:       768 |        768 | zipimport
import time:      3279 |       3279 | _frozen_importlib_external
import time:       283 |        283 |     _codecs
import time:     17384 |      17667 |   codecs
import time:      2292 |       2292 |   encodings.aliases
import time:    246192 |     266151 | encodings
import time:      7747 |       7747 | encodings.utf_8
import time:       889 |        889 | _signal
import time:      1606 |       1606 | encodings.latin_1
import time:       218 |        218 |     _abc
import time:      1259 |       1476 |   abc
import time:     11188 |      12663 | io
import time:       549 |        549 |       _stat
import time:      1129 |       1677 |     stat
import time:     19708 |      19708 |       genericpath
import time:      1247 |      20955 |     posixpath
import time:      7359 |       7359 |     _collections_abc
import time:     20439 |      50428 |   os
import time:     31344 |      31344 |   _sitebuiltins
import time:       338 |        338 |     _locale
import time:      6493 |       6830 |   _bootlocale
import time:     11694 |      11694 |   types
import time:      1783 |       1783 |       warnings
import time:      1891 |       3673 |     importlib
import time:     10855 |      10855 |       importlib.machinery
import time:     13547 |      24401 |     importlib.abc
import time:       452 |        452 |           _operator
import time:      4309 |       4760 |         operator
import time:      1123 |       1123 |         keyword
import time:       299 |        299 |           _heapq
import time:      4602 |       4901 |         heapq
import time:       523 |        523 |         itertools
import time:      1251 |       1251 |         reprlib
import time:       357 |        357 |         _collections
import time:      7818 |      20731 |       collections
import time:       312 |        312 |         _functools
import time:      2785 |       3097 |       functools
import time:      4156 |      27982 |     contextlib
import time:      9092 |      65147 |   importlib.util
import time:     27805 |      27805 |   sitecustomize
import time:       512 |        512 |   usercustomize
import time:    166918 |     360674 | site
import time:      4145 |       4145 |   enum
import time:       441 |        441 |     _sre
import time:      1880 |       1880 |       sre_constants
import time:      3005 |       4885 |     sre_parse
import time:      5546 |      10871 |   sre_compile
import time:      2922 |       2922 |   copyreg
import time:     16934 |      34869 | re
import time:       973 |        973 |       __future__
import time:     19803 |      19803 |         termios
import time:     12263 |      32066 |       getpass
import time:      9730 |       9730 |             collections.abc
import time:     17876 |      27606 |           configparser
import time:       557 |        557 |             time
import time:      9963 |       9963 |                   token
import time:      5038 |      15001 |                 tokenize
import time:      1072 |      16072 |               linecache
import time:      9707 |      25779 |             traceback
import time:      1959 |       1959 |               _weakrefset
import time:     12590 |      14549 |             weakref
import time:       290 |        290 |               _string
import time:     10711 |      11001 |             string
import time:      4749 |       4749 |             threading
import time:       249 |        249 |             atexit
import time:     22105 |      78985 |           logging
import time:      9149 |       9149 |                 fnmatch
import time:      2327 |      11476 |               glob
import time:       442 |        442 |                   errno
import time:       411 |        411 |                   zlib
import time:      7384 |       7384 |                     _compression
import time:     25629 |      25629 |                     _bz2
import time:      2817 |      35829 |                   bz2
import time:     21435 |      21435 |                     _lzma
import time:     10985 |      32420 |                   lzma
import time:       362 |        362 |                   pwd
import time:       226 |        226 |                   grp
import time:     13921 |      83608 |                 shutil
import time:       460 |        460 |                   _struct
import time:      1368 |       1827 |                 struct
import time:       403 |        403 |                 binascii
import time:     13890 |      99726 |               zipfile
import time:     39334 |     150535 |             entrypoints
import time:      1322 |       1322 |             keyring.credentials
import time:      1434 |       1434 |             keyring.errors
import time:      7812 |       7812 |             keyring.util
import time:      5759 |       5759 |             keyring.util.properties
import time:      3085 |     169945 |           keyring.backend
import time:      5712 |       5712 |                 signal
import time:       199 |        199 |                 _posixsubprocess
import time:       211 |        211 |                 select
import time:       239 |        239 |                   math
import time:      2159 |       2397 |                 selectors
import time:      6761 |      15278 |               subprocess
import time:     12452 |      27729 |             platform
import time:     33977 |      61706 |           keyring.util.platform_
import time:     19226 |      19226 |           keyring.backends
import time:      1591 |       1591 |           keyring.backends.fail
import time:      1524 |       1524 |             dbus._compat
import time:      4435 |       4435 |             dbus.exceptions
import time:       506 |        506 |               _dbus_bindings
import time:      2073 |       2578 |             dbus.types
import time:      8305 |      16840 |           dbus
import time:      4491 |       4491 |                   jeepney.auth
import time:      9620 |       9620 |                   jeepney.low_level
import time:      6498 |       6498 |                   jeepney.bus
import time:     17346 |      17346 |                     typing
import time:      2294 |      19639 |                   jeepney.wrappers
import time:      9680 |      49927 |                 jeepney
import time:     13428 |      63355 |               jeepney.integrate
import time:      1116 |       1116 |                 _socket
import time:     19296 |      20412 |               socket
import time:      1246 |       1246 |               jeepney.routing
import time:     10357 |      10357 |               jeepney.bus_messages
import time:      5894 |     101262 |             jeepney.integrate.blocking
import time:      4893 |       4893 |               secretstorage.defines
import time:     61452 |      61452 |                   _hashlib
import time:       450 |        450 |                     _blake2
import time:       418 |        418 |                     _sha3
import time:     16004 |      16872 |                   hashlib
import time:     14028 |      92350 |                 hmac
import time:      1471 |       1471 |                     cryptography.__about__
import time:     17439 |      18909 |                   cryptography
import time:      1664 |       1664 |                         _opcode
import time:      9378 |      11042 |                       opcode
import time:     10436 |      21478 |                     dis
import time:     18606 |      40083 |                   inspect
import time:     29592 |      88583 |                 cryptography.utils
import time:      1335 |     182267 |               secretstorage.dhcrypto
import time:      1705 |       1705 |               secretstorage.exceptions
import time:     39146 |      39146 |                       cryptography.hazmat
import time:     14603 |      53748 |                     cryptography.hazmat.primitives
import time:     35486 |      35486 |                       six
import time:      1460 |       1460 |                       cryptography.exceptions
import time:     21588 |      21588 |                         cryptography.hazmat.backends
import time:     10114 |      31701 |                       cryptography.hazmat.backends.interfaces
import time:      9004 |       9004 |                       cryptography.hazmat.primitives.ciphers.modes
import time:      3118 |      80767 |                     cryptography.hazmat.primitives.ciphers.base
import time:      3105 |     137620 |                   cryptography.hazmat.primitives.ciphers
import time:       967 |        967 |                   cryptography.hazmat.primitives.ciphers.algorithms
import time:      1446 |     140031 |                 secretstorage.util
import time:      1565 |     141596 |               secretstorage.item
import time:      9828 |     340286 |             secretstorage.collection
import time:     14082 |     455630 |           secretstorage
import time:       234 |        234 |             win32ctypes
import time:        71 |        304 |           win32ctypes.pywin32
import time:       311 |        311 |           pywintypes
import time:     38693 |      38693 |               _ctypes
import time:      1921 |       1921 |               ctypes._endian
import time:     18014 |      58626 |             ctypes
import time:      1167 |      59792 |           keyring.backends._OS_X_API
import time:   1442746 |    2334676 |         keyring.core
import time:     25152 |    2359828 |       keyring
import time:     23293 |    2416158 |     grepme.login
import time:     13908 |      13908 |           locale
import time:     10128 |      24036 |         gettext
import time:     16892 |      40928 |       argparse
import time:       872 |        872 |         _datetime
import time:      9337 |      10208 |       datetime
import time:      8781 |       8781 |             _json
import time:      8400 |      17181 |           json.scanner
import time:     11755 |      28935 |         json.decoder
import time:     14838 |      14838 |         json.encoder
import time:      1996 |      45768 |       json
import time:      2638 |       2638 |         certifi.core
import time:     19319 |      21956 |       certifi
import time:       192 |        192 |               six.moves
import time:     13342 |      13342 |                 http
import time:      6713 |       6713 |                   email
import time:      2044 |       2044 |                     email.errors
import time:      4531 |       4531 |                         email.quoprimime
import time:      6906 |       6906 |                           base64
import time:      1646 |       8552 |                         email.base64mime
import time:      4804 |       4804 |                             quopri
import time:       964 |       5767 |                           email.encoders
import time:      2131 |       7898 |                         email.charset
import time:      4095 |      25075 |                       email.header
import time:       417 |        417 |                             _bisect
import time:     10834 |      11251 |                           bisect
import time:       246 |        246 |                           _random
import time:     10402 |      21897 |                         random
import time:     36630 |      36630 |                           urllib
import time:     18743 |      55372 |                         urllib.parse
import time:      7499 |       7499 |                           calendar
import time:      2147 |       9645 |                         email._parseaddr
import time:      3594 |      90507 |                       email.utils
import time:     11125 |     126706 |                     email._policybase
import time:      4160 |     132909 |                   email.feedparser
import time:      9230 |     148851 |                 email.parser
import time:     13226 |      13226 |                   uu
import time:     11462 |      11462 |                   email._encoded_words
import time:      1148 |       1148 |                   email.iterators
import time:      3869 |      29705 |                 email.message
import time:     38313 |      38313 |                   _ssl
import time:     16403 |      54715 |                 ssl
import time:      4559 |     251170 |               http.client
import time:       360 |     251721 |             six.moves.http_client
import time:      6719 |     258440 |           urllib3.exceptions
import time:     24642 |      24642 |             urllib3.packages
import time:      1821 |      26462 |           urllib3.packages.ssl_match_hostname
import time:       967 |        967 |             _queue
import time:      1144 |       2111 |           queue
import time:      9385 |       9385 |                     urllib3.util.selectors
import time:      1196 |      10581 |                   urllib3.util.wait
import time:      2190 |      12771 |                 urllib3.util.connection
import time:      1903 |       1903 |                 urllib3.util.request
import time:      1397 |       1397 |                 urllib3.util.response
import time:      4164 |       4164 |                 urllib3.util.ssl_
import time:      2280 |       2280 |                 urllib3.util.timeout
import time:      4156 |       4156 |                 urllib3.util.retry
import time:        87 |         87 |                     six.moves.urllib
import time:       123 |        209 |                   six.moves.urllib.parse
import time:      2567 |       2776 |                 urllib3.util.url
import time:      1520 |      30964 |               urllib3.util
import time:        71 |      31034 |             urllib3.util.ssl_
import time:      2867 |       2867 |             urllib3._collections
import time:      5314 |      39214 |           urllib3.connection
import time:       148 |        148 |                 _uuid
import time:       803 |        950 |               uuid
import time:       805 |        805 |                   winreg
import time:     58974 |      59778 |                 mimetypes
import time:      1260 |      61038 |               urllib3.fields
import time:      1039 |      63026 |             urllib3.filepost
import time:      1263 |      64288 |           urllib3.request
import time:     12631 |      12631 |           urllib3.response
import time:     16711 |     419855 |         urllib3.connectionpool
import time:     10176 |      10176 |         urllib3.poolmanager
import time:     47314 |     477344 |       urllib3
import time:     18942 |      18942 |       grepme.constants
import time:     33812 |     648955 |     grepme.lib
import time:     48898 |    3114010 |   grepme
import time:     12787 |    3126797 | grepme.__main__

[jyn514]

See also jaraco/keyring#403.

[jyn514]

Maybe use https://pypi.org/project/lazy-import/?