Skip to content

Commit

Permalink
replace dateutil with fromisoformat in google backend
Browse files Browse the repository at this point in the history
dateutil.parser.parse is particularly slow and covers a lot of use cases
that are not relevant to parsing RFC 3339 datetimes.

This was bad enough that rohmu could eat 50% of CPU while listing
entries from Google object storage.
  • Loading branch information
kmichel-aiven committed May 3, 2024
1 parent d30a6a9 commit d33b77e
Showing 1 changed file with 2 additions and 2 deletions.
4 changes: 2 additions & 2 deletions rohmu/object_storage/google.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@
from oauth2client.client import GoogleCredentials
from rohmu.common.models import StorageOperation
from rohmu.common.statsd import StatsClient, StatsdConfig
from rohmu.dates import parse_timestamp
from rohmu.errors import FileNotFoundFromStorageError, InvalidByteRangeError, InvalidConfigurationError
from rohmu.notifier.interface import Notifier
from rohmu.object_storage.base import (
Expand All @@ -46,6 +45,7 @@

import codecs
import dataclasses
import datetime
import errno

# NOTE: this import is not needed per-se, but it's imported here first to point the
Expand Down Expand Up @@ -387,7 +387,7 @@ def initial_op(domain: Any) -> HttpRequest:
if (size := item.get("size")) is not None:
value["size"] = int(size)
if (updated := item.get("updated")) is not None:
value["last_modified"] = parse_timestamp(updated)
value["last_modified"] = datetime.datetime.fromisoformat(updated)
if (md5 := item.get("md5Hash")) is not None:
value["md5"] = base64_to_hex(md5)
yield IterKeyItem(type=KEY_TYPE_OBJECT, value=value)
Expand Down

0 comments on commit d33b77e

Please sign in to comment.