Extensive memory consumptuion 8.1.1 #757
Description
See my graphql type below.
I put breakpoints at the beginnging of every prefetcher inside the type. As soon as code reaches 1st prefetcher the memory usage on 4500 items is already 173 MB.
In the same time at the end of my resolver the memory usage is only 28 mb (see the screenshot) for the same 4500 items. Even if i return array of doctrine entities.
So my understanding is graphql library inflates the memory usage 5-6x times. Any ideas?... Another endpoint that does not have prefetch uses only 37 MB for 5000 items. So i think it might be related to prefetch. Note that i took measurements in the very beginning of prefetch thus nothing was prefetched so this is not the reason why memory was used up. I had this problem with graphqlite 6.0 thought that update to 8.1.1 may have helped.
<?php
declare(strict_types=1);
...
class GetIntermediateXGroupsQuery
{
/**
* @return iterable<IntermediateXGroupDataType>
*/
#[GQL\Query(name: 'reiseIntermediateXGroups')]
#[GQL\Logged, GQL\Security('this.canPerformQuery(user)')]
public function __invoke(
#[Autowire] EntityManagerInterface $em,
#[GQL\InjectUser] SessionUser $sessionUser,
?bool $isEnabled = null,
): iterable {
$qb = $em->getRepository(IntermediateXGroupDataEntity::class)
->createQueryBuilder('i')
->orderBy('i.id', 'ASC')
->setMaxResults(5000);
if ($isEnabled !== null) {
$qb->andWhere('i.isEnabled = :isEnabled')
->setParameter('isEnabled', (int)$isEnabled);
}
$query = $qb->getQuery()->setHint(Query::HINT_READ_ONLY, true);
foreach ($query->toIterable() as $row) {
yield IntermediateXGroupDataType::createFromEntity($row);
}
}
public function canPerformQuery(SessionUser $user): bool
{
return $user->hasRight($this) && $user->isPlatformPermitted(PlatformEnum::X);
}
}
<?php
declare(strict_types=1);
...
#[GQL\Type(name: self::GQL_TYPE)]
class IntermediateXGroupDataType
{
public const string GQL_TYPE = 'IntermediateXGroupData';
#[GQL\Field]
public int $id;
#[GQL\Field]
public bool $isEnabled;
#[GQL\Field]
public string $name;
#[GQL\Field]
public string $type;
#[GQL\Field]
public ?int $cityId = null;
#[GQL\Field]
public ?int $groupId = null;
#[GQL\Field]
public ?float $latitude = null;
#[GQL\Field]
public ?float $longitude = null;
#[GQL\Field]
public bool $isActive;
#[GQL\Field]
public ?\DateTimeImmutable $createdAt = null;
#[GQL\Field]
public ?\DateTimeImmutable $updatedAt = null;
#[GQL\Field]
public ?\DateTimeImmutable $deletedAt = null;
#[GQL\Field(name: 'distanceFromCityKm', prefetchMethod: 'prefetchCityDistancesKm')]
public function distanceFromCityKm(array $prefetchedDistancesByXId): ?float
{
return $prefetchedDistancesByXId[$this->id];
}
/**
*
* @param array<int, string> $prefetchedCityNamesById
*/
#[GQL\Field(name: 'cityName', prefetchMethod: 'prefetchCityNames')]
public function getCityName(array $prefetchedCityNamesById): ?string
{
return $prefetchedCityNamesById[$this->cityId] ?? null;
}
/**
*
* @param iterable<self> $Xs
* @return array<int, float|null> Mapping of XId => distanceKm|null
*/
public static function prefetchCityDistancesKm(
iterable $Xs,
#[Autowire] EntityManagerInterface $em,
#[Autowire] DistanceRepository $distanceRepository
): array {
$XIds = [];
$cityIds = [];
$XsNeedingDistance = [];
foreach ($Xs as $X) {
$XIds[] = $X->id;
if ($X->cityId !== null && $X->latitude !== null && $X->longitude !== null) {
$cityIds[] = $X->cityId;
$XsNeedingDistance[$X->id] = $X;
}
}
// Default all results to null
$distancesByXId = array_fill_keys($XIds, null);
if ($cityIds === []) {
return $distancesByXId;
}
$cityIds = array_unique($cityIds);
$cityCoordsById = self::fetchCityCoordinatesByIds($cityIds, $em);
// Compute distances
foreach ($XsNeedingDistance as $XId => $X) {
$cityId = $X->cityId;
if (!isset($cityCoordsById[$cityId])) {
continue;
}
$cityLat = $cityCoordsById[$cityId]['latitude'];
$cityLon = $cityCoordsById[$cityId]['longitude'];
if ($X->latitude === null || $X->longitude === null || $cityLat === null || $cityLon === null) {
continue;
}
$distancesByXId[$XId] = $distanceRepository->calculateSphericalLawOfCosinesDistanceKm(
$X->latitude,
$X->longitude,
$cityLat,
$cityLon
);
}
return $distancesByXId;
}
/**
* @param iterable<self> $Xs
* @return array<int, string>
*/
public function prefetchCityNames(iterable $Xs, #[Autowire] EntityManagerInterface $em): array
{
$cityIds = [];
foreach ($Xs as $X) {
if ($X->cityId !== null) {
$cityIds[] = $X->cityId;
}
}
$cityIds = array_unique($cityIds);
return $this->fetchCityNamesByIds($cityIds, $em);
}
/**
*
* @param array<int, array<string>> $prefetchedSourcesByXId Mapping of XId => list of sourceType strings
* @return string[]
*/
#[GQL\Field(name: 'sources', prefetchMethod: 'prefetchXSources')]
public function getSources(array $prefetchedSourcesByXId): array
{
return $prefetchedSourcesByXId[$this->id] ?? [];
}
/**
* @param iterable<self> $Xs
* @return array<int, array<string>> Mapping of XId => list of sourceType strings
*/
public static function prefetchXSources(iterable $Xs, #[Autowire] EntityManagerInterface $em): array
{
$XIds = [];
foreach ($Xs as $X) {
$XIds[] = $X->id;
}
$qb = $em->getRepository(MappingXGroupDataEntity::class)
->createQueryBuilder('m')
->join('m.X', 'h')
->select('h.id AS XId', 'm.sourceType AS sourceType')
->where('h.id IN (:XIds)')
->setParameter('XIds', $XIds);
$result = array_fill_keys($XIds, []);
foreach ($qb->getQuery()->toIterable() as $row) {
if (!in_array($row['sourceType'], $result[$row['XId']], true)) {
$result[$row['XId']][] = $row['sourceType'];
}
}
return $result;
}
/**
* @param int[] $ids
* @return array<int, array{latitude: float|null, longitude: float|null}>
*/
private static function fetchCityCoordinatesByIds(array $ids, EntityManagerInterface $em): array
{
if ($ids === []) {
return [];
}
$byId = [];
$qb = $em->getRepository(IntermediateLocationGroupDataEntity::class)
->createQueryBuilder('c')
->select('c.id AS id', 'c.latitude AS latitude', 'c.longitude AS longitude')
->where('c.id IN (:ids)')
->setParameter('ids', $ids);
foreach ($qb->getQuery()->toIterable([], \Doctrine\ORM\AbstractQuery::HYDRATE_ARRAY) as $row) {
$byId[(int) $row['id']] = [
'latitude' => $row['latitude'] !== null ? (float) $row['latitude'] : null,
'longitude' => $row['longitude'] !== null ? (float) $row['longitude'] : null,
];
}
return $byId;
}
/**
* @param int[] $ids
* @return array<int, string>
*/
private function fetchCityNamesByIds(array $ids, EntityManagerInterface $em): array
{
if ($ids === []) {
return [];
}
$names = [];
$qb = $em->getRepository(IntermediateLocationGroupDataEntity::class)
->createQueryBuilder('c')
->select('c.id AS id', 'c.name AS name')
->where('c.id IN (:ids)')
->setParameter('ids', $ids);
foreach ($qb->getQuery()->toIterable([], \Doctrine\ORM\AbstractQuery::HYDRATE_ARRAY) as $row) {
$names[(int) $row['id']] = (string) $row['name'];
}
return $names;
}
public static function createFromEntity(IntermediateXGroupDataEntity $entity): self
{
$instance = new self();
$instance->id = $entity->getId();
$instance->isEnabled = $entity->getIsEnabled();
$instance->name = $entity->getName();
$instance->type = $entity->getType();
$instance->cityId = $entity->getCity()?->getId();
$instance->groupId = $entity->getGroup()?->getId();
$instance->latitude = $entity->getLatitude();
$instance->longitude = $entity->getLongitude();
$instance->isActive = $entity->getIsActive();
$instance->createdAt = $entity->getCreatedAt();
$instance->updatedAt = $entity->getUpdatedAt();
$instance->deletedAt = $entity->getDeletedAt();
return $instance;
}
}