Navigation Menu

Skip to content

Commit

Permalink
Add NetworkTimelineService cache
Browse files Browse the repository at this point in the history
  • Loading branch information
@dansup
dansup committed Jun 9, 2022
1 parent 9c17def commit 1310d95
Show file tree
Hide file tree
Showing 4 changed files with 199 additions and 53 deletions.
140 changes: 87 additions & 53 deletions app/Http/Controllers/PublicApiController.php
View file
Expand Up @@ -32,6 +32,7 @@
LikeService,
PublicTimelineService,
ProfileService,
NetworkTimelineService,
ReblogService,
RelationshipService,
StatusService,
Expand Down Expand Up @@ -608,59 +609,92 @@ public function networkTimelineApi(Request $request)

$filtered = $user ? UserFilterService::filters($user->profile_id) : [];

if($min || $max) {
$dir = $min ? '>' : '<';
$id = $min ?? $max;
$timeline = Status::select(
'id',
'uri',
'type',
'scope',
'created_at',
)
->where('id', $dir, $id)
->whereNull(['in_reply_to_id', 'reblog_of_id'])
->whereNotIn('profile_id', $filtered)
->whereIn('type', ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])
->whereNotNull('uri')
->whereScope('public')
->where('id', '>', $amin)
->orderBy('created_at', 'desc')
->limit($limit)
->get()
->map(function($s) use ($user) {
$status = StatusService::get($s->id);
$status['favourited'] = (bool) LikeService::liked($user->profile_id, $s->id);
$status['bookmarked'] = (bool) BookmarkService::get($user->profile_id, $s->id);
$status['reblogged'] = (bool) ReblogService::get($user->profile_id, $s->id);
return $status;
});
$res = $timeline->toArray();
} else {
$timeline = Status::select(
'id',
'uri',
'type',
'scope',
'created_at',
)
->whereNull(['in_reply_to_id', 'reblog_of_id'])
->whereNotIn('profile_id', $filtered)
->whereIn('type', ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])
->whereNotNull('uri')
->whereScope('public')
->where('id', '>', $amin)
->orderBy('created_at', 'desc')
->limit($limit)
->get()
->map(function($s) use ($user) {
$status = StatusService::get($s->id);
$status['favourited'] = (bool) LikeService::liked($user->profile_id, $s->id);
$status['bookmarked'] = (bool) BookmarkService::get($user->profile_id, $s->id);
$status['reblogged'] = (bool) ReblogService::get($user->profile_id, $s->id);
return $status;
});
$res = $timeline->toArray();
if(config('instance.timeline.network.cached') == false) {
if($min || $max) {
$dir = $min ? '>' : '<';
$id = $min ?? $max;
$timeline = Status::select(
'id',
'uri',
'type',
'scope',
'created_at',
)
->where('id', $dir, $id)
->whereNull(['in_reply_to_id', 'reblog_of_id'])
->whereNotIn('profile_id', $filtered)
->whereIn('type', ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])
->whereNotNull('uri')
->whereScope('public')
->where('id', '>', $amin)
->orderBy('created_at', 'desc')
->limit($limit)
->get()
->map(function($s) use ($user) {
$status = StatusService::get($s->id);
$status['favourited'] = (bool) LikeService::liked($user->profile_id, $s->id);
$status['bookmarked'] = (bool) BookmarkService::get($user->profile_id, $s->id);
$status['reblogged'] = (bool) ReblogService::get($user->profile_id, $s->id);
return $status;
});
$res = $timeline->toArray();
} else {
$timeline = Status::select(
'id',
'uri',
'type',
'scope',
'created_at',
)
->whereNull(['in_reply_to_id', 'reblog_of_id'])
->whereNotIn('profile_id', $filtered)
->whereIn('type', ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])
->whereNotNull('uri')
->whereScope('public')
->where('id', '>', $amin)
->orderBy('created_at', 'desc')
->limit($limit)
->get()
->map(function($s) use ($user) {
$status = StatusService::get($s->id);
$status['favourited'] = (bool) LikeService::liked($user->profile_id, $s->id);
$status['bookmarked'] = (bool) BookmarkService::get($user->profile_id, $s->id);
$status['reblogged'] = (bool) ReblogService::get($user->profile_id, $s->id);
return $status;
});
$res = $timeline->toArray();
}
} else {
Cache::remember('api:v1:timelines:network:cache_check', 10368000, function() {
if(NetworkTimelineService::count() == 0) {
NetworkTimelineService::warmCache(true, 400);
}
});

if ($max) {
$feed = NetworkTimelineService::getRankedMaxId($max, $limit);
} else if ($min) {
$feed = NetworkTimelineService::getRankedMinId($min, $limit);
} else {
$feed = NetworkTimelineService::get(0, $limit);
}

$res = collect($feed)
->map(function($k) use($user) {
$status = StatusService::get($k);
if($status && isset($status['account']) && $user) {
$status['favourited'] = (bool) LikeService::liked($user->profile_id, $k);
$status['bookmarked'] = (bool) BookmarkService::get($user->profile_id, $k);
$status['reblogged'] = (bool) ReblogService::get($user->profile_id, $k);
$status['relationship'] = RelationshipService::get($user->profile_id, $status['account']['id']);
}
return $status;
})
->filter(function($s) use($filtered) {
return $s && isset($s['account']) && in_array($s['account']['id'], $filtered) == false;
})
->values()
->toArray();
}

return response()->json($res);
Expand Down
95 changes: 95 additions & 0 deletions app/Services/NetworkTimelineService.php
View file
@@ -0,0 +1,95 @@
<?php

namespace App\Services;

use Illuminate\Support\Facades\Redis;
use App\{
Profile,
Status,
UserFilter
};

class NetworkTimelineService
{
const CACHE_KEY = 'pf:services:timeline:network';

public static function get($start = 0, $stop = 10)
{
if($stop > 100) {
$stop = 100;
}

return Redis::zrevrange(self::CACHE_KEY, $start, $stop);
}

public static function getRankedMaxId($start = null, $limit = 10)
{
if(!$start) {
return [];
}

return array_keys(Redis::zrevrangebyscore(self::CACHE_KEY, $start, '-inf', [
'withscores' => true,
'limit' => [1, $limit]
]));
}

public static function getRankedMinId($end = null, $limit = 10)
{
if(!$end) {
return [];
}

return array_keys(Redis::zrevrangebyscore(self::CACHE_KEY, '+inf', $end, [
'withscores' => true,
'limit' => [0, $limit]
]));
}

public static function add($val)
{
if(self::count() > config('instance.timeline.network.cache_dropoff')) {
if(config('database.redis.client') === 'phpredis') {
Redis::zpopmin(self::CACHE_KEY);
}
}

return Redis::zadd(self::CACHE_KEY, $val, $val);
}

public static function rem($val)
{
return Redis::zrem(self::CACHE_KEY, $val);
}

public static function del($val)
{
return self::rem($val);
}

public static function count()
{
return Redis::zcard(self::CACHE_KEY);
}

public static function warmCache($force = false, $limit = 100)
{
if(self::count() == 0 || $force == true) {
Redis::del(self::CACHE_KEY);
$ids = Status::whereNotNull('uri')
->whereScope('public')
->whereNull('in_reply_to_id')
->whereNull('reblog_of_id')
->whereIn('type', ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])
->where('created_at', '>', now()->subHours(config('instance.timeline.network.max_hours_old')))
->orderByDesc('created_at')
->limit($limit)
->pluck('id');
foreach($ids as $id) {
self::add($id);
}
return 1;
}
return 0;
}
}
11 changes: 11 additions & 0 deletions app/Util/ActivityPub/Helpers.php
View file
Expand Up @@ -32,6 +32,7 @@
use App\Services\InstanceService;
use App\Services\MediaPathService;
use App\Services\MediaStorageService;
use App\Services\NetworkTimelineService;
use App\Jobs\MediaPipeline\MediaStoragePipeline;
use App\Jobs\AvatarPipeline\RemoteAvatarFetch;
use App\Util\Media\License;
Expand Down Expand Up @@ -490,6 +491,16 @@ public static function storeStatus($url, $profile, $activity)
if(isset($activity['tag']) && is_array($activity['tag']) && !empty($activity['tag'])) {
StatusTagsPipeline::dispatch($activity, $status);
}

if( config('instance.timeline.network.cached') &&
$status->in_reply_to_id === null &&
$status->reblog_of_id === null &&
in_array($status->type, ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album']) &&
$status->created_at->gt(now()->subHours(config('instance.timeline.network.max_hours_old'))
) {
NetworkTimelineService::add($status->id);
}

return $status;
});
}
Expand Down
6 changes: 6 additions & 0 deletions config/instance.php
View file
Expand Up @@ -24,6 +24,12 @@
'timeline' => [
'local' => [
'is_public' => env('INSTANCE_PUBLIC_LOCAL_TIMELINE', false)
],

'network' => [
'cached' => env('PF_NETWORK_TIMELINE') ? env('INSTANCE_NETWORK_TIMELINE_CACHED', false) : false,
'cache_dropoff' => env('INSTANCE_NETWORK_TIMELINE_CACHE_DROPOFF', 100),
'max_hours_old' => env('INSTANCE_NETWORK_TIMELINE_CACHE_MAX_HOUR_INGEST', 6)
]
],

Expand Down

0 comments on commit 1310d95

Please sign in to comment.