Refactor IP History

app/Console/Kernel.php

@@ -6,6 +6,7 @@ use App\Jobs\CheckCleanup;
 use App\Jobs\CheckQueueFailedJobs;
 use App\Jobs\MaintainPluginState;
 use App\Jobs\ManagePlugin;
+use App\Jobs\SaveIpLogCacheToDB;
 use App\Jobs\UpdateIsSeedBoxFromUserRecordsCache;
 use App\Utils\ThirdPartyJob;
 use Carbon\Carbon;
@@ -48,10 +49,10 @@ class Kernel extends ConsoleKernel
         $schedule->command('meilisearch:import')->weeklyOn(1, "03:00");
         $schedule->command('torrent:load_pieces_hash')->dailyAt("01:00");
         $schedule->job(new CheckQueueFailedJobs())->everySixHours();
-//        $schedule->job(new ThirdPartyJob())->everyMinute();
         $schedule->job(new MaintainPluginState())->everyMinute();
         $schedule->job(new UpdateIsSeedBoxFromUserRecordsCache())->everySixHours();
         $schedule->job(new CheckCleanup())->everyFifteenMinutes();
+        $schedule->job(new SaveIpLogCacheToDB())->hourly();
 
     }
 

app/Filament/Resources/System/IpLogs/IpLogResource.php

@@ -0,0 +1,139 @@
+<?php
+
+namespace App\Filament\Resources\System\IpLogs;
+
+use App\Filament\Resources\System\IpLogs\Pages\ManageIpLogs;
+use App\Models\IpLog;
+use BackedEnum;
+use Filament\Actions\BulkActionGroup;
+use Filament\Actions\DeleteAction;
+use Filament\Actions\DeleteBulkAction;
+use Filament\Actions\EditAction;
+use Filament\Actions\ViewAction;
+use Filament\Forms\Components\DateTimePicker;
+use Filament\Forms\Components\TextInput;
+use Filament\Resources\Resource;
+use Filament\Schemas\Schema;
+use Filament\Support\Icons\Heroicon;
+use Filament\Tables\Columns\TextColumn;
+use Filament\Tables\Filters\Filter;
+use Filament\Tables\Table;
+use Illuminate\Database\Eloquent\Builder;
+use UnitEnum;
+
+class IpLogResource extends Resource
+{
+    protected static ?string $model = IpLog::class;
+
+    protected static string|BackedEnum|null $navigationIcon = Heroicon::OutlinedRectangleStack;
+
+    protected static ?int $navigationSort = 3;
+
+    protected static string | UnitEnum | null $navigationGroup = 'System';
+
+    public static function getLabel(): ?string
+    {
+        return __('ip-log.label');
+    }
+
+    public static function form(Schema $schema): Schema
+    {
+        return $schema
+            ->components([
+                //
+            ]);
+    }
+
+    public static function table(Table $table): Table
+    {
+        return $table
+            ->columns([
+                TextColumn::make('userid')
+                    ->label('UID')
+                ,
+                TextColumn::make('usernameForAdmin')
+                    ->label(__('label.username'))
+                ,
+                TextColumn::make('ip')
+                    ->label('IP')
+                ,
+                TextColumn::make('ipLocation')
+                    ->label(__('ip-log.ip_location'))
+                ,
+                TextColumn::make('uri')
+                    ->label(__('ip-log.uri'))
+                ,
+                TextColumn::make('count')
+                    ->label(__('ip-log.count'))
+                ,
+                TextColumn::make('access')
+                    ->label(__('ip-log.access'))
+                    ->tooltip(__('ip-log.access_tooltip'))
+                ,
+            ])
+            ->defaultSort('id', 'desc')
+            ->filters([
+                Filter::make('uid')
+                    ->schema([
+                        TextInput::make('uid')->label('UID'),
+                    ])
+                    ->query(function (Builder $query, array $data) {
+                        return $query
+                            ->when(
+                                $data['uid'],
+                                fn (Builder $query, $value): Builder => $query->where('userid', $value),
+                            );
+                    }),
+                Filter::make('ip')
+                    ->schema([
+                        TextInput::make('ip')->label('IP'),
+                    ])
+                    ->query(function (Builder $query, array $data) {
+                        return $query
+                            ->when(
+                                $data['ip'],
+                                fn (Builder $query, $value): Builder => $query->where('ip', $value),
+                            );
+                    }),
+                Filter::make('access_begin')
+                    ->schema([
+                        DateTimePicker::make('access_begin')->label(__('ip-log.access_begin')),
+                    ])
+                    ->query(function (Builder $query, array $data) {
+                        return $query
+                            ->when(
+                                $data['access_begin'],
+                                fn (Builder $query, $value): Builder => $query->where('access', '>=', $value),
+                            );
+                    }),
+                Filter::make('access_end')
+                    ->schema([
+                        DateTimePicker::make('access_end')->label(__('ip-log.access_end')),
+                    ])
+                    ->query(function (Builder $query, array $data) {
+                        return $query
+                            ->when(
+                                $data['access_end'],
+                                fn (Builder $query, $value): Builder => $query->where('access', '<=', $value),
+                            );
+                    }),
+            ])
+            ->recordActions([
+//                ViewAction::make(),
+//                EditAction::make(),
+//                DeleteAction::make(),
+            ])
+            ->toolbarActions([
+                BulkActionGroup::make([
+//                    DeleteBulkAction::make(),
+                ]),
+            ]);
+    }
+
+    public static function getPages(): array
+    {
+        return [
+            'index' => ManageIpLogs::route('/'),
+        ];
+    }
+}

app/Filament/Resources/System/IpLogs/Pages/ManageIpLogs.php

@@ -0,0 +1,20 @@
+<?php
+
+namespace App\Filament\Resources\System\IpLogs\Pages;
+
+use App\Filament\PageListSingle;
+use App\Filament\Resources\System\IpLogs\IpLogResource;
+use Filament\Actions\CreateAction;
+use Filament\Resources\Pages\ManageRecords;
+
+class ManageIpLogs extends PageListSingle
+{
+    protected static string $resource = IpLogResource::class;
+
+    protected function getHeaderActions(): array
+    {
+        return [
+//            CreateAction::make(),
+        ];
+    }
+}

app/Http/Kernel.php

@@ -4,6 +4,7 @@ namespace App\Http;
 
 use App\Http\Middleware\Filament;
 use App\Http\Middleware\Locale;
+use App\Http\Middleware\LogUserIp;
 use Illuminate\Foundation\Http\Kernel as HttpKernel;
 
 class Kernel extends HttpKernel
@@ -24,7 +25,7 @@ class Kernel extends HttpKernel
         \App\Http\Middleware\TrimStrings::class,
 //        \Illuminate\Foundation\Http\Middleware\ConvertEmptyStringsToNull::class,
         \App\Http\Middleware\BootNexus::class,
-        Locale::class,
+        LogUserIp::class,
     ];
 
     /**
@@ -46,7 +47,6 @@ class Kernel extends HttpKernel
         'api' => [
             'throttle:api',
             \Illuminate\Routing\Middleware\SubstituteBindings::class,
-//            \App\Http\Middleware\Platform::class,
         ],
         'filament' => [
             \Illuminate\Session\Middleware\StartSession::class,
@@ -73,8 +73,6 @@ class Kernel extends HttpKernel
         'signed' => \Illuminate\Routing\Middleware\ValidateSignature::class,
         'throttle' => \Illuminate\Routing\Middleware\ThrottleRequestsWithRedis::class,
         'verified' => \Illuminate\Auth\Middleware\EnsureEmailIsVerified::class,
-        'permission' => \App\Http\Middleware\Permission::class,
-        'admin' => \App\Http\Middleware\Admin::class,
         'locale' => \App\Http\Middleware\Locale::class,
         'checkUserStatus' => \App\Http\Middleware\CheckUserStatus::class,
     ];

app/Http/Middleware/Admin.php

@@ -1,30 +0,0 @@
-<?php
-
-namespace App\Http\Middleware;
-
-use App\Models\User;
-use Closure;
-use Illuminate\Http\Request;
-use Illuminate\Validation\UnauthorizedException;
-
-class Admin
-{
-    /**
-     * Handle an incoming request.
-     *
-     * @param  \Illuminate\Http\Request  $request
-     * @param  \Closure  $next
-     * @return mixed
-     */
-    public function handle(Request $request, Closure $next)
-    {
-        /** @var CheckUserStatus $user */
-        $user = $request->user();
-        if (!$user || !$user->canAccessAdmin()) {
-            do_log("denied!");
-            throw new UnauthorizedException('Unauthorized!');
-        }
-        do_log("allow!");
-        return $next($request);
-    }
-}

app/Http/Middleware/BootNexus.php

@@ -2,6 +2,7 @@
 
 namespace App\Http\Middleware;
 
+use App\Repositories\IpLogRepository;
 use Closure;
 use Illuminate\Http\Request;
 use Nexus\Nexus;

app/Http/Middleware/LogUserIp.php

@@ -0,0 +1,26 @@
+<?php
+
+namespace App\Http\Middleware;
+
+use App\Repositories\IpLogRepository;
+use Closure;
+use Illuminate\Http\Request;
+use Symfony\Component\HttpFoundation\Response;
+
+class LogUserIp
+{
+    /**
+     * Handle an incoming request.
+     *
+     * @param  \Closure(\Illuminate\Http\Request): (\Symfony\Component\HttpFoundation\Response)  $next
+     */
+    public function handle(Request $request, Closure $next): Response
+    {
+        $response = $next($request);
+        $user = $request->user();
+        if ($user) {
+            IpLogRepository::saveToCache($user->id);
+        }
+        return $response;
+    }
+}

app/Http/Middleware/Permission.php

@@ -1,30 +0,0 @@
-<?php
-
-namespace App\Http\Middleware;
-
-use App\Models\User;
-use Closure;
-use Illuminate\Http\Request;
-use Illuminate\Validation\UnauthorizedException;
-
-class Permission
-{
-    /**
-     * Handle an incoming request.
-     *
-     * @param  \Illuminate\Http\Request  $request
-     * @param  \Closure  $next
-     * @return mixed
-     */
-    public function handle(Request $request, Closure $next)
-    {
-        /** @var CheckUserStatus $user */
-        $user = $request->user();
-        if (!$user || (nexus()->isPlatformAdmin() && !$user->canAccessAdmin())) {
-            do_log("denied!");
-            throw new UnauthorizedException('Unauthorized!');
-        }
-        do_log("allow!");
-        return $next($request);
-    }
-}

app/Http/Middleware/Platform.php

@@ -1,29 +0,0 @@
-<?php
-
-namespace App\Http\Middleware;
-
-use Closure;
-use Illuminate\Http\Request;
-use Illuminate\Validation\UnauthorizedException;
-
-class Platform
-{
-    /**
-     * Handle an incoming request.
-     *
-     * @param  \Illuminate\Http\Request  $request
-     * @param  \Closure  $next
-     * @return mixed
-     */
-    public function handle(Request $request, Closure $next)
-    {
-        $platform = nexus()->getPlatform();
-        if (empty($platform)) {
-            throw new \InvalidArgumentException("Require platform header.");
-        }
-        if (!nexus()->isPlatformValid()) {
-            throw new \InvalidArgumentException("Invalid platform: " . $platform);
-        }
-        return $next($request);
-    }
-}

app/Jobs/CalculateUserSeedBonus.php

@@ -3,8 +3,10 @@
 namespace App\Jobs;
 
 use App\Models\BonusLogs;
+use App\Models\IpLog;
 use App\Models\Setting;
 use App\Models\User;
+use App\Repositories\IpLogRepository;
 use Illuminate\Bus\Queueable;
 use Illuminate\Contracts\Queue\ShouldBeUnique;
 use Illuminate\Contracts\Queue\ShouldQueue;
@@ -238,7 +240,7 @@ class CalculateUserSeedBonus implements ShouldQueue
             $client = app(\ClickHouseDB\Client::class);
             $fields = ['business_type', 'uid', 'old_total_value', 'value', 'new_total_value', 'comment', 'created_at'];
             $client->insert("bonus_logs", $bonusLogInsert, $fields);
-            do_log("insertIntoClickHouseBulk done, created_at: {$bonusLogInsert[0]['created_at']}");
+            do_log("insertIntoClickHouseBulk done, created_at: {$bonusLogInsert[0]['created_at']}, count: " . count($bonusLogInsert));
         } catch (\Exception $e) {
             do_log($e->getMessage(), 'error');
         }

app/Jobs/SaveIpLogCacheToDB.php

@@ -0,0 +1,31 @@
+<?php
+
+namespace App\Jobs;
+
+use App\Repositories\IpLogRepository;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Queue\Queueable;
+
+class SaveIpLogCacheToDB implements ShouldQueue
+{
+    use Queueable;
+
+    public $tries = 1;
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct()
+    {
+        //
+    }
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        IpLogRepository::saveToDB();
+        do_log("done");
+    }
+}

app/Models/IpLog.php

@@ -0,0 +1,40 @@
+<?php
+
+namespace App\Models;
+
+
+use Illuminate\Database\Eloquent\Casts\Attribute;
+
+class IpLog extends NexusModel
+{
+    protected $table = 'iplog';
+
+    protected $fillable = ['ip', 'userid', 'access', 'uri', 'count'];
+
+    protected function ipLocation(): Attribute
+    {
+        return new Attribute(
+            get: fn (mixed $value, array $attributes) => $this->getIpLocation($attributes['ip'])
+        );
+    }
+
+    private function getIpLocation(string $ip)
+    {
+        $result = get_ip_location_from_geoip($ip);
+        $out = $result['name'];
+        $suffix = [];
+        if (!empty($result['city_en'])) {
+            $suffix[] = $result['city_en'];
+        }
+        if (!empty($result['country_en'])) {
+            $suffix[] = $result['country_en'];
+        }
+        if (!empty($result['continent_en'])) {
+            $suffix[] = $result['continent_en'];
+        }
+        if (!empty($suffix)) {
+            $out .= " " . implode(', ', $suffix);
+        }
+        return $out;
+    }
+}

app/Models/NexusModel.php

@@ -2,6 +2,7 @@
 
 namespace App\Models;
 
+use Illuminate\Database\Eloquent\Casts\Attribute;
 use Illuminate\Database\Eloquent\Factories\HasFactory;
 use Illuminate\Database\Eloquent\Model;
 use Nexus\Database\NexusDB;
@@ -16,6 +17,13 @@ class NexusModel extends Model
 
     protected $connection = NexusDB::ELOQUENT_CONNECTION_NAME;
 
+    protected function usernameForAdmin(): Attribute
+    {
+        return Attribute::make(
+            get: fn (mixed $value, array $attributes) => username_for_admin($attributes['uid'] ?? $attributes['userid'] ?? $attributes['user_id'])
+        );
+    }
+
     /**
      *
      * @param  \DateTimeInterface  $date

app/Models/Setting.php

@@ -108,7 +108,7 @@ class Setting extends NexusModel
     {
         $redis = NexusDB::redis();
         $key = self::USER_TOKEN_PERMISSION_ALLOWED_CACHE_KRY;
-        $redis->del($key);
+        $redis->unlink($key);
         //must not use cache
         if (empty($allowed)) {
             $allowed = self::getFromDb("permission.user_token_allowed");

app/Models/TrackerUrl.php

@@ -35,7 +35,7 @@ class TrackerUrl extends NexusModel
     {
         //添加 id 与 URL 映射
         $redis = NexusDB::redis();
-        $redis->del(self::TRACKER_URL_CACHE_KEY);
+        $redis->unlink(self::TRACKER_URL_CACHE_KEY);
         $list = self::listAll();
         $first = $list->first();
         $hasDefault = false;

app/Providers/RouteServiceProvider.php

@@ -40,11 +40,11 @@ class RouteServiceProvider extends ServiceProvider
 
         $this->routes(function () {
             Route::prefix('api/v1')
-                ->middleware('api')
+                ->middleware(['api', 'locale'])
                 ->namespace($this->namespace)
                 ->group(base_path('routes/api.php'));
 
-            Route::middleware('web')
+            Route::middleware(['web', 'locale'])
                 ->namespace($this->namespace)
                 ->group(base_path('routes/web.php'));
 

app/Repositories/CleanupRepository.php

@@ -132,7 +132,7 @@ class CleanupRepository extends BaseRepository
 
         //remove this batch
         if ($batchKey != self::USER_SEED_BONUS_BATCH_KEY) {
-            $redis->del($batch);
+            $redis->unlink($batch);
         }
         $endTimestamp = time();
         do_log(sprintf("$logPrefix, [DONE], batch: $batch, count: $count, cost time: %d seconds", $endTimestamp - $beginTimestamp));

app/Repositories/IpLogRepository.php

@@ -0,0 +1,87 @@
+<?php
+
+namespace App\Repositories;
+
+use App\Models\IpLog;
+use Carbon\Carbon;
+use Nexus\Database\NexusDB;
+
+class IpLogRepository extends BaseRepository
+{
+    const CACHE_KEY_PREFIX = 'nexus_ip_logs';
+
+    private const CACHE_TIME = 72 * 3600;
+
+    public static function saveToCache($userId, $uri = null, $ipArr = null): void
+    {
+        if (!is_numeric($userId) || $userId <= 0) {
+            do_log("invalid userId: $userId", "error");
+            return;
+        }
+        $redis = NexusDB::redis();
+        if (is_null($uri)) {
+            $parsed_uri = parse_url($_SERVER['REQUEST_URI']);
+            $uri = $parsed_uri['path'];
+        }
+        if (is_null($ipArr)) {
+            $ipArr = [getip()];
+        }
+        $key = sprintf("%s:%s", self::CACHE_KEY_PREFIX, date('Y-m-d-H'));
+        foreach ($ipArr as $ip) {
+            $field = sprintf("%s|%s|%s", $userId, $ip, $uri);
+            $result = $redis->hincrby($key, $field, 1);
+            do_log("success hincrby $key $field, result: $result", "debug");
+            if ($result === 1) {
+                $redis->expire($key, self::CACHE_TIME);
+            }
+        }
+    }
+
+    public static function saveToDB(): void
+    {
+        $beginTimestamp = microtime(true);
+        $redis = NexusDB::redis();
+        $begin = Carbon::now()->subSeconds(self::CACHE_TIME);
+        $end = Carbon::now()->subHours(1);
+        $interval =\DateInterval::createFromDateString("1 hour");
+        $period = new \DatePeriod($begin->clone(), $interval, $end);
+        $size = 2000;
+        do_log(sprintf("begin: %s, end: %s, size: %s", $begin->toDateTimeString(), $end->toDateTimeString(), $size));
+        $redis->setOption(\Redis::OPT_SCAN, \Redis::SCAN_RETRY);
+        foreach ($period as $dt) {
+            $key = sprintf("%s:%s", self::CACHE_KEY_PREFIX, $dt->format('Y-m-d-H'));
+            if (!$redis->exists($key)) {
+                do_log("key: $key not found", "debug");
+                continue;
+            }
+            if ($redis->hlen($key) == 0) {
+                do_log("key: $key length = 0", "debug");
+                $redis->unlink($key);
+            }
+            do_log("handing key: $key");
+            //遍历hash
+            $it = NULL;
+            while($arr_keys = $redis->hScan($key, $it, "*", $size)) {
+                $insert = [];
+                foreach ($arr_keys as $field => $value) {
+                    list($userId, $ip, $uri) = explode("|", $field);
+                    $insert[] = [
+                        'userid' => $userId,
+                        'ip' => $ip,
+                        'uri' => $uri,
+                        'access' => date("Y-m-d H:i:s"),
+                        'count' => intval($value),
+                    ];
+                }
+                if (!empty($insert)) {
+                    IpLog::query()->insert($insert);
+                }
+                do_log("key: $key, it: $it, count: " . count($insert));
+            }
+            $redis->unlink($key);
+            do_log("handle key: $key done!");
+        }
+        do_log(sprintf("all done! cost time: %.3f sec.", microtime(true) - $beginTimestamp));
+    }
+
+}

app/Repositories/RequireSeedTorrentRepository.php

@@ -104,7 +104,7 @@ class RequireSeedTorrentRepository extends BaseRepository
             //remove torrent from list
             $redis->hDel(self::getTorrentCacheKey(), $torrent->id);
             //remove all users under torrent
-            $redis->del(self::getTorrentUserCacheKey($torrent->id));
+            $redis->unlink(self::getTorrentUserCacheKey($torrent->id));
         }
         RequireSeedTorrent::query()->whereIn('torrent_id', $idArr)->delete();
         UserRequireSeedTorrent::query()->whereIn('torrent_id', $idArr)->delete();

app/Repositories/TorrentRepository.php

@@ -221,9 +221,7 @@ class TorrentRepository extends BaseRepository
             }
 
             if ($apiQueryBuilder->hasIncludeField('description') && $apiQueryBuilder->hasInclude('extra')) {
-                do_log("before format_description of torrent: {$torrent->id}");
                 $descriptionArr = format_description($torrent->extra->descr ?? '');
-                do_log("after format_description of torrent: {$torrent->id}");
                 $torrent->description = $descriptionArr;
                 $torrent->images = get_image_from_description($descriptionArr);
             }

app/Utils/ApiQueryBuilder.php

@@ -101,10 +101,10 @@ class ApiQueryBuilder
     {
         $includeCounts = explode(',', $this->request->query(self::PARAM_NAME_INCLUDE_COUNTS, ''));
         $valid = array_intersect($this->allowedIncludeCounts, $includeCounts);
-        do_log(sprintf(
-            "includeCounts: %s, allow: %s, valid: %s",
-            json_encode($includeCounts), json_encode($this->allowedIncludeCounts), json_encode($valid)
-        ));
+//        do_log(sprintf(
+//            "includeCounts: %s, allow: %s, valid: %s",
+//            json_encode($includeCounts), json_encode($this->allowedIncludeCounts), json_encode($valid)
+//        ));
         $this->query->withCount($valid);
     }