lkddi/nexusphp
1
0
Fork 0
mirror of https://github.com/lkddi/nexusphp.git synced 2026-04-22 19:13:28 +08:00
Code Issues Packages Projects Releases Wiki Activity

improve cleanup

Browse Source
This commit is contained in:
xiaomlove
2022-11-05 18:43:49 +08:00
parent 4f4026aed1
commit 2f958f2a9c
15 changed files with 283 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/Console/Commands/Cleanup.php
+14 -10
View File
@@ -2,8 +2,9 @@
namespace App\Console\Commands;
use App\Jobs\CalculateSeedBonus;
use App\Jobs\UpdateSeedingLeechingTime;
use App\Jobs\CalculateUserSeedBonus;
use App\Jobs\UpdateTorrentSeedersEtc;
use App\Jobs\UpdateUserSeedingLeechingTime;
use Illuminate\Console\Command;
class Cleanup extends Command
@@ -13,14 +14,14 @@ class Cleanup extends Command
*
* @var string
*/
protected $signature = 'cleanup {--action=} {--begin_uid=} {--end_uid=}';
protected $signature = 'cleanup {--action=} {--begin_id=} {--end_id=} {--request_id=}';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Cleanup async job trigger, options: --begin_uid, --end_uid, --action (seed_bonus, seeding_leeching_time)';
protected $description = 'Cleanup async job trigger, options: --begin_id, --end_id, --request_id, --action (seed_bonus, seeding_leeching_time, seeders_etc)';
/**
* Execute the console command.
@@ -30,15 +31,18 @@ class Cleanup extends Command
public function handle()
{
$action = $this->option('action');
$beginUid = $this->option('begin_uid');
$endUid = $this->option('end_uid');
$this->info("beginUid: $beginUid, endUid: $endUid, action: $action");
$beginId = $this->option('begin_id');
$endId = $this->option('end_id');
$commentRequestId = $this->option('request_id');
$this->info("beginId: $beginId, endId: $endId, commentRequestId: $commentRequestId, action: $action");
if ($action == 'seed_bonus') {
CalculateSeedBonus::dispatch($beginUid, $endUid);
CalculateUserSeedBonus::dispatch($beginId, $endId, $commentRequestId);
} elseif ($action == 'seeding_leeching_time') {
UpdateSeedingLeechingTime::dispatch($beginUid, $endUid);
UpdateUserSeedingLeechingTime::dispatch($beginId, $endId, $commentRequestId);
}elseif ($action == 'seeders_etc') {
UpdateTorrentSeedersEtc::dispatch($beginId, $endId, $commentRequestId);
} else {
$msg = "Invalid action: $action";
$msg = "[$commentRequestId], Invalid action: $action";
do_log($msg, 'error');
$this->error($msg);
}
app/Filament/Resources/Section/CategoryResource.php
+20 -2
View File
@@ -6,7 +6,10 @@ use App\Filament\Resources\Section\CategoryResource\Pages;
use App\Filament\Resources\Section\CategoryResource\RelationManagers;
use App\Models\Category;
use App\Models\Icon;
use App\Models\NexusModel;
use App\Models\SearchBox;
use App\Repositories\SearchBoxRepository;
use Filament\Facades\Filament;
use Filament\Forms;
use Filament\Resources\Form;
use Filament\Resources\Resource;
@@ -14,6 +17,7 @@ use Filament\Resources\Table;
use Filament\Tables;
use Illuminate\Database\Eloquent\Builder;
use Illuminate\Database\Eloquent\SoftDeletingScope;
use Illuminate\Support\Collection;
class CategoryResource extends Resource
{
@@ -89,10 +93,24 @@ class CategoryResource extends Resource
])
->actions([
Tables\Actions\EditAction::make(),
Tables\Actions\DeleteAction::make(),
Tables\Actions\DeleteAction::make()->using(function (NexusModel $record) {
try {
$rep = new SearchBoxRepository();
$rep->deleteCategory($record->id);
} catch (\Exception $exception) {
Filament::notify('danger', $exception->getMessage() ?: class_basename($exception));
}
}),
])
->bulkActions([
Tables\Actions\DeleteBulkAction::make(),
Tables\Actions\DeleteBulkAction::make()->using(function (Collection $records) {
try {
$rep = new SearchBoxRepository();
$rep->deleteCategory($records->pluck('id')->toArray());
} catch (\Exception $exception) {
Filament::notify('danger', $exception->getMessage() ?: class_basename($exception));
}
}),
]);
}
app/Filament/Resources/Section/CategoryResource/Pages/CreateCategory.php
+5
View File
@@ -9,4 +9,9 @@ use Filament\Resources\Pages\CreateRecord;
class CreateCategory extends CreateRecord
{
protected static string $resource = CategoryResource::class;
protected function afterCreate()
{
clear_category_cache();
}
}
app/Filament/Resources/Section/CategoryResource/Pages/EditCategory.php
+9
View File
@@ -5,6 +5,7 @@ namespace App\Filament\Resources\Section\CategoryResource\Pages;
use App\Filament\Resources\Section\CategoryResource;
use Filament\Pages\Actions;
use Filament\Resources\Pages\EditRecord;
use Nexus\Database\NexusDB;
class EditCategory extends EditRecord
{
@@ -16,4 +17,12 @@ class EditCategory extends EditRecord
Actions\DeleteAction::make(),
];
}
/**
* @see functions.php::get_category_row()
*/
protected function afterSave()
{
clear_category_cache();
}
}
app/Filament/Resources/Torrent/TorrentResource.php
+6 -1
View File
@@ -5,6 +5,7 @@ namespace App\Filament\Resources\Torrent;
use App\Filament\OptionsTrait;
use App\Filament\Resources\Torrent\TorrentResource\Pages;
use App\Filament\Resources\Torrent\TorrentResource\RelationManagers;
use App\Models\Category;
use App\Models\Setting;
use App\Models\Tag;
use App\Models\Torrent;
@@ -122,6 +123,9 @@ class TorrentResource extends Resource
return $query->when($data['owner'], fn (Builder $query, $owner) => $query->where("owner", $owner));
})
,
Tables\Filters\SelectFilter::make('category')
->options(Category::query()->pluck('name', 'id')->toArray())
->label(__('label.torrent.category')),
Tables\Filters\SelectFilter::make('visible')
->options(self::$yesOrNo)
@@ -381,7 +385,8 @@ class TorrentResource extends Resource
private static function shouldShowApproval(): bool
{
return Setting::get('torrent.approval_status_none_visible') == 'no' || Setting::get('torrent.approval_status_icon_enabled') == 'yes';
return false;
// return Setting::get('torrent.approval_status_none_visible') == 'no' || Setting::get('torrent.approval_status_icon_enabled') == 'yes';
}
}
app/Jobs/CalculateSeedBonus.php → app/Jobs/CalculateUserSeedBonus.php
+16 -3
View File
@@ -12,7 +12,7 @@ use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Nexus\Database\NexusDB;
class CalculateSeedBonus implements ShouldQueue
class CalculateUserSeedBonus implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
@@ -20,15 +20,28 @@ class CalculateSeedBonus implements ShouldQueue
private int $endUid;
private string $requestId;
/**
* Create a new job instance.
*
* @return void
*/
public function __construct(int $beginUid, int $endUid)
public function __construct(int $beginUid, int $endUid, string $requestId = '')
{
$this->beginUid = $beginUid;
$this->endUid = $endUid;
$this->requestId = $requestId;
}
/**
* Determine the time at which the job should timeout.
*
* @return \DateTime
*/
public function retryUntil()
{
return now()->addSeconds(Setting::get('main.autoclean_interval_one'));
}
/**
@@ -39,7 +52,7 @@ class CalculateSeedBonus implements ShouldQueue
public function handle()
{
$beginTimestamp = time();
$logPrefix = sprintf("[CLEANUP_CLI_CALCULATE_SEED_BONUS], beginUid: %s, endUid: %s", $this->beginUid, $this->endUid);
$logPrefix = sprintf("[CLEANUP_CLI_CALCULATE_SEED_BONUS], commonRequestId: %s, beginUid: %s, endUid: %s", $this->requestId, $this->beginUid, $this->endUid);
$sql = sprintf("select userid from peers where userid > %s and userid <= %s and seeder = 'yes' group by userid", $this->beginUid, $this->endUid);
$results = NexusDB::select($sql);
$count = count($results);
app/Jobs/UpdateSeedingLeechingTime.php
-56
View File
@@ -1,56 +0,0 @@
<?php
namespace App\Jobs;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeUnique;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Nexus\Database\NexusDB;
class UpdateSeedingLeechingTime implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
private int $beginUid;
private int $endUid;
/**
* Create a new job instance.
*
* @return void
*/
public function __construct(int $beginUid, int $endUid)
{
$this->beginUid = $beginUid;
$this->endUid = $endUid;
}
/**
* Execute the job.
*
* @return void
*/
public function handle()
{
$beginTimestamp = time();
$logPrefix = sprintf("[CLEANUP_CLI_UPDATE_SEEDING_LEECHING_TIME], beginUid: %s, endUid: %s", $this->beginUid, $this->endUid);
$sql = sprintf("select id from users where id > %s and id <= %s and enabled = 'yes' and status = 'confirmed'", $this->beginUid, $this->endUid);
$results = NexusDB::select($sql);
do_log("$logPrefix, [GET_UID], sql: $sql, count: " . count($results));
foreach ($results as $arr) {
$uid = $arr['id'];
$sql = sprintf('select sum(seedtime) as st, sum(leechtime) as lt from snatched where userid = %s limit 1', $uid);
$row = NexusDB::select($sql);
if (is_numeric($row[0]['st'])) {
$sql = sprintf('update users set seedtime = %s, leechtime = %s where id = %s limit 1', $row[0]['st'], $row[0]['lt'], $uid);
NexusDB::statement($sql);
}
}
$costTime = time() - $beginTimestamp;
do_log("$logPrefix, [DONE], cost time: $costTime seconds");
}
}
app/Jobs/UpdateTorrentSeedersEtc.php
+66
View File
@@ -0,0 +1,66 @@
<?php
namespace App\Jobs;
use App\Models\Setting;
use App\Models\User;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeUnique;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Nexus\Database\NexusDB;
class UpdateTorrentSeedersEtc implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
private int $beginTorrentId;
private int $endTorrentId;
private string $requestId;
/**
* Create a new job instance.
*
* @return void
*/
public function __construct(int $beginTorrentId, int $endTorrentId, string $requestId = '')
{
$this->beginTorrentId = $beginTorrentId;
$this->endTorrentId = $endTorrentId;
$this->requestId = $requestId;
}
/**
* Determine the time at which the job should timeout.
*
* @return \DateTime
*/
public function retryUntil()
{
return now()->addSeconds(Setting::get('main.autoclean_interval_three'));
}
/**
* Execute the job.
*
* @return void
*/
public function handle()
{
$beginTimestamp = time();
$logPrefix = sprintf("[CLEANUP_CLI_UPDATE_TORRENT_SEEDERS_ETC], commonRequestId: %s, beginTorrentId: %s, endTorrentId: %s", $this->requestId, $this->beginTorrentId, $this->endTorrentId);
$sql = sprintf("update torrents set seeders = (select count(*) from peers where torrent = torrents.id and seeder = 'yes'), leechers = (select count(*) from peers where torrent = torrents.id and seeder = 'no'), comments = (select count(*) from comments where torrent = torrents.id) where id > %s and id <= %s",
$this->beginTorrentId, $this->endTorrentId
);
$result = NexusDB::statement($sql);
$costTime = time() - $beginTimestamp;
do_log(sprintf(
"$logPrefix, [DONE], sql: %s, result: %s, cost time: %s seconds",
preg_replace('/[\r\n\t]+/', ' ', $sql), var_export($result, true), $costTime
));
}
}
app/Jobs/UpdateUserSeedingLeechingTime.php
+63
View File
@@ -0,0 +1,63 @@
<?php
namespace App\Jobs;
use App\Models\Setting;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeUnique;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Nexus\Database\NexusDB;
class UpdateUserSeedingLeechingTime implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
private int $beginUid;
private int $endUid;
private string $requestId;
/**
* Create a new job instance.
*
* @return void
*/
public function __construct(int $beginUid, int $endUid, string $requestId = '')
{
$this->beginUid = $beginUid;
$this->endUid = $endUid;
$this->requestId = $requestId;
}
/**
* Determine the time at which the job should timeout.
*
* @return \DateTime
*/
public function retryUntil()
{
return now()->addSeconds(Setting::get('main.autoclean_interval_four'));
}
/**
* Execute the job.
*
* @return void
*/
public function handle()
{
$beginTimestamp = time();
$logPrefix = sprintf("[CLEANUP_CLI_UPDATE_SEEDING_LEECHING_TIME], commonRequestId: %s, beginUid: %s, endUid: %s", $this->requestId, $this->beginUid, $this->endUid);
$sql = sprintf(
"update users set seedtime = (select sum(seedtime) from snatched where userid = users.id), leechtime=(select sum(leechtime) from snatched where userid = users.id) where id > %s and id <= %s and status = 'confirmed' and enabled = 'yes'",
$this->beginUid, $this->endUid
);
$results = NexusDB::statement($sql);
$costTime = time() - $beginTimestamp;
do_log("$logPrefix, [DONE], sql: $sql, results: " . var_export($results, true) . " cost time: $costTime seconds");
}
}
app/Repositories/SearchBoxRepository.php
+20
View File
@@ -2,16 +2,21 @@
namespace App\Repositories;
use App\Exceptions\InsufficientPermissionException;
use App\Http\Middleware\Locale;
use App\Models\Category;
use App\Models\Icon;
use App\Models\NexusModel;
use App\Models\SearchBox;
use App\Models\SearchBoxField;
use App\Models\SecondIcon;
use App\Models\Setting;
use App\Models\Torrent;
use App\Models\User;
use Elasticsearch\Endpoints\Search;
use Illuminate\Database\Query\Builder;
use Illuminate\Support\Arr;
use Illuminate\Support\Facades\Auth;
use Nexus\Database\NexusDB;
use Filament\Forms;
@@ -260,5 +265,20 @@ class SearchBoxRepository extends BaseRepository
}
}
public function deleteCategory($id)
{
if (get_user_class() < User::CLASS_SYSOP) {
throw new InsufficientPermissionException();
}
$idArr = Arr::wrap($id);
$exists = Torrent::query()->whereHas('basic_category', function (\Illuminate\Database\Eloquent\Builder $query) use ($idArr) {
return $query->whereIn('id', $idArr);
})->exists();
if ($exists) {
throw new \RuntimeException("There are torrents that belong to this category and cannot be deleted!");
}
return Category::query()->whereIn('id', $idArr)->delete();
}
}
database/migrations/2021_06_08_113437_create_peers_table.php
+1 -1
View File
@@ -17,7 +17,7 @@ class CreatePeersTable extends Migration
return;
}
Schema::create('peers', function (Blueprint $table) {
$table->increments('id');
$table->bigIncrements('id');
$table->unsignedMediumInteger('torrent')->default(0);
$table->char('peer_id', 20)->charset('binary')->index();
$table->string('ip', 64)->default('');
database/migrations/2021_06_08_113437_create_snatched_table.php
+1 -1
View File
@@ -17,7 +17,7 @@ class CreateSnatchedTable extends Migration
return;
}
Schema::create('snatched', function (Blueprint $table) {
$table->integer('id', true);
$table->bigIncrements('id', true);
$table->unsignedMediumInteger('torrentid')->default(0);
$table->unsignedMediumInteger('userid')->default(0)->index('userid');
$table->string('ip', 64)->default('');
include/cleanup.php
+54 -35
View File
@@ -299,16 +299,18 @@ function docleanup($forceAll = 0, $printProgress = false) {
// }
//chunk async
$requestId = nexus()->getRequestId();
$maxUidRes = mysql_fetch_assoc(sql_query("select max(id) as max_uid from users limit 1"));
$maxUid = $maxUidRes['max_uid'];
$phpPath = nexus_env('PHP_PATH', 'php');
$webRoot = rtrim(ROOT_PATH, '/');
$chunk = 2000;
$beginUid = 0;
do_log("maxUid: $maxUid, chunk: $chunk");
do {
$command = sprintf(
'%s %s/artisan cleanup --action=seed_bonus --begin_uid=%s --end_uid=%s',
$phpPath, $webRoot, $beginUid, $beginUid + $chunk
'%s %s/artisan cleanup --action=seed_bonus --begin_id=%s --end_id=%s --request_id=%s',
$phpPath, $webRoot, $beginUid, $beginUid + $chunk, $requestId
);
$result = exec("$command 2>&1", $output, $result_code);
do_log(sprintf('command: %s, result_code: %s, result: %s, output: %s', $command, $result_code, $result, json_encode($output)));
@@ -366,38 +368,53 @@ function docleanup($forceAll = 0, $printProgress = false) {
}
//4.update count of seeders, leechers, comments for torrents
$torrents = array();
$res = sql_query("SELECT torrent, seeder, COUNT(*) AS c FROM peers GROUP BY torrent, seeder") or sqlerr(__FILE__, __LINE__);
while ($row = mysql_fetch_assoc($res)) {
if ($row["seeder"] == "yes")
$key = "seeders";
else
$key = "leechers";
$torrents[$row["torrent"]][$key] = $row["c"];
}
// $torrents = array();
// $res = sql_query("SELECT torrent, seeder, COUNT(*) AS c FROM peers GROUP BY torrent, seeder") or sqlerr(__FILE__, __LINE__);
// while ($row = mysql_fetch_assoc($res)) {
// if ($row["seeder"] == "yes")
// $key = "seeders";
// else
// $key = "leechers";
// $torrents[$row["torrent"]][$key] = $row["c"];
// }
//
// $res = sql_query("SELECT torrent, COUNT(*) AS c FROM comments GROUP BY torrent") or sqlerr(__FILE__, __LINE__);
// while ($row = mysql_fetch_assoc($res)) {
// $torrents[$row["torrent"]]["comments"] = $row["c"];
// }
//
// $fields = explode(":", "comments:leechers:seeders");
// $res = sql_query("SELECT id, seeders, leechers, comments FROM torrents") or sqlerr(__FILE__, __LINE__);
// while ($row = mysql_fetch_assoc($res)) {
// $id = $row["id"];
// $torr = $torrents[$id] ?? [];
// foreach ($fields as $field) {
// if (!isset($torr[$field]))
// $torr[$field] = 0;
// }
// $update = array();
// foreach ($fields as $field) {
// if ($torr[$field] != $row[$field])
// $update[] = "$field = " . $torr[$field];
// }
// if (count($update))
// sql_query("UPDATE torrents SET " . implode(",", $update) . " WHERE id = $id") or sqlerr(__FILE__, __LINE__);
// }
$res = sql_query("SELECT torrent, COUNT(*) AS c FROM comments GROUP BY torrent") or sqlerr(__FILE__, __LINE__);
while ($row = mysql_fetch_assoc($res)) {
$torrents[$row["torrent"]]["comments"] = $row["c"];
}
$fields = explode(":", "comments:leechers:seeders");
$res = sql_query("SELECT id, seeders, leechers, comments FROM torrents") or sqlerr(__FILE__, __LINE__);
while ($row = mysql_fetch_assoc($res)) {
$id = $row["id"];
$torr = $torrents[$id] ?? [];
foreach ($fields as $field) {
if (!isset($torr[$field]))
$torr[$field] = 0;
}
$update = array();
foreach ($fields as $field) {
if ($torr[$field] != $row[$field])
$update[] = "$field = " . $torr[$field];
}
if (count($update))
sql_query("UPDATE torrents SET " . implode(",", $update) . " WHERE id = $id") or sqlerr(__FILE__, __LINE__);
}
$maxTorrentIdRes = mysql_fetch_assoc(sql_query("select max(id) as max_torrent_id from torrents limit 1"));
$maxTorrentId = $maxTorrentIdRes['max_torrent_id'];
$chunk = 5000;
$beginTorrentId = 0;
do_log("maxTorrentId: $maxTorrentId, chunk: $chunk");
do {
$command = sprintf(
'%s %s/artisan cleanup --action=seeders_etc --begin_id=%s --end_id=%s --request_id=%s',
$phpPath, $webRoot, $beginTorrentId, $beginTorrentId + $chunk, $requestId
);
$result = exec("$command 2>&1", $output, $result_code);
do_log(sprintf('command: %s, result_code: %s, result: %s, output: %s', $command, $result_code, $result, json_encode($output)));
$beginTorrentId += $chunk;
} while ($beginTorrentId < $maxTorrentId);
$log = "update count of seeders, leechers, comments for torrents";
do_log($log);
if ($printProgress) {
@@ -841,11 +858,13 @@ function docleanup($forceAll = 0, $printProgress = false) {
// sql_query("UPDATE users SET seedtime = " . intval($arr2['st']) . ", leechtime = " . intval($arr2['lt']) . " WHERE id = " . $arr['id']) or sqlerr(__FILE__, __LINE__);
// }
$chunk = 2000;
$beginUid = 0;
do_log("maxUid: $maxUid, chunk: $chunk");
do {
$command = sprintf(
'%s %s/artisan cleanup --action=seeding_leeching_time --begin_uid=%s --end_uid=%s',
$phpPath, $webRoot, $beginUid, $beginUid + $chunk
'%s %s/artisan cleanup --action=seeding_leeching_time --begin_id=%s --end_id=%s --request_id=%s',
$phpPath, $webRoot, $beginUid, $beginUid + $chunk, $requestId
);
$result = exec("$command 2>&1", $output, $result_code);
do_log(sprintf('command: %s, result_code: %s, result: %s, output: %s', $command, $result_code, $result, json_encode($output)));
include/functions.php
+1 -1
View File
@@ -2911,7 +2911,7 @@ function stdfoot() {
$yearfounded = ($year ? $year : 2007);
print(" (c) "." <a href=\"" . get_protocol_prefix() . $BASEURL."\" target=\"_self\">".$SITENAME."</a> ".($icplicense_main ? " ".$icplicense_main." " : "").(date("Y") != $yearfounded ? $yearfounded."-" : "").date("Y")." ".VERSION."<br /><br />");
printf ("[page created in <b> %s </b> sec", sprintf("%.3f", $totaltime));
print (", takes up <b>".mksize(memory_get_usage())."</b> ram]");
print (" with <b>".count($query_name)."</b> db queries, <b>".$Cache->getCacheReadTimes()."</b> reads and <b>".$Cache->getCacheWriteTimes()."</b> writes of Redis and <b>".mksize(memory_get_usage())."</b> ram]");
print ("</div>\n");
if ($enablesqldebug_tweak == 'yes' && get_user_class() >= $sqldebug_tweak) {
print("<div id=\"sql_debug\" style='text-align: left;'>SQL query list: <ul>");
include/globalfunctions.php
+7
View File
@@ -986,7 +986,14 @@ function clear_setting_cache()
{
\Nexus\Database\NexusDB::cache_del('nexus_settings_in_laravel');
\Nexus\Database\NexusDB::cache_del('nexus_settings_in_nexus');
}
/**
* @see functions.php::get_category_row()
*/
function clear_category_cache()
{
\Nexus\Database\NexusDB::cache_del('category_content');
}
function clear_staff_message_cache()