From 47346c48441d2e8954d120b5e91291bb95247744 Mon Sep 17 00:00:00 2001 From: xiaomlove Date: Wed, 26 Jul 2023 03:29:50 +0800 Subject: [PATCH] add torrent pieces hash --- .../Commands/TorrentLoadPiecesHash.php | 39 ++++++++ app/Http/Controllers/TorrentController.php | 9 ++ app/Repositories/TorrentRepository.php | 97 +++++++++++++++++++ ...0623_add_pieces_hash_to_torrents_table.php | 32 ++++++ include/constants.php | 2 +- include/functions.php | 13 ++- public/takeupload.php | 3 + routes/api.php | 1 + 8 files changed, 193 insertions(+), 3 deletions(-) create mode 100644 app/Console/Commands/TorrentLoadPiecesHash.php create mode 100644 database/migrations/2023_07_25_010623_add_pieces_hash_to_torrents_table.php diff --git a/app/Console/Commands/TorrentLoadPiecesHash.php b/app/Console/Commands/TorrentLoadPiecesHash.php new file mode 100644 index 00000000..1872c8d9 --- /dev/null +++ b/app/Console/Commands/TorrentLoadPiecesHash.php @@ -0,0 +1,39 @@ +option('id'); + $rep = new TorrentRepository(); + $this->info("id: $id"); + $total = $rep->loadPiecesHashCache($id); + $this->info(sprintf("total: %s, cost time: %s seconds.", $total, time() - $begin)); + return Command::SUCCESS; + } +} diff --git a/app/Http/Controllers/TorrentController.php b/app/Http/Controllers/TorrentController.php index 4ffed7a2..cb8e74f3 100644 --- a/app/Http/Controllers/TorrentController.php +++ b/app/Http/Controllers/TorrentController.php @@ -148,4 +148,13 @@ class TorrentController extends Controller return $this->success($params); } + public function queryByPiecesHash(Request $request) + { + $request->validate([ + 'pieces_hash' => 'required|array', + ]); + $result = $this->repository->getPiecesHashCache($request->pieces_hash); + return $this->success($result); + } + } diff --git a/app/Repositories/TorrentRepository.php b/app/Repositories/TorrentRepository.php index 6d5b3849..929aefe4 100644 --- a/app/Repositories/TorrentRepository.php +++ b/app/Repositories/TorrentRepository.php @@ -34,11 +34,14 @@ use Illuminate\Support\Facades\Auth; use Illuminate\Support\Facades\DB; use Illuminate\Support\Str; use Nexus\Database\NexusDB; +use Rhilip\Bencode\Bencode; class TorrentRepository extends BaseRepository { const BOUGHT_USER_CACHE_KEY_PREFIX = "torrent_purchasers:"; + const PIECES_HASH_CACHE_KEY = "torrent_pieces_hash"; + /** * fetch torrent list * @@ -758,4 +761,98 @@ HTML; return self::BOUGHT_USER_CACHE_KEY_PREFIX . $torrentId; } + public function addPiecesHashCache(int $torrentId, string $piecesHash): bool|int|\Redis + { + $value = $this->buildPiecesHashCacheValue($torrentId, $piecesHash); + return NexusDB::redis()->hSet(self::PIECES_HASH_CACHE_KEY, $piecesHash, $value); + } + + private function buildPiecesHashCacheValue(int $torrentId, string $piecesHash): bool|string + { + return json_encode(['torrent_id' => $torrentId, 'pieces_hash' => $piecesHash]); + } + + public function delPiecesHashCache(string $piecesHash): bool|int|\Redis + { + return NexusDB::redis()->hDel(self::PIECES_HASH_CACHE_KEY, $piecesHash); + } + + public function getPiecesHashCache($piecesHash): array + { + if (!is_array($piecesHash)) { + $piecesHash = [$piecesHash]; + } + $maxCount = 100; + if (count($piecesHash) > $maxCount) { + throw new \InvalidArgumentException("too many pieces hash, must less then $maxCount"); + } + $pipe = NexusDB::redis()->multi(\Redis::PIPELINE); + foreach ($piecesHash as $hash) { + $pipe->hGet(self::PIECES_HASH_CACHE_KEY, $hash); + } + $results = $pipe->exec(); + $out = []; + foreach ($results as $item) { + $arr = json_decode($item, true); + if (is_array($arr) && isset($arr['torrent_id'], $arr['pieces_hash'])) { + $out[$arr['pieces_hash']] = $arr['torrent_id']; + } else { + do_log("invalid item: $item", 'error'); + } + } + return $out; + } + + public function loadPiecesHashCache($id = 0): int + { + $page = 1; + $size = 1000; + $query = Torrent::query(); + if ($id) { + $query = $query->whereIn("id", Arr::wrap($id)); + } + $total = 0; + $torrentDir = sprintf( + "%s/%s/", + rtrim(ROOT_PATH, '/'), + rtrim(get_setting("main.torrent_dir"), '/') + ); + while (true) { + $list = (clone $query)->forPage($page, $size)->get(['id', 'pieces_hash']); + if ($list->isEmpty()) { + do_log("page: $page, size: $size, no more data..."); + break; + } + $pipe = NexusDB::redis()->multi(\Redis::PIPELINE); + $piecesHashCaseWhen = $updateIdArr = []; + foreach ($list as $item) { + $piecesHash = $item->pieces_hash; + if (!$piecesHash) { + $torrentFile = $torrentDir . $item->id . ".torrent"; + $loadResult = Bencode::load($torrentFile); + $piecesHash = sha1($loadResult['info']['pieces']); + $piecesHashCaseWhen[] = sprintf("when %s then '%s'", $item->id, $piecesHash); + $updateIdArr[] = $item->id; + do_log(sprintf("torrent: %s no pieces hash, load from torrent file: %s, pieces hash: %s", $item->id, $torrentFile, $piecesHash)); + } + $pipe->hSet(self::PIECES_HASH_CACHE_KEY, $piecesHash, $this->buildPiecesHashCacheValue($item->id, $piecesHash)); + } + $pipe->exec(); + if (!empty($piecesHashCaseWhen)) { + $sql = sprintf( + "update torrents set pieces_hash = case id %s end where id in (%s)", + implode(' ', $piecesHashCaseWhen), + implode(", ", $updateIdArr) + ); + NexusDB::statement($sql); + } + $count = $list->count(); + $total += $count; + do_log("success load page: $page, size: $size, count: $count"); + $page++; + } + do_log("[DONE], total: $total"); + return $total; + } + } diff --git a/database/migrations/2023_07_25_010623_add_pieces_hash_to_torrents_table.php b/database/migrations/2023_07_25_010623_add_pieces_hash_to_torrents_table.php new file mode 100644 index 00000000..23466848 --- /dev/null +++ b/database/migrations/2023_07_25_010623_add_pieces_hash_to_torrents_table.php @@ -0,0 +1,32 @@ +char("pieces_hash", 40)->default("")->index(); + }); + } + + /** + * Reverse the migrations. + * + * @return void + */ + public function down() + { + Schema::table('torrents', function (Blueprint $table) { + $table->dropColumn("pieces_hash"); + }); + } +}; diff --git a/include/constants.php b/include/constants.php index f42afe8e..9240a90b 100644 --- a/include/constants.php +++ b/include/constants.php @@ -1,6 +1,6 @@ whereIn("id", $idArr) + ->get(['id', 'pieces_hash']) + ->KeyBy("id") + ; + $torrentRep = new \App\Repositories\TorrentRepository(); $idStr = implode(', ', $idArr ?: [0]); $torrent_dir = get_setting('main.torrent_dir'); \Nexus\Database\NexusDB::statement("DELETE FROM torrents WHERE id in ($idStr)"); @@ -3123,7 +3129,10 @@ function deletetorrent($id, $notify = false) { } \Nexus\Database\NexusDB::statement("DELETE FROM hit_and_runs WHERE torrent_id in ($idStr)"); \Nexus\Database\NexusDB::statement("DELETE FROM claims WHERE torrent_id in ($idStr)"); - foreach ($idArr as $_id) { + foreach ($torrentInfo as $_id => $info) { + if ($torrentInfo->has($_id)) { + $torrentRep->delPiecesHashCache($torrentInfo->get($_id)->pieces_hash); + } do_action("torrent_delete", $_id); do_log("delete torrent: $_id", "error"); unlink(getFullDirectory("$torrent_dir/$_id.torrent")); diff --git a/public/takeupload.php b/public/takeupload.php index 74ac716a..9d78d472 100644 --- a/public/takeupload.php +++ b/public/takeupload.php @@ -348,6 +348,7 @@ $insert = [ 'pt_gen' => $_POST['pt_gen'] ?? '', 'technical_info' => $_POST['technical_info'] ?? '', 'cover' => $cover, + 'pieces_hash' => sha1($info['pieces']), ]; if (isset($_POST['hr'][$catmod]) && isset(\App\Models\Torrent::$hrStatus[$_POST['hr'][$catmod]]) && user_can('torrent_hr')) { $insert['hr'] = $_POST['hr'][$catmod]; @@ -431,6 +432,8 @@ foreach ($filelist as $file) { KPS("+",$uploadtorrent_bonus,$CURUSER["id"]); //===end +$torrentRep = new \App\Repositories\TorrentRepository(); +$torrentRep->addPiecesHashCache($id, $insert['pieces_hash']); write_log("Torrent $id ($torrent) was uploaded by $anon"); diff --git a/routes/api.php b/routes/api.php index 8935de81..b99140d3 100644 --- a/routes/api.php +++ b/routes/api.php @@ -28,6 +28,7 @@ Route::group(['middleware' => ['auth:sanctum', 'locale']], function () { Route::resource('messages', \App\Http\Controllers\MessageController::class); Route::get('messages-unread', [\App\Http\Controllers\MessageController::class, 'listUnread']); Route::resource('torrents', \App\Http\Controllers\TorrentController::class); + Route::get("pieces-hash", [\App\Http\Controllers\TorrentController::class, "queryByPiecesHash"])->name("torrent.pieces_hash.query"); Route::resource('comments', \App\Http\Controllers\CommentController::class); Route::resource('peers', \App\Http\Controllers\PeerController::class); Route::resource('files', \App\Http\Controllers\FileController::class);