lkddi/nexusphp
1
0
Fork 0
mirror of https://github.com/lkddi/nexusphp.git synced 2026-04-21 02:20:54 +08:00
Code Issues Packages Projects Releases Wiki Activity

elastic basically finish

Browse Source
This commit is contained in:
xiaomlove
2022-03-26 04:27:04 +08:00
parent eb7451f574
commit 0bc1e2f9d3
17 changed files with 1146 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env.example
+3 -1
View File
@@ -16,7 +16,7 @@ DB_PASSWORD=
BROADCAST_DRIVER=log
CACHE_DRIVER=redis
QUEUE_CONNECTION=sync
QUEUE_CONNECTION=redis
SESSION_DRIVER=file
SESSION_LIFETIME=120
@@ -71,3 +71,5 @@ ELASTICSEARCH_PORT=
ELASTICSEARCH_SCHEME=
ELASTICSEARCH_USER=
ELASTICSEARCH_PASS=
ELASTICSEARCH_ENABLED=
app/Console/Commands/Test.php
+48 -8
View File
@@ -2,6 +2,7 @@
namespace App\Console\Commands;
use App\Events\TorrentUpdated;
use App\Http\Resources\TagResource;
use App\Models\Attendance;
use App\Models\Exam;
@@ -20,6 +21,7 @@ use App\Repositories\AttendanceRepository;
use App\Repositories\ExamRepository;
use App\Repositories\HitAndRunRepository;
use App\Repositories\SearchBoxRepository;
use App\Repositories\SearchRepository;
use App\Repositories\TagRepository;
use App\Repositories\TorrentRepository;
use App\Repositories\UserRepository;
@@ -31,6 +33,7 @@ use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Hash;
use Illuminate\Support\Facades\Redis;
use Illuminate\Support\Facades\Storage;
use JeroenG\Explorer\Domain\Syntax\Matching;
use JeroenG\Explorer\Infrastructure\Scout\ElasticEngine;
use Rhilip\Bencode\Bencode;
@@ -67,15 +70,52 @@ class Test extends Command
*/
public function handle()
{
$class = Torrent::class;
try {
$this->call("scout:import App\\Models\Torrent");
} catch (\Throwable $e) {
$this->info($e->getMessage());
$lastQueryAsJson = ElasticEngine::debug()->json();
dd($lastQueryAsJson);
}
$searchRep = new SearchRepository();
// $r = $searchRep->deleteIndex();
// $r = $searchRep->createIndex();
// $r = $searchRep->import();
$arr = [
'cat' => 'category',
'source' => 'source',
'medium' => 'medium',
'codec' => 'codec',
'audiocodec' => 'audiocodec',
'standard' => 'standard',
'processing' => 'processing',
'team' => 'team',
];
$queryString = 'cat401=1&cat404=1&source2=1&medium2=1&medium3=1&codec3=1&audiocodec3=1&standard2=1&standard3=1&processing2=1&team3=1&team4=1&incldead=1&spstate=0&inclbookmarked=0&search=&search_area=0&search_mode=0';
$userSetting = '[cat401][cat404][sou1][med1][cod1][sta2][sta3][pro2][tea2][aud2][incldead=0][spstate=3][inclbookmarked=2]';
// foreach ($arr as $queryField => $value) {
//// $pattern = sprintf("/\[%s([\d]+)\]/", substr($queryField, 0, 3));
// $pattern = "/{$queryField}([\d]+)=/";
// if (preg_match_all($pattern, $queryString, $matches)) {
// dump($matches);
// echo '----------------------' . PHP_EOL;
// }
// }
// $r = preg_match("/\[incldead=([\d]+)\]/", $userSetting, $matches);
// dump($matches);
$params = [
'tag_id' => 1,
// 'incldead' => 0,
// 'spstate' => 0,
// 'inclbookmarked' => 0,
// 'search' => '5034',
// 'search_area' => 4,
// 'search_mode' => 0,
];
$queryString = "cat401=1&cat404=1&cat405=1&cat402=1&cat403=1&cat406=1&cat407=1&cat409=1&cat408=1&incldead=0&spstate=0&inclbookmarked=0&search=5034838&search_area=4&search_mode=0";
// $r = $searchRep->listTorrentFromEs($params, 1, '');
// $r = $searchRep->updateTorrent(1);
// $r = $searchRep->updateUser(1);
$r = $searchRep->addTorrent(1);
// TorrentUpdated::dispatch(1);
dd($r);
}
app/Events/TorrentUpdated.php
+38
View File
@@ -0,0 +1,38 @@
<?php
namespace App\Events;
use Illuminate\Broadcasting\Channel;
use Illuminate\Broadcasting\InteractsWithSockets;
use Illuminate\Broadcasting\PresenceChannel;
use Illuminate\Broadcasting\PrivateChannel;
use Illuminate\Contracts\Broadcasting\ShouldBroadcast;
use Illuminate\Foundation\Events\Dispatchable;
use Illuminate\Queue\SerializesModels;
class TorrentUpdated
{
use Dispatchable, InteractsWithSockets, SerializesModels;
public int $torrentId;
/**
* Create a new event instance.
*
* @return void
*/
public function __construct(int $torrentId)
{
$this->torrentId = $torrentId;
}
/**
* Get the channels the event should broadcast on.
*
* @return \Illuminate\Broadcasting\Channel|array
*/
public function broadcastOn()
{
return new PrivateChannel('channel-name');
}
}
app/Listeners/SyncTorrentToEs.php
+62
View File
@@ -0,0 +1,62 @@
<?php
namespace App\Listeners;
use App\Models\Setting;
use App\Repositories\SearchRepository;
use App\Repositories\ToolRepository;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Queue\InteractsWithQueue;
class SyncTorrentToEs implements ShouldQueue
{
public $tries = 3;
/**
* Create the event listener.
*
* @return void
*/
public function __construct()
{
//
}
/**
* Handle the event.
*
* @param object $event
* @return void
*/
public function handle($event)
{
$id = $event->torrentId;
$searchRep = new SearchRepository();
$result = $searchRep->updateTorrent($id);
do_log("result: " . var_export($result, true));
}
/**
* handle failed
*
* @param object $event
* @return void
*/
public function failed($event, \Throwable $exception)
{
$toolRep = new ToolRepository();
$to = Setting::get('main.SITEEMAIL');
$subject = sprintf('Event: %s listener: %s handle error', get_class($event), __CLASS__);
$body = sprintf("%s\n%s", $exception->getMessage(), $exception->getTraceAsString());
try {
$result = $toolRep->sendMail($to, $subject, $body);
if ($result === false) {
do_log("$subject send mail fail", 'alert');
}
} catch (\Throwable $exception) {
do_log("$subject send mail fail: " . $exception->getMessage() . $exception->getTraceAsString(), 'alert');
}
}
}
app/Models/Torrent.php
+24 -5
View File
@@ -6,10 +6,8 @@ use App\Repositories\TagRepository;
use JeroenG\Explorer\Application\Explored;
use Laravel\Scout\Searchable;
class Torrent extends NexusModel implements Explored
class Torrent extends NexusModel
{
use Searchable;
protected $fillable = [
'name', 'filename', 'save_as', 'descr', 'small_descr', 'ori_descr',
'category', 'source', 'medium', 'codec', 'standard', 'processing', 'team', 'audiocodec',
@@ -79,12 +77,22 @@ class Torrent extends NexusModel implements Explored
{
return [
'id' => 'long',
'name' => 'text',
'descr' => 'text',
'name' => [
'type' => 'text',
'analyzer' => 'ik_max_word',
],
'descr' => [
'type' => 'text',
'analyzer' => 'ik_max_word',
],
'owner' => 'long',
'source' => 'long',
'leechers' => 'long',
'seeders' => 'long',
'added' => 'date',
'owner_info' => [
]
];
}
@@ -94,6 +102,7 @@ class Torrent extends NexusModel implements Explored
'id' => $this->id,
'name' => $this->name,
'descr' => $this->descr,
'owner' => $this->owner,
'source' => $this->source,
'leechers' => $this->leechers,
'seeders' => $this->seeders,
@@ -196,6 +205,11 @@ class Torrent extends NexusModel implements Explored
return true;
}
public function bookmarks(): \Illuminate\Database\Eloquent\Relations\HasMany
{
return $this->hasMany(Bookmark::class, 'torrentid');
}
public function user()
{
return $this->belongsTo(User::class, 'owner')->withDefault(User::getDefaultUserAttributes());
@@ -296,6 +310,11 @@ class Torrent extends NexusModel implements Explored
$query->where('visible', $visible);
}
public function torrent_tags(): \Illuminate\Database\Eloquent\Relations\HasMany
{
return $this->hasMany(TorrentTag::class, 'torrent_id');
}
public function tags(): \Illuminate\Database\Eloquent\Relations\BelongsToMany
{
return $this->belongsToMany(Tag::class, 'torrent_tags', 'torrent_id', 'tag_id')
app/Models/User.php
+26
View File
@@ -79,6 +79,32 @@ class User extends Authenticatable
'invites' => '邀请',
];
public function mappableAs(): array
{
return [
'id' => 'long',
'username' => [
'type' => 'text',
'analyzer' => 'ik_max_word',
],
'email' => [
'type' => 'text',
'analyzer' => 'ik_max_word',
],
'added' => 'date',
];
}
public function toSearchableArray()
{
return [
'id' => $this->id,
'username' => $this->username,
'email' => $this->email,
'added' => $this->added,
];
}
public function getClassTextAttribute(): string
{
return self::$classes[$this->class]['text'] ?? '';
app/Providers/EventServiceProvider.php
+5
View File
@@ -2,6 +2,8 @@
namespace App\Providers;
use App\Events\TorrentUpdated;
use App\Listeners\SyncTorrentToEs;
use Illuminate\Auth\Events\Registered;
use Illuminate\Auth\Listeners\SendEmailVerificationNotification;
use Illuminate\Foundation\Support\Providers\EventServiceProvider as ServiceProvider;
@@ -18,6 +20,9 @@ class EventServiceProvider extends ServiceProvider
Registered::class => [
SendEmailVerificationNotification::class,
],
TorrentUpdated::class => [
SyncTorrentToEs::class,
],
];
/**
app/Repositories/SearchRepository.php
+826
View File
@@ -0,0 +1,826 @@
<?php
namespace App\Repositories;
use App\Models\Bookmark;
use App\Models\Setting;
use App\Models\Torrent;
use App\Models\TorrentTag;
use App\Models\User;
use Elasticsearch\Client;
use Elasticsearch\ClientBuilder;
use Illuminate\Support\Arr;
use Nexus\Database\NexusDB;
class SearchRepository extends BaseRepository
{
private Client $es;
const INDEX_NAME = 'nexus_torrents';
const DOC_TYPE_TORRENT = 'torrent';
const DOC_TYPE_TAG = 'tag';
const DOC_TYPE_BOOKMARK = 'bookmark';
const DOC_TYPE_USER = 'user';
const SEARCH_MODE_AND = '0';
const SEARCH_MODE_OR = '1';
const SEARCH_MODE_EXACT = '2';
const SEARCH_MODES = [
self::SEARCH_MODE_AND => ['text' => 'and'],
self::SEARCH_MODE_OR => ['text' => 'or'],
self::SEARCH_MODE_EXACT => ['text' => 'exact'],
];
const SEARCH_AREA_TITLE = '0';
const SEARCH_AREA_DESC = '1';
const SEARCH_AREA_OWNER = '3';
const SEARCH_AREA_IMDB = '4';
const SEARCH_AREAS = [
self::SEARCH_AREA_TITLE => ['text' => 'title'],
self::SEARCH_AREA_DESC => ['text' => 'desc'],
self::SEARCH_AREA_OWNER => ['text' => 'owner'],
self::SEARCH_AREA_IMDB => ['text' => 'imdb'],
];
private array $indexSetting = [
'index' => self::INDEX_NAME,
'body' => [
'settings' => [
'number_of_shards' => 1,
'number_of_replicas' => 0,
],
'mappings' => [
'properties' => [
'_doc_type' => ['type' => 'keyword'],
//torrent
'torrent_id' => ['type' => 'long', ],
//user
'username' => ['type' => 'text', 'analyzer' => 'ik_max_word', 'fields' => ['keyword' => ['type' => 'keyword', 'ignore_above' => 256]]],
//bookmark + user + tag
'user_id' => ['type' => 'long', ],
//tag
'tag_id' => ['type' => 'long', ],
//relations
'torrent_relations' => [
'type' => 'join',
'eager_global_ordinals' => true,
'relations' => [
'user' => ['torrent'],
'torrent' => ['bookmark', 'tag'],
],
],
],
]
],
];
//cat401=1&source1=1&medium1=1&codec1=1&audiocodec1=1&standard1=1&processing1=1&team1=1&incldead=1&spstate=2&inclbookmarked=1&search=tr&search_area=1&search_mode=1
private static array $queryFieldToTorrentFieldMaps = [
'cat' => 'category',
'source' => 'source',
'medium' => 'medium',
'codec' => 'codec',
'audiocodec' => 'audiocodec',
'standard' => 'standard',
'processing' => 'processing',
'team' => 'team',
];
private static array $sortFieldMaps = [
'1' => 'name',
'2' => 'numfiles',
'3' => 'comments',
'4' => 'added',
'5' => 'size',
'6' => 'times_completed',
'7' => 'seeders',
'8' => 'leechers',
'9' => 'owner',
];
public function __construct()
{
$this->es = $this->getEs();
}
private function getEs(): Client
{
$config = nexus_config('nexus.elasticsearch');
$es = ClientBuilder::create()->setHosts($config['hosts']);
if (!empty($config['ssl_verification'])) {
$es->setSSLVerification($config['ssl_verification']);
}
return $es->build();
}
private function getTorrentRawMappingFields(): array
{
return [
'name' => ['type' => 'text', 'analyzer' => 'ik_max_word', 'fields' => ['keyword' => ['type' => 'keyword', 'ignore_above' => 256]]],
'descr' => ['type' => 'text', 'analyzer' => 'ik_max_word', 'fields' => ['keyword' => ['type' => 'keyword', 'ignore_above' => 256]]],
'small_descr' => ['type' => 'text', 'analyzer' => 'ik_max_word', 'fields' => ['keyword' => ['type' => 'keyword', 'ignore_above' => 256]]],
'category' => ['type' => 'long', ],
'source' => ['type' => 'long', ],
'medium' => ['type' => 'long', ],
'codec' => ['type' => 'long', ],
'standard' => ['type' => 'long', ],
'processing' => ['type' => 'long', ],
'team' => ['type' => 'long', ],
'audiocodec' => ['type' => 'long', ],
'size' => ['type' => 'long', ],
'added' => ['type' => 'date', 'format' => 'yyyy-MM-dd HH:mm:ss'],
'numfiles' => ['type' => 'long', ],
'comments' => ['type' => 'long', ],
'views' => ['type' => 'long', ],
'hits' => ['type' => 'long', ],
'times_completed' => ['type' => 'long', ],
'leechers' => ['type' => 'long', ],
'seeders' => ['type' => 'long', ],
'last_action' => ['type' => 'date', 'format' => 'yyyy-MM-dd HH:mm:ss'],
'visible' => ['type' => 'keyword', ],
'banned' => ['type' => 'keyword', ],
'owner' => ['type' => 'long', ],
'sp_state' => ['type' => 'long', ],
'url' => ['type' => 'text', 'analyzer' => 'ik_max_word', 'fields' => ['keyword' => ['type' => 'keyword', 'ignore_above' => 256]]],
'pos_state' => ['type' => 'keyword', ],
'picktype' => ['type' => 'keyword', ],
'hr' => ['type' => 'long', ],
];
}
public function getEsInfo(): callable|array
{
return $this->es->info();
}
public function createIndex()
{
$params = $this->indexSetting;
$properties = $params['body']['mappings']['properties'];
$properties = array_merge($properties, $this->getTorrentRawMappingFields());
$params['body']['mappings']['properties'] = $properties;
return $this->es->indices()->create($params);
}
public function deleteIndex()
{
$params = ['index' => self::INDEX_NAME];
return $this->es->indices()->delete($params);
}
public function import($torrentId = null)
{
$page = 1;
$size = 1000;
$fields = $this->getTorrentBaseFields();
array_unshift($fields, 'id');
$query = Torrent::query()
->with(['user', 'torrent_tags', 'bookmarks'])
->select($fields);
if (!is_null($torrentId)) {
$idArr = preg_split('/[,\s]+/', $torrentId);
$query->whereIn('id', $idArr);
}
while (true) {
$log = "page: $page, size: $size";
$torrentResults = (clone $query)->forPage($page, $size)->get();
if ($torrentResults->isEmpty()) {
do_log("$log, no more data...", 'info', true);
break;
}
do_log("$log, get counts: " . $torrentResults->count(), 'info', true);
$torrentBodyBulk = $userBodyBulk = $tagBodyBulk = $bookmarkBodyBulk = ['body' => []];
foreach ($torrentResults as $torrent) {
$body = $this->buildUserBody($torrent->user, true);
$userBodyBulk['body'][] = ['index' => $body['index']];
$userBodyBulk['body'][] = $body['body'];
$body = $this->buildTorrentBody($torrent, true);
$torrentBodyBulk['body'][] = ['index' => $body['index']];
$torrentBodyBulk['body'][] = $body['body'];
foreach ($torrent->torrent_tags as $torrentTag) {
$body = $this->buildTorrentTagBody($torrent, $torrentTag, true);
$tagBodyBulk['body'][] = ['index' => $body['index']];
$tagBodyBulk['body'][] = $body['body'];
}
foreach ($torrent->bookmarks as $bookmark) {
$body = $this->buildBookmarkBody($torrent, $bookmark, true);
$bookmarkBodyBulk['body'][] = ['index' => $body['index']];
$bookmarkBodyBulk['body'][] = $body['body'];
}
}
//index user
$result = $this->es->bulk($userBodyBulk);
$this->logEsResponse("$log, bulk index user done!", $result);
//index torrent
$result = $this->es->bulk($torrentBodyBulk);
$this->logEsResponse("$log, bulk index torrent done!", $result);
//index tag
$result = $this->es->bulk($tagBodyBulk);
$this->logEsResponse("$log, bulk index tag done!", $result);
//index bookmark
$result = $this->es->bulk($bookmarkBodyBulk);
$this->logEsResponse("$log, bulk index bookmark done!", $result);
$page++;
}
}
private function buildUserBody(User $user, bool $underlinePrefix = false)
{
$docType = self::DOC_TYPE_USER;
$indexName = 'index';
$idName = 'id';
if ($underlinePrefix) {
$indexName = "_$indexName";
$idName = "_$idName";
}
$index = [
$indexName => self::INDEX_NAME,
$idName => $this->getUserId($user->id),
'routing' => $user->id,
];
$body = [
'_doc_type' => $docType,
'user_id' => $user->id,
'username' => $user->username,
'torrent_relations' => [
'name' => $docType,
],
];
return compact('index', 'body');
}
private function buildTorrentBody($torrent, bool $underlinePrefix = false): array
{
$baseFields = $this->getTorrentBaseFields();
if (!$torrent instanceof Torrent) {
$torrent = Torrent::query()->findOrFail((int)$torrent, array_merge(['id'], $baseFields));
}
$docType = self::DOC_TYPE_TORRENT;
$indexName = 'index';
$idName = 'id';
if ($underlinePrefix) {
$indexName = "_$indexName";
$idName = "_$idName";
}
$index = [
$indexName => self::INDEX_NAME,
$idName => $this->getTorrentId($torrent->id),
'routing' => $torrent->owner,
];
$data = Arr::only($torrent->toArray(), $baseFields);
$body = array_merge($data, [
'_doc_type' => $docType,
'torrent_id' => $torrent->id,
'torrent_relations' => [
'name' => $docType,
'parent' => 'user_' . $torrent->owner,
],
]);
return compact('index', 'body');
}
private function buildTorrentTagBody(Torrent $torrent, TorrentTag $torrentTag, bool $underlinePrefix = false)
{
$docType = self::DOC_TYPE_TAG;
$indexName = 'index';
$idName = 'id';
if ($underlinePrefix) {
$indexName = "_$indexName";
$idName = "_$idName";
}
$index = [
$indexName => self::INDEX_NAME,
$idName => $this->getTorrentTagId($torrentTag->id),
'routing' => $torrent->owner,
];
$body = [
'_doc_type' => $docType,
'torrent_id' => $torrentTag->torrent_id,
'tag_id' => $torrentTag->tag_id,
'torrent_relations' => [
'name' => $docType,
'parent' => 'torrent_' . $torrent->id,
],
];
return compact('index', 'body');
}
private function buildBookmarkBody(Torrent $torrent, Bookmark $bookmark, bool $underlinePrefix = false)
{
$docType = self::DOC_TYPE_BOOKMARK;
$indexName = 'index';
$idName = 'id';
if ($underlinePrefix) {
$indexName = "_$indexName";
$idName = "_$idName";
}
$index = [
$indexName => self::INDEX_NAME,
$idName => $this->getBookmarkId($bookmark->id),
'routing' => $torrent->owner,
];
$body = [
'_doc_type' => $docType,
'torrent_id' => $bookmark->torrentid,
'user_id' => $bookmark->userid,
'torrent_relations' => [
'name' => $docType,
'parent' => 'torrent_' . $torrent->id,
],
];
return compact('index', 'body');
}
private function logEsResponse($msg, $response)
{
if (isset($response['errors']) && $response['errors'] == true) {
$msg .= var_export($response, true);
}
do_log($msg, 'info', app()->runningInConsole());
}
private function getTorrentId($id): string
{
return "torrent_" . intval($id);
}
private function getTorrentTagId($id): string
{
return "torrent_tag_" . intval($id);
}
private function getUserId($id): string
{
return "user_" . intval($id);
}
private function getBookmarkId($id): string
{
return "bookmark_" . intval($id);
}
/**
* detect elastic response has error or not
*
* @param $esResponse
* @return bool
*/
private function isEsResponseError($esResponse)
{
if (isset($esResponse['error'])) {
return true;
}
//bulk insert
if (isset($esResponse['errors']) && $esResponse['errors']) {
return true;
}
//update by query
if (!empty($esResponse['failures'])) {
return true;
}
return false;
}
/**
* build es query
*
* @param array $params
* @param $user
* @param string $queryString cat401=1&cat404=1&source2=1&medium2=1&medium3=1&codec3=1&audiocodec3=1&standard2=1&standard3=1&processing2=1&team3=1&team4=1&incldead=1&spstate=0&inclbookmarked=0&search=&search_area=0&search_mode=0
* @return array
*/
public function buildQuery(array $params, $user, string $queryString)
{
if (!($user instanceof User) || !$user->torrentsperpage || !$user->notifs) {
$user = User::query()->findOrFail(intval($user));
}
//[cat401][cat404][sou1][med1][cod1][sta2][sta3][pro2][tea2][aud2][incldead=0][spstate=3][inclbookmarked=2]
$userSetting = $user->notifs;
$must = $must_not = [];
$mustBoolShould = [];
$must[] = ['match' => ['_doc_type' => self::DOC_TYPE_TORRENT]];
foreach (self::$queryFieldToTorrentFieldMaps as $queryField => $torrentField) {
if (isset($params[$queryField]) && $params[$queryField] !== '') {
$mustBoolShould[$torrentField][] = ['match' => [$torrentField => $params[$queryField]]];
do_log("get mustBoolShould for $torrentField from params through $queryField: {$params[$queryField]}");
} elseif (preg_match_all("/{$queryField}([\d]+)=/", $queryString, $matches)) {
if (count($matches) == 2 && !empty($matches[1])) {
foreach ($matches[1] as $match) {
$mustBoolShould[$torrentField][] = ['match' => [$torrentField => $match]];
do_log("get mustBoolShould for $torrentField from params through $queryField: $match");
}
}
} else {
//get user setting
$pattern = sprintf("/\[%s([\d]+)\]/", substr($queryField, 0, 3));
if (preg_match($pattern, $userSetting, $matches)) {
if (count($matches) == 2 && !empty($matches[1])) {
foreach ($matches[1] as $match) {
$mustBoolShould[$torrentField][] = ['match' => [$torrentField => $match]];
do_log("get mustBoolShould for $torrentField from user setting through $queryField: $match");
}
}
}
}
}
$includeDead = 1;
if (isset($params['incldead'])) {
$includeDead = (int)$params['incldead'];
do_log("maybe get must for visible from params");
} elseif (preg_match("/\[incldead=([\d]+)\]/", $userSetting, $matches)) {
$includeDead = $matches[1];
do_log("maybe get must for visible from user setting");
}
if ($includeDead == 1) {
//active torrent
$must[] = ['match' => ['visible' => 'yes']];
do_log("get must for visible = yes through incldead: $includeDead");
} elseif ($includeDead == 2) {
//dead torrent
$must[] = ['match' => ['visible' => 'no']];
do_log("get must for visible = no through incldead: $includeDead");
}
$includeBookmarked = 0;
if (isset($params['inclbookmarked'])) {
$includeBookmarked = (int)$params['inclbookmarked'];
do_log("maybe get must or must_not for has_child.bookmark from params");
} elseif (preg_match("/\[inclbookmarked=([\d]+)\]/", $userSetting, $matches)) {
$includeBookmarked = $matches[1];
do_log("maybe get must or must_not for has_child.bookmark from user setting");
}
if ($includeBookmarked == 1) {
//only bookmark
$must[] = ['has_child' => ['type' => 'bookmark', 'query' => ['match' => ['user_id' => $user->id]]]];
do_log("get must for has_child.bookmark through inclbookmarked: $includeBookmarked");
} elseif ($includeBookmarked == 2) {
//only not bookmark
$must_not[] = ['has_child' => ['type' => 'bookmark', 'query' => ['match' => ['user_id' => $user->id]]]];
do_log("get must_not for has_child.bookmark through inclbookmarked: $includeBookmarked");
}
$spState = 0;
if (isset($params['spstate'])) {
$spState = (int)$params['spstate'];
do_log("maybe get must for spstate from params");
} elseif (preg_match("/\[spstate=([\d]+)\]/", $userSetting, $matches)) {
$spState = $matches[1];
do_log("maybe get must for spstate from user setting");
}
if ($spState > 0) {
$must[] = ['match' => ['sp_state' => $spState]];
do_log("get must for sp_state = $spState through spstate: $spState");
}
if (!empty($params['tag_id'])) {
$must[] = ['has_child' => ['type' => 'tag', 'query' => ['match' => ['tag_id' => $params['tag_id']]]]];
do_log("get must for has_child.tag through params.tag_id: {$params['tag_id']}");
}
if (!empty($params['search'])) {
$searchMode = isset($params['search_mode']) && isset(self::SEARCH_MODES[$params['search_mode']]) ? $params['search_mode'] : self::SEARCH_MODE_AND;
if (in_array($searchMode, [self::SEARCH_MODE_AND, self::SEARCH_MODE_OR])) {
//and, or
$keywordsArr = preg_split("/[\.\s]+/", trim($params['search']));
} else {
$keywordsArr = [trim($params['search'])];
}
$keywordsArr = array_slice($keywordsArr, 0, 10);
$searchArea = isset($params['search_area']) && isset(self::SEARCH_AREAS[$params['search_area']]) ? $params['search_area'] : self::SEARCH_AREA_TITLE;
if ($searchMode == self::SEARCH_MODE_AND || $searchMode == self::SEARCH_MODE_EXACT) {
$keywordFlag = $searchMode == self::SEARCH_MODE_EXACT ? ".keyword" : "";
if ($searchArea == self::SEARCH_AREA_TITLE) {
foreach ($keywordsArr as $keyword) {
$tmpMustBoolShould = [];
$tmpMustBoolShould[] = ['match' => ["name{$keywordFlag}" => $keyword]];
$tmpMustBoolShould[] = ['match' => ["small_descr{$keywordFlag}" => $keyword]];
$must[]['bool']['should'] = $tmpMustBoolShould;
do_log("get must bool should [SEARCH_MODE_AND + SEARCH_MODE_EXACT] for name+small_descr match '$keyword' through search");
}
} elseif ($searchArea == self::SEARCH_AREA_DESC) {
foreach ($keywordsArr as $keyword) {
$must[] = ['match' => ["descr{$keywordFlag}" => $keyword]];
do_log("get must [SEARCH_MODE_AND + SEARCH_MODE_EXACT] for descr match '$keyword' through search");
}
} elseif ($searchArea == self::SEARCH_AREA_IMDB) {
foreach ($keywordsArr as $keyword) {
$must[] = ['match' => ["url{$keywordFlag}" => $keyword]];
do_log("get must [SEARCH_MODE_AND + SEARCH_MODE_EXACT] for url match '$keyword' through search");
}
} elseif ($searchArea == self::SEARCH_AREA_OWNER) {
foreach ($keywordsArr as $keyword) {
$must[] = ['has_parent' => ['parent_type' => 'user', 'query' => ['match' => ["username{$keywordFlag}" => $keyword]]]];
do_log("get must [SEARCH_MODE_AND + SEARCH_MODE_EXACT] has_parent.user match '$keyword' through search");
}
}
} elseif ($searchMode == self::SEARCH_MODE_OR) {
if ($searchArea == self::SEARCH_AREA_TITLE) {
$tmpMustBoolShould = [];
foreach ($keywordsArr as $keyword) {
$tmpMustBoolShould[] = ['match' => ['name' => $keyword]];
$tmpMustBoolShould[] = ['match' => ['small_descr' => $keyword]];
do_log("get must bool should [SEARCH_MODE_OR] for name+small_descr match '$keyword' through search");
}
$must[]['bool']['should'] = $tmpMustBoolShould;
} elseif ($searchArea == self::SEARCH_AREA_DESC) {
$tmpMustBoolShould = [];
foreach ($keywordsArr as $keyword) {
$tmpMustBoolShould[] = ['match' => ['descr' => $keyword]];
do_log("get must bool should [SEARCH_MODE_OR] for descr match '$keyword' through search");
}
$must[]['bool']['should'] = $tmpMustBoolShould;
} elseif ($searchArea == self::SEARCH_AREA_IMDB) {
$tmpMustBoolShould = [];
foreach ($keywordsArr as $keyword) {
$tmpMustBoolShould[] = ['match' => ['url' => $keyword]];
do_log("get must bool should [SEARCH_MODE_OR] for url match '$keyword' through search");
}
$must[]['bool']['should'] = $tmpMustBoolShould;
} elseif ($searchArea == self::SEARCH_AREA_OWNER) {
$tmpMustBoolShould = [];
foreach ($keywordsArr as $keyword) {
$tmpMustBoolShould[] = ['has_parent' => ['parent_type' => 'user', 'query' => ['match' => ['username' => $keyword]]]];
do_log("get must bool should [SEARCH_MODE_OR] has_parent.user match '$keyword' through search");
}
$must[]['bool']['should'] = $tmpMustBoolShould;
}
}
}
$query = [
'bool' => [
'must' => $must
]
];
foreach ($mustBoolShould as $torrentField => $boolShoulds) {
$query['bool']['must'][]['bool']['should'] = $boolShoulds;
}
if (!empty($must_not)) {
$query['bool']['must_not'] = $must_not;
}
$sort = [];
$sort[] = ['pos_state' => ['order' => 'desc']];
$hasAddSetSortField = false;
if (!empty($params['sort'])) {
$direction = isset($params['type']) && in_array($params['type'], ['asc', 'desc']) ? $params['type'] : 'desc';
foreach (self::$sortFieldMaps as $key => $value) {
if ($key == $params['sort']) {
$hasAddSetSortField = true;
$sort[] = [$value => ['order' => $direction]];
}
}
}
if (!$hasAddSetSortField) {
$sort[] = ['torrent_id' => ['order' => 'desc']];
}
$page = isset($params['page']) && is_numeric($params['page']) ? $params['page'] : 0;
if ($user->torrentsperpage) {
$size = $user->torrentsperpage;
} elseif (($sizeFromConfig = Setting::get('main.torrentsperpage')) > 0) {
$size = $sizeFromConfig;
} else {
$size = 50;
}
$size = min($size, 200);
$offset = $page * $size;
$result = [
'query' => $query,
'sort' => $sort,
'from' => $offset,
'size' => $size,
'_source' => ['torrent_id', 'name', 'small_descr', 'owner']
];
do_log(sprintf(
"params: %s, user: %s, queryString: %s, result: %s",
nexus_json_encode($params), $user->id, $queryString, nexus_json_encode($result)
));
return $result;
}
public function listTorrentFromEs(array $params, $user, string $queryString)
{
$query = $this->buildQuery($params, $user, $queryString);
$esParams = [
'index' => self::INDEX_NAME,
'body' => $query,
];
$response = $this->es->search($esParams);
$result = [
'total' => 0,
'data' => [],
];
if (isset($response['errors']) && $response['errors']) {
do_log("error response: " . nexus_json_encode($response), 'error');
return $result;
}
if (empty($response['hits'])) {
do_log("empty response hits: " . nexus_json_encode($response), 'error');
return $result;
}
if ($response['hits']['total']['value'] == 0) {
do_log("total = 0, " . nexus_json_encode($response));
return $result;
}
$result['total'] = $response['hits']['total']['value'];
$torrentIdArr = [];
foreach ($response['hits']['hits'] as $value) {
$torrentIdArr[] = $value['_source']['torrent_id'];
}
$fieldStr = 'id, sp_state, promotion_time_type, promotion_until, banned, picktype, pos_state, category, source, medium, codec, standard, processing, team, audiocodec, leechers, seeders, name, small_descr, times_completed, size, added, comments,anonymous,owner,url,cache_stamp, pt_gen, hr';
$idStr = implode(',', $torrentIdArr);
$result['data'] = Torrent::query()
->selectRaw($fieldStr)
->whereIn('id', $torrentIdArr)
->orderByRaw("field(id,$idStr)")
->get()
->toArray()
;
return $result;
}
private function getTorrentBaseFields()
{
return array_keys($this->getTorrentRawMappingFields());
}
public function updateTorrent(int $id): bool
{
$log = "update torrent: $id";
$baseFields = $this->getTorrentBaseFields();
$torrent = Torrent::query()->findOrFail($id, array_merge(['id'], $baseFields));
$data = $this->buildTorrentBody($torrent);
$params = $data['index'];
$params['body']['doc'] = $data['body'];
$result = $this->es->update($params);
if ($this->isEsResponseError($result)) {
do_log("$log, update torrent fail: " . nexus_json_encode($result), 'error');
return false;
}
do_log("$log, update torrent success: " . nexus_json_encode($result));
return $this->syncTorrentTags($torrent);
}
public function addTorrent(int $id): bool
{
$log = "add torrent: $id";
$baseFields = $this->getTorrentBaseFields();
$torrent = Torrent::query()->findOrFail($id, array_merge(['id'], $baseFields));
$data = $this->buildTorrentBody($torrent, true);
$params = ['body' => []];
$params['body'][] = ['index' => $data['index']];
$params['body'][] = $data['body'];
$result = $this->es->bulk($params);
if ($this->isEsResponseError($result)) {
do_log("$log, add torrent fail: " . nexus_json_encode($result), 'error');
return false;
}
do_log("$log, add torrent success: " . nexus_json_encode($result));
return $this->syncTorrentTags($torrent);
}
public function syncTorrentTags($torrent): bool
{
if (!$torrent instanceof Torrent) {
$torrent = Torrent::query()->findOrFail((int)$torrent, ['id']);
}
$log = "sync torrent tags, torrent: " . $torrent->id;
//remove first
$params = [
'index' => self::INDEX_NAME,
'body' => [
'query' => [
'bool' => [
'must' => [
['match' => ['_doc_type' => self::DOC_TYPE_TAG]],
['has_parent' => ['parent_type' => 'torrent', 'query' => ['match' => ['torrent_id' => $torrent->id]]]]
]
]
]
]
];
$result = $this->es->deleteByQuery($params);
if ($this->isEsResponseError($result)) {
do_log("$log, delete torrent tag fail: " . nexus_json_encode($result), 'error');
return false;
}
do_log("$log, delete torrent tag success: " . nexus_json_encode($result));
//then insert new
$bulk = ['body' => []];
foreach ($torrent->torrent_tags as $torrentTag) {
$body = $this->buildTorrentTagBody($torrent, $torrentTag, true);
$bulk['body'][] = ['index' => $body['index']];
$bulk['body'][] = $body['body'];
}
if (empty($bulk['body'])) {
do_log("$log, no tags, return true");
return true;
}
$result = $this->es->bulk($bulk);
if ($this->isEsResponseError($result)) {
do_log("$log, insert torrent tag fail: " . nexus_json_encode($result), 'error');
return false;
}
do_log("$log, insert torrent tag success: " . nexus_json_encode($result));
return true;
}
public function updateUser($user): bool
{
if (!$user instanceof User) {
$user = User::query()->findOrFail((int)$user, ['id', 'username']);
}
$log = "update user: " . $user->id;
$data = $this->buildUserBody($user);
$params = $data['index'];
$params['body']['doc'] = $data['body'];
$result = $this->es->update($params);
if ($this->isEsResponseError($result)) {
do_log("$log, update user fail: " . nexus_json_encode($result), 'error');
return false;
}
do_log("$log, update user success: " . nexus_json_encode($result));
return true;
}
public function addBookmark($bookmark): bool
{
if (!$bookmark instanceof Bookmark) {
$bookmark = Bookmark::query()->with('torrent')->findOrFail((int)$bookmark);
}
$log = "add bookmark: " . $bookmark->toJson();
$bulk = ['body' => []];
$body = $this->buildBookmarkBody($bookmark->torrent, $bookmark);
$bulk['body'][] = ['index' => $body['index']];
$bulk['body'][] = $body['body'];
$result = $this->es->bulk($bulk);
if ($this->isEsResponseError($result)) {
do_log("$log, add bookmark fail: " . nexus_json_encode($result), 'error');
return false;
}
do_log("$log, add bookmark success: " . nexus_json_encode($result));
return true;
}
public function deleteBookmark($bookmark): bool
{
if (!$bookmark instanceof Bookmark) {
$bookmark = Bookmark::query()->with('torrent')->findOrFail((int)$bookmark);
}
$log = "add bookmark: " . $bookmark->toJson();
$params = [
'index' => self::INDEX_NAME,
'id' => $this->getBookmarkId($bookmark->id),
];
$result = $this->es->delete($params);
if ($this->isEsResponseError($result)) {
do_log("$log, delete bookmark fail: " . nexus_json_encode($result), 'error');
return false;
}
do_log("$log, delete bookmark success: " . nexus_json_encode($result));
return true;
}
}
app/Repositories/ToolRepository.php
+44
View File
@@ -148,4 +148,48 @@ class ToolRepository extends BaseRepository
{
return new Encrypter($key, 'AES-256-CBC');
}
/**
* @param $to
* @param $subject
* @param $body
* @return bool
*/
public function sendMail($to, $subject, $body): bool
{
do_log("to: $to, subject: $subject, body: $body");
$smtp = Setting::get('smtp');
// Create the Transport
$encryption = null;
if (isset($smtp['encryption']) && in_array($smtp['encryption'], ['ssl', 'tls'])) {
$encryption = $smtp['encryption'];
}
$transport = (new \Swift_SmtpTransport($smtp['smtpaddress'], $smtp['smtpport'], $encryption))
->setUsername($smtp['accountname'])
->setPassword($smtp['accountpassword'])
;
// Create the Mailer using your created Transport
$mailer = new \Swift_Mailer($transport);
// Create a message
$message = (new \Swift_Message($subject))
->setFrom($smtp['accountname'], Setting::get('basic.SITENAME'))
->setTo([$to])
->setBody($body, 'text/html')
;
// Send the message
try {
$result = $mailer->send($message);
if ($result == 0) {
do_log("send mail fail, unknown error", 'error');
return false;
}
return true;
} catch (\Exception $e) {
do_log("send email fail: " . $e->getMessage() . "\n" . $e->getTraceAsString(), 'error');
return false;
}
}
}
config/elastic.php
+16
View File
@@ -0,0 +1,16 @@
<?php
return [
'hosts' => [
[
'host' => env('ELASTICSEARCH_HOST','localhost'),
'port' => env('ELASTICSEARCH_PORT','9200'),
'scheme' => env('ELASTICSEARCH_SCHEME','https'),
'user' => env('ELASTICSEARCH_USER','elastic'),
'pass' => env('ELASTICSEARCH_PASS',''),
]
],
'ssl_verification' => env('ELASTICSEARCH_SSL_VERIFICATION', ''),
];
config/explorer.php
+2 -1
View File
@@ -22,7 +22,8 @@ return [
* of the mapping possibilities can be found in the documentation of Explorer's repository.
*/
'indexes' => [
\App\Models\Torrent::class
\App\Models\Torrent::class,
\App\Models\User::class,
],
/**
config/nexus.php
+14
View File
@@ -24,4 +24,18 @@ return [
'database' => nexus_env('REDIS_DB', 0),
],
'elasticsearch' => [
'hosts' => [
[
'host' => nexus_env('ELASTICSEARCH_HOST','localhost'),
'port' => nexus_env('ELASTICSEARCH_PORT','9200'),
'scheme' => nexus_env('ELASTICSEARCH_SCHEME','https'),
'user' => nexus_env('ELASTICSEARCH_USER','elastic'),
'pass' => nexus_env('ELASTICSEARCH_PASS',''),
]
],
'ssl_verification' => nexus_env('ELASTICSEARCH_SSL_VERIFICATION', ''),
]
];
config/queue.php
+2 -2
View File
@@ -13,7 +13,7 @@ return [
|
*/
'default' => env('QUEUE_CONNECTION', 'sync'),
'default' => env('QUEUE_CONNECTION', 'redis'),
/*
|--------------------------------------------------------------------------
@@ -68,7 +68,7 @@ return [
'queue' => env('REDIS_QUEUE', 'default'),
'retry_after' => 90,
'block_for' => null,
'after_commit' => false,
'after_commit' => true,
],
],
include/globalfunctions.php
+4 -1
View File
@@ -156,7 +156,7 @@ function nexus_dd($vars)
* @param $log
* @param string $level
*/
function do_log($log, $level = 'info')
function do_log($log, $level = 'info', $echo = false)
{
static $env, $setLogLevel;
if (is_null($setLogLevel)) {
@@ -211,6 +211,9 @@ function do_log($log, $level = 'info')
);
fwrite($fd, $content);
fclose($fd);
if (is_bool($echo) && $echo) {
echo $content . PHP_EOL;
}
if (nexus()) {
nexus()->incrementLogSequence();
}
public/takeedit.php
+1
View File
@@ -243,6 +243,7 @@ else
{
write_log("Torrent $id ($name) was edited by {$CURUSER['username']}, Mod Edit" . $pick_info . $place_info);
}
\App\Events\TorrentUpdated::dispatch($id);
$returl = "details.php?id=$id&edited=1";
if (isset($_POST["returnto"]))
$returl = $_POST["returnto"];
public/takeupload.php
+2
View File
@@ -372,6 +372,8 @@ KPS("+",$uploadtorrent_bonus,$CURUSER["id"]);
write_log("Torrent $id ($torrent) was uploaded by $anon");
\App\Events\TorrentUpdated::dispatch($id);
//===notify people who voted on offer thanks CoLdFuSiOn :)
if ($is_offer)
{
public/torrents.php
+29 -15
View File
@@ -12,6 +12,7 @@ parked();
$tagRep = new \App\Repositories\TagRepository();
$tagKeyById = $tagRep->createBasicQuery()->get()->keyBy('id');
$renderKeyArr = $tagKeyById->keys()->toArray();
$elasticsearchEnabled = nexus_env('ELASTICSEARCH_ENABLED');
//check searchbox
switch (nexus()->getScript()) {
@@ -847,10 +848,17 @@ else
$sql = "SELECT COUNT(*), categories.mode FROM torrents LEFT JOIN categories ON category = categories.id " . ($search_area == 3 || $column == "owner" ? "LEFT JOIN users ON torrents.owner = users.id " : "") . $tagFilter . $where . " GROUP BY categories.mode";
}
$res = sql_query($sql);
$count = 0;
while($row = mysql_fetch_array($res))
$count += $row[0];
if ($elasticsearchEnabled) {
$searchRep = new \App\Repositories\SearchRepository();
$resultFromElastic = $searchRep->listTorrentFromEs($_GET, $CURUSER['id'], $_SERVER['QUERY_STRING']);
$count = $resultFromElastic['total'];
} else {
$res = sql_query($sql);
$count = 0;
while($row = mysql_fetch_array($res)) {
$count += $row[0];
}
}
if ($CURUSER["torrentsperpage"])
$torrentsperpage = (int)$CURUSER["torrentsperpage"];
@@ -883,17 +891,19 @@ if ($count)
//echo $addparam;
list($pagertop, $pagerbottom, $limit) = pager($torrentsperpage, $count, "?" . $addparam);
if ($allsec == 1 || $enablespecial != 'yes'){
$query = "SELECT torrents.id, torrents.sp_state, torrents.promotion_time_type, torrents.promotion_until, torrents.banned, torrents.picktype, torrents.pos_state, torrents.category, torrents.source, torrents.medium, torrents.codec, torrents.standard, torrents.processing, torrents.team, torrents.audiocodec, torrents.leechers, torrents.seeders, torrents.name, torrents.small_descr, torrents.times_completed, torrents.size, torrents.added, torrents.comments,torrents.anonymous,torrents.owner,torrents.url,torrents.cache_stamp,torrents.pt_gen,torrents.hr FROM torrents ".($search_area == 3 || $column == "owner" ? "LEFT JOIN users ON torrents.owner = users.id " : "")." $tagFilter $where $orderby $limit";
}
else{
$query = "SELECT torrents.id, torrents.sp_state, torrents.promotion_time_type, torrents.promotion_until, torrents.banned, torrents.picktype, torrents.pos_state, torrents.category, torrents.source, torrents.medium, torrents.codec, torrents.standard, torrents.processing, torrents.team, torrents.audiocodec, torrents.leechers, torrents.seeders, torrents.name, torrents.small_descr, torrents.times_completed, torrents.size, torrents.added, torrents.comments,torrents.anonymous,torrents.owner,torrents.url,torrents.cache_stamp,torrents.pt_gen,torrents.hr FROM torrents ".($search_area == 3 || $column == "owner" ? "LEFT JOIN users ON torrents.owner = users.id " : "")." LEFT JOIN categories ON torrents.category=categories.id $tagFilter $where $orderby $limit";
if ($allsec == 1 || $enablespecial != 'yes'){
$query = "SELECT torrents.id, torrents.sp_state, torrents.promotion_time_type, torrents.promotion_until, torrents.banned, torrents.picktype, torrents.pos_state, torrents.category, torrents.source, torrents.medium, torrents.codec, torrents.standard, torrents.processing, torrents.team, torrents.audiocodec, torrents.leechers, torrents.seeders, torrents.name, torrents.small_descr, torrents.times_completed, torrents.size, torrents.added, torrents.comments,torrents.anonymous,torrents.owner,torrents.url,torrents.cache_stamp,torrents.pt_gen,torrents.hr FROM torrents ".($search_area == 3 || $column == "owner" ? "LEFT JOIN users ON torrents.owner = users.id " : "")." $tagFilter $where $orderby $limit";
}
else{
$query = "SELECT torrents.id, torrents.sp_state, torrents.promotion_time_type, torrents.promotion_until, torrents.banned, torrents.picktype, torrents.pos_state, torrents.category, torrents.source, torrents.medium, torrents.codec, torrents.standard, torrents.processing, torrents.team, torrents.audiocodec, torrents.leechers, torrents.seeders, torrents.name, torrents.small_descr, torrents.times_completed, torrents.size, torrents.added, torrents.comments,torrents.anonymous,torrents.owner,torrents.url,torrents.cache_stamp,torrents.pt_gen,torrents.hr FROM torrents ".($search_area == 3 || $column == "owner" ? "LEFT JOIN users ON torrents.owner = users.id " : "")." LEFT JOIN categories ON torrents.category=categories.id $tagFilter $where $orderby $limit";
}
if (!$elasticsearchEnabled) {
$res = sql_query($query);
}
} else {
unset($res);
}
$res = sql_query($query);
}
else
unset($res);
if (isset($searchstr))
stdhead($lang_torrents['head_search_results_for'].$searchstr_ori);
elseif ($sectiontype == $browsecatmode)
@@ -1120,8 +1130,12 @@ elseif($inclbookmarked == 2)
if ($count) {
$rows = [];
while ($row = mysql_fetch_assoc($res)) {
$rows[] = $row;
if ($elasticsearchEnabled) {
$rows = $resultFromElastic['data'];
} else {
while ($row = mysql_fetch_assoc($res)) {
$rows[] = $row;
}
}
print($pagertop);
if ($sectiontype == $browsecatmode)