diff --git a/app/Book.php b/app/Book.php index 91f74ca64..06c00945d 100644 --- a/app/Book.php +++ b/app/Book.php @@ -56,4 +56,13 @@ class Book extends Entity return strlen($description) > $length ? substr($description, 0, $length-3) . '...' : $description; } + /** + * Return a generalised, common raw query that can be 'unioned' across entities. + * @return string + */ + public function entityRawQuery() + { + return "'BookStack\\\\Book' as entity_type, id, id as entity_id, slug, name, {$this->textField} as text,'' as html, '0' as book_id, '0' as priority, '0' as chapter_id, '0' as draft, created_by, updated_by, updated_at, created_at"; + } + } diff --git a/app/Chapter.php b/app/Chapter.php index dc23f5ebd..b08cb913a 100644 --- a/app/Chapter.php +++ b/app/Chapter.php @@ -51,4 +51,13 @@ class Chapter extends Entity return strlen($description) > $length ? substr($description, 0, $length-3) . '...' : $description; } + /** + * Return a generalised, common raw query that can be 'unioned' across entities. + * @return string + */ + public function entityRawQuery() + { + return "'BookStack\\\\Chapter' as entity_type, id, id as entity_id, slug, name, {$this->textField} as text, '' as html, book_id, priority, '0' as chapter_id, '0' as draft, created_by, updated_by, updated_at, created_at"; + } + } diff --git a/app/Console/Commands/RegenerateSearch.php b/app/Console/Commands/RegenerateSearch.php new file mode 100644 index 000000000..ccc2a20e5 --- /dev/null +++ b/app/Console/Commands/RegenerateSearch.php @@ -0,0 +1,46 @@ +searchService = $searchService; + } + + /** + * Execute the console command. + * + * @return mixed + */ + public function handle() + { + $this->searchService->indexAllEntities(); + } +} diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php index 0112e72ca..4fa0b3c80 100644 --- a/app/Console/Kernel.php +++ b/app/Console/Kernel.php @@ -1,6 +1,4 @@ -morphMany(Tag::class, 'entity')->orderBy('order', 'asc'); } + /** + * Get the related search terms. + * @return \Illuminate\Database\Eloquent\Relations\MorphMany + */ + public function searchTerms() + { + return $this->morphMany(SearchTerm::class, 'entity'); + } + /** * Get this entities restrictions. */ @@ -152,11 +161,26 @@ class Entity extends Ownable return substr($this->name, 0, $length - 3) . '...'; } + /** + * Get the body text of this entity. + * @return mixed + */ + public function getText() + { + return $this->{$this->textField}; + } + + /** + * Return a generalised, common raw query that can be 'unioned' across entities. + * @return string + */ + public function entityRawQuery(){return '';} + /** * Perform a full-text search on this entity. - * @param string[] $fieldsToSearch * @param string[] $terms * @param string[] array $wheres + * TODO - REMOVE * @return mixed */ public function fullTextSearchQuery($terms, $wheres = []) @@ -178,21 +202,21 @@ class Entity extends Ownable } $isFuzzy = count($exactTerms) === 0 && count($fuzzyTerms) > 0; - + $fieldsToSearch = ['name', $this->textField]; // Perform fulltext search if relevant terms exist. if ($isFuzzy) { $termString = implode(' ', $fuzzyTerms); - $fields = implode(',', $this->fieldsToSearch); + $search = $search->selectRaw('*, MATCH(name) AGAINST(? IN BOOLEAN MODE) AS title_relevance', [$termString]); - $search = $search->whereRaw('MATCH(' . $fields . ') AGAINST(? IN BOOLEAN MODE)', [$termString]); + $search = $search->whereRaw('MATCH(' . implode(',', $fieldsToSearch ). ') AGAINST(? IN BOOLEAN MODE)', [$termString]); } // Ensure at least one exact term matches if in search if (count($exactTerms) > 0) { - $search = $search->where(function ($query) use ($exactTerms) { + $search = $search->where(function ($query) use ($exactTerms, $fieldsToSearch) { foreach ($exactTerms as $exactTerm) { - foreach ($this->fieldsToSearch as $field) { + foreach ($fieldsToSearch as $field) { $query->orWhere($field, 'like', $exactTerm); } } diff --git a/app/Http/Controllers/SearchController.php b/app/Http/Controllers/SearchController.php index 37aaccece..dca5fa0af 100644 --- a/app/Http/Controllers/SearchController.php +++ b/app/Http/Controllers/SearchController.php @@ -1,6 +1,7 @@ entityRepo = $entityRepo; $this->viewService = $viewService; + $this->searchService = $searchService; parent::__construct(); } @@ -33,15 +37,13 @@ class SearchController extends Controller return redirect()->back(); } $searchTerm = $request->get('term'); - $paginationAppends = $request->only('term'); - $pages = $this->entityRepo->getBySearch('page', $searchTerm, [], 20, $paginationAppends); - $books = $this->entityRepo->getBySearch('book', $searchTerm, [], 10, $paginationAppends); - $chapters = $this->entityRepo->getBySearch('chapter', $searchTerm, [], 10, $paginationAppends); +// $paginationAppends = $request->only('term'); TODO - Check pagination $this->setPageTitle(trans('entities.search_for_term', ['term' => $searchTerm])); + + $entities = $this->searchService->searchEntities($searchTerm); + return view('search/all', [ - 'pages' => $pages, - 'books' => $books, - 'chapters' => $chapters, + 'entities' => $entities, 'searchTerm' => $searchTerm ]); } diff --git a/app/Page.php b/app/Page.php index b24e7778a..d53e02a0b 100644 --- a/app/Page.php +++ b/app/Page.php @@ -8,8 +8,7 @@ class Page extends Entity protected $simpleAttributes = ['name', 'id', 'slug']; protected $with = ['book']; - - protected $fieldsToSearch = ['name', 'text']; + protected $textField = 'text'; /** * Converts this page into a simplified array. @@ -96,4 +95,14 @@ class Page extends Entity return mb_convert_encoding($text, 'UTF-8'); } + /** + * Return a generalised, common raw query that can be 'unioned' across entities. + * @param bool $withContent + * @return string + */ + public function entityRawQuery($withContent = false) + { $htmlQuery = $withContent ? 'html' : "'' as html"; + return "'BookStack\\\\Page' as entity_type, id, id as entity_id, slug, name, {$this->textField} as text, {$htmlQuery}, book_id, priority, chapter_id, draft, created_by, updated_by, updated_at, created_at"; + } + } diff --git a/app/Repos/EntityRepo.php b/app/Repos/EntityRepo.php index 4db69137f..8e3f68859 100644 --- a/app/Repos/EntityRepo.php +++ b/app/Repos/EntityRepo.php @@ -8,6 +8,7 @@ use BookStack\Page; use BookStack\PageRevision; use BookStack\Services\AttachmentService; use BookStack\Services\PermissionService; +use BookStack\Services\SearchService; use BookStack\Services\ViewService; use Carbon\Carbon; use DOMDocument; @@ -58,6 +59,11 @@ class EntityRepo */ protected $tagRepo; + /** + * @var SearchService + */ + protected $searchService; + /** * Acceptable operators to be used in a query * @var array @@ -65,7 +71,7 @@ class EntityRepo protected $queryOperators = ['<=', '>=', '=', '<', '>', 'like', '!=']; /** - * EntityService constructor. + * EntityRepo constructor. * @param Book $book * @param Chapter $chapter * @param Page $page @@ -73,10 +79,12 @@ class EntityRepo * @param ViewService $viewService * @param PermissionService $permissionService * @param TagRepo $tagRepo + * @param SearchService $searchService */ public function __construct( Book $book, Chapter $chapter, Page $page, PageRevision $pageRevision, - ViewService $viewService, PermissionService $permissionService, TagRepo $tagRepo + ViewService $viewService, PermissionService $permissionService, + TagRepo $tagRepo, SearchService $searchService ) { $this->book = $book; @@ -91,6 +99,7 @@ class EntityRepo $this->viewService = $viewService; $this->permissionService = $permissionService; $this->tagRepo = $tagRepo; + $this->searchService = $searchService; } /** @@ -608,6 +617,7 @@ class EntityRepo $entity->updated_by = user()->id; $isChapter ? $book->chapters()->save($entity) : $entity->save(); $this->permissionService->buildJointPermissionsForEntity($entity); + $this->searchService->indexEntity($entity); return $entity; } @@ -628,6 +638,7 @@ class EntityRepo $entityModel->updated_by = user()->id; $entityModel->save(); $this->permissionService->buildJointPermissionsForEntity($entityModel); + $this->searchService->indexEntity($entityModel); return $entityModel; } @@ -961,6 +972,8 @@ class EntityRepo $this->savePageRevision($page, $input['summary']); } + $this->searchService->indexEntity($page); + return $page; } @@ -1064,6 +1077,7 @@ class EntityRepo $page->text = strip_tags($page->html); $page->updated_by = user()->id; $page->save(); + $this->searchService->indexEntity($page); return $page; } @@ -1156,6 +1170,7 @@ class EntityRepo $book->views()->delete(); $book->permissions()->delete(); $this->permissionService->deleteJointPermissionsForEntity($book); + $this->searchService->deleteEntityTerms($book); $book->delete(); } @@ -1175,6 +1190,7 @@ class EntityRepo $chapter->views()->delete(); $chapter->permissions()->delete(); $this->permissionService->deleteJointPermissionsForEntity($chapter); + $this->searchService->deleteEntityTerms($chapter); $chapter->delete(); } @@ -1190,6 +1206,7 @@ class EntityRepo $page->revisions()->delete(); $page->permissions()->delete(); $this->permissionService->deleteJointPermissionsForEntity($page); + $this->searchService->deleteEntityTerms($page); // Delete Attached Files $attachmentService = app(AttachmentService::class); diff --git a/app/SearchTerm.php b/app/SearchTerm.php new file mode 100644 index 000000000..a7e3814f8 --- /dev/null +++ b/app/SearchTerm.php @@ -0,0 +1,20 @@ +morphTo('entity'); + } + +} diff --git a/app/Services/PermissionService.php b/app/Services/PermissionService.php index 8b47e1246..616c81443 100644 --- a/app/Services/PermissionService.php +++ b/app/Services/PermissionService.php @@ -479,8 +479,7 @@ class PermissionService * @return \Illuminate\Database\Query\Builder */ public function bookChildrenQuery($book_id, $filterDrafts = false, $fetchPageContent = false) { - $pageContentSelect = $fetchPageContent ? 'html' : "''"; - $pageSelect = $this->db->table('pages')->selectRaw("'BookStack\\\\Page' as entity_type, id, slug, name, text, {$pageContentSelect} as description, book_id, priority, chapter_id, draft")->where('book_id', '=', $book_id)->where(function($query) use ($filterDrafts) { + $pageSelect = $this->db->table('pages')->selectRaw($this->page->entityRawQuery($fetchPageContent))->where('book_id', '=', $book_id)->where(function($query) use ($filterDrafts) { $query->where('draft', '=', 0); if (!$filterDrafts) { $query->orWhere(function($query) { @@ -488,7 +487,7 @@ class PermissionService }); } }); - $chapterSelect = $this->db->table('chapters')->selectRaw("'BookStack\\\\Chapter' as entity_type, id, slug, name, '' as text, description, book_id, priority, 0 as chapter_id, 0 as draft")->where('book_id', '=', $book_id); + $chapterSelect = $this->db->table('chapters')->selectRaw($this->chapter->entityRawQuery())->where('book_id', '=', $book_id); $query = $this->db->query()->select('*')->from($this->db->raw("({$pageSelect->toSql()} UNION {$chapterSelect->toSql()}) AS U")) ->mergeBindings($pageSelect)->mergeBindings($chapterSelect); @@ -540,7 +539,7 @@ class PermissionService } /** - * Filter items that have entities set a a polymorphic relation. + * Filter items that have entities set as a polymorphic relation. * @param $query * @param string $tableName * @param string $entityIdColumn diff --git a/app/Services/SearchService.php b/app/Services/SearchService.php new file mode 100644 index 000000000..be1303ca0 --- /dev/null +++ b/app/Services/SearchService.php @@ -0,0 +1,150 @@ +searchTerm = $searchTerm; + $this->book = $book; + $this->chapter = $chapter; + $this->page = $page; + $this->db = $db; + } + + public function searchEntities($searchString, $entityType = 'all') + { + // TODO - Add Tag Searches + // TODO - Add advanced custom column searches + // TODO - Add exact match searches ("") + + $termArray = explode(' ', $searchString); + + $subQuery = $this->db->table('search_terms')->select('entity_id', 'entity_type', \DB::raw('SUM(score) as score')); + $subQuery->where(function($query) use ($termArray) { + foreach ($termArray as $inputTerm) { + $query->orWhere('term', 'like', $inputTerm .'%'); + } + }); + + $subQuery = $subQuery->groupBy('entity_type', 'entity_id'); + $pageSelect = $this->db->table('pages as e')->join(\DB::raw('(' . $subQuery->toSql() . ') as s'), function(JoinClause $join) { + $join->on('e.id', '=', 's.entity_id'); + })->selectRaw('e.*, s.score')->orderBy('score', 'desc'); + $pageSelect->mergeBindings($subQuery); + dd($pageSelect->toSql()); + // TODO - Continue from here + } + + /** + * Index the given entity. + * @param Entity $entity + */ + public function indexEntity(Entity $entity) + { + $this->deleteEntityTerms($entity); + $nameTerms = $this->generateTermArrayFromText($entity->name, 5); + $bodyTerms = $this->generateTermArrayFromText($entity->getText(), 1); + $terms = array_merge($nameTerms, $bodyTerms); + $entity->searchTerms()->createMany($terms); + } + + /** + * Index multiple Entities at once + * @param Entity[] $entities + */ + protected function indexEntities($entities) { + $terms = []; + foreach ($entities as $entity) { + $nameTerms = $this->generateTermArrayFromText($entity->name, 5); + $bodyTerms = $this->generateTermArrayFromText($entity->getText(), 1); + foreach (array_merge($nameTerms, $bodyTerms) as $term) { + $term['entity_id'] = $entity->id; + $term['entity_type'] = $entity->getMorphClass(); + $terms[] = $term; + } + } + $this->searchTerm->insert($terms); + } + + /** + * Delete and re-index the terms for all entities in the system. + */ + public function indexAllEntities() + { + $this->searchTerm->truncate(); + + // Chunk through all books + $this->book->chunk(500, function ($books) { + $this->indexEntities($books); + }); + + // Chunk through all chapters + $this->chapter->chunk(500, function ($chapters) { + $this->indexEntities($chapters); + }); + + // Chunk through all pages + $this->page->chunk(500, function ($pages) { + $this->indexEntities($pages); + }); + } + + /** + * Delete related Entity search terms. + * @param Entity $entity + */ + public function deleteEntityTerms(Entity $entity) + { + $entity->searchTerms()->delete(); + } + + /** + * Create a scored term array from the given text. + * @param $text + * @param float|int $scoreAdjustment + * @return array + */ + protected function generateTermArrayFromText($text, $scoreAdjustment = 1) + { + $tokenMap = []; // {TextToken => OccurrenceCount} + $splitText = explode(' ', $text); + foreach ($splitText as $token) { + if ($token === '') continue; + if (!isset($tokenMap[$token])) $tokenMap[$token] = 0; + $tokenMap[$token]++; + } + + $terms = []; + foreach ($tokenMap as $token => $count) { + $terms[] = [ + 'term' => $token, + 'score' => $count * $scoreAdjustment + ]; + } + return $terms; + } + +} \ No newline at end of file diff --git a/database/migrations/2017_03_19_091553_create_search_index_table.php b/database/migrations/2017_03_19_091553_create_search_index_table.php new file mode 100644 index 000000000..288283958 --- /dev/null +++ b/database/migrations/2017_03_19_091553_create_search_index_table.php @@ -0,0 +1,43 @@ +increments('id'); + $table->string('term', 200); + $table->string('entity_type', 100); + $table->integer('entity_id'); + $table->integer('score'); + + $table->index('term'); + $table->index('entity_type'); + $table->index(['entity_type', 'entity_id']); + $table->index('score'); + }); + + // TODO - Drop old fulltext indexes + + app(\BookStack\Services\SearchService::class)->indexAllEntities(); + } + + /** + * Reverse the migrations. + * + * @return void + */ + public function down() + { + Schema::dropIfExists('search_terms'); + } +} diff --git a/resources/views/search/all.blade.php b/resources/views/search/all.blade.php index d4053752f..54626daf1 100644 --- a/resources/views/search/all.blade.php +++ b/resources/views/search/all.blade.php @@ -20,36 +20,30 @@

{{ trans('entities.search_results') }}

- @if(count($pages) > 0) - {{ trans('entities.search_view_pages') }} - @endif + {{--TODO - Remove these pages--}} + Remove these links (Commented out) + {{--@if(count($pages) > 0)--}} + {{--{{ trans('entities.search_view_pages') }}--}} + {{--@endif--}} - @if(count($chapters) > 0) -      - {{ trans('entities.search_view_chapters') }} - @endif + {{--@if(count($chapters) > 0)--}} + {{--    --}} + {{--{{ trans('entities.search_view_chapters') }}--}} + {{--@endif--}} - @if(count($books) > 0) -      - {{ trans('entities.search_view_books') }} - @endif + {{--@if(count($books) > 0)--}} + {{--    --}} + {{--{{ trans('entities.search_view_books') }}--}} + {{--@endif--}}

{{ trans('entities.pages') }}

- @include('partials/entity-list', ['entities' => $pages, 'style' => 'detailed']) + @include('partials/entity-list', ['entities' => $entities, 'style' => 'detailed'])
- @if(count($books) > 0) -

{{ trans('entities.books') }}

- @include('partials/entity-list', ['entities' => $books]) - @endif - - @if(count($chapters) > 0) -

{{ trans('entities.chapters') }}

- @include('partials/entity-list', ['entities' => $chapters]) - @endif + Sidebar filter controls