Infra-Mirrors/BookStack
1
0
Fork 0
mirror of https://github.com/BookStackApp/BookStack.git synced 2024-07-07 20:02:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

Started implementation of new search system

Browse Source
This commit is contained in:
Dan Brown 2017-03-19 12:48:44 +00:00
parent 668ce26269
commit 070d4aeb6c
No known key found for this signature in database
GPG Key ID: 46D9F943C24A2EF9
13 changed files with 372 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
app/Book.php
View File

@ -56,4 +56,13 @@ class Book extends Entity
return strlen($description) > $length ? substr($description, 0, $length-3) . '...' : $description;
}
/**
* Return a generalised, common raw query that can be 'unioned' across entities.
* @return string
*/
public function entityRawQuery()
{
return "'BookStack\\\\Book' as entity_type, id, id as entity_id, slug, name, {$this->textField} as text,'' as html, '0' as book_id, '0' as priority, '0' as chapter_id, '0' as draft, created_by, updated_by, updated_at, created_at";
}
}

9
app/Chapter.php
View File

@ -51,4 +51,13 @@ class Chapter extends Entity
return strlen($description) > $length ? substr($description, 0, $length-3) . '...' : $description;
}
/**
* Return a generalised, common raw query that can be 'unioned' across entities.
* @return string
*/
public function entityRawQuery()
{
return "'BookStack\\\\Chapter' as entity_type, id, id as entity_id, slug, name, {$this->textField} as text, '' as html, book_id, priority, '0' as chapter_id, '0' as draft, created_by, updated_by, updated_at, created_at";
}
}

46
app/Console/Commands/RegenerateSearch.php Normal file
View File

@ -0,0 +1,46 @@
<?php
namespace BookStack\Console\Commands;
use BookStack\Services\SearchService;
use Illuminate\Console\Command;
class RegenerateSearch extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'bookstack:regenerate-search';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Command description';
protected $searchService;
/**
* Create a new command instance.
*
* @param SearchService $searchService
*/
public function __construct(SearchService $searchService)
{
parent::__construct();
$this->searchService = $searchService;
}
/**
* Execute the console command.
*
* @return mixed
*/
public function handle()
{
$this->searchService->indexAllEntities();
}
}

13
app/Console/Kernel.php
View File

@ -1,6 +1,4 @@
<?php
namespace BookStack\Console;
<?php namespace BookStack\Console;
use Illuminate\Console\Scheduling\Schedule;
use Illuminate\Foundation\Console\Kernel as ConsoleKernel;
@ -13,10 +11,11 @@ class Kernel extends ConsoleKernel
* @var array
*/
protected $commands = [
\BookStack\Console\Commands\ClearViews::class,
\BookStack\Console\Commands\ClearActivity::class,
\BookStack\Console\Commands\ClearRevisions::class,
\BookStack\Console\Commands\RegeneratePermissions::class,
Commands\ClearViews::class,
Commands\ClearActivity::class,
Commands\ClearRevisions::class,
Commands\RegeneratePermissions::class,
Commands\RegenerateSearch::class
];
/**

38
app/Entity.php
View File

@ -4,7 +4,7 @@
class Entity extends Ownable
{
protected $fieldsToSearch = ['name', 'description'];
protected $textField = 'description';
/**
* Compares this entity to another given entity.
@ -65,6 +65,15 @@ class Entity extends Ownable
return $this->morphMany(Tag::class, 'entity')->orderBy('order', 'asc');
}
/**
* Get the related search terms.
* @return \Illuminate\Database\Eloquent\Relations\MorphMany
*/
public function searchTerms()
{
return $this->morphMany(SearchTerm::class, 'entity');
}
/**
* Get this entities restrictions.
*/
@ -152,11 +161,26 @@ class Entity extends Ownable
return substr($this->name, 0, $length - 3) . '...';
}
/**
* Get the body text of this entity.
* @return mixed
*/
public function getText()
{
return $this->{$this->textField};
}
/**
* Return a generalised, common raw query that can be 'unioned' across entities.
* @return string
*/
public function entityRawQuery(){return '';}
/**
* Perform a full-text search on this entity.
* @param string[] $fieldsToSearch
* @param string[] $terms
* @param string[] array $wheres
* TODO - REMOVE
* @return mixed
*/
public function fullTextSearchQuery($terms, $wheres = [])
@ -178,21 +202,21 @@ class Entity extends Ownable
}
$isFuzzy = count($exactTerms) === 0 && count($fuzzyTerms) > 0;
$fieldsToSearch = ['name', $this->textField];
// Perform fulltext search if relevant terms exist.
if ($isFuzzy) {
$termString = implode(' ', $fuzzyTerms);
$fields = implode(',', $this->fieldsToSearch);
$search = $search->selectRaw('*, MATCH(name) AGAINST(? IN BOOLEAN MODE) AS title_relevance', [$termString]);
$search = $search->whereRaw('MATCH(' . $fields . ') AGAINST(? IN BOOLEAN MODE)', [$termString]);
$search = $search->whereRaw('MATCH(' . implode(',', $fieldsToSearch ). ') AGAINST(? IN BOOLEAN MODE)', [$termString]);
}
// Ensure at least one exact term matches if in search
if (count($exactTerms) > 0) {
$search = $search->where(function ($query) use ($exactTerms) {
$search = $search->where(function ($query) use ($exactTerms, $fieldsToSearch) {
foreach ($exactTerms as $exactTerm) {
foreach ($this->fieldsToSearch as $field) {
foreach ($fieldsToSearch as $field) {
$query->orWhere($field, 'like', $exactTerm);
}
}

18
app/Http/Controllers/SearchController.php
View File

@ -1,6 +1,7 @@
<?php namespace BookStack\Http\Controllers;
use BookStack\Repos\EntityRepo;
use BookStack\Services\SearchService;
use BookStack\Services\ViewService;
use Illuminate\Http\Request;
@ -8,16 +9,19 @@ class SearchController extends Controller
{
protected $entityRepo;
protected $viewService;
protected $searchService;
/**
* SearchController constructor.
* @param EntityRepo $entityRepo
* @param ViewService $viewService
* @param SearchService $searchService
*/
public function __construct(EntityRepo $entityRepo, ViewService $viewService)
public function __construct(EntityRepo $entityRepo, ViewService $viewService, SearchService $searchService)
{
$this->entityRepo = $entityRepo;
$this->viewService = $viewService;
$this->searchService = $searchService;
parent::__construct();
}
@ -33,15 +37,13 @@ class SearchController extends Controller
return redirect()->back();
}
$searchTerm = $request->get('term');
$paginationAppends = $request->only('term');
$pages = $this->entityRepo->getBySearch('page', $searchTerm, [], 20, $paginationAppends);
$books = $this->entityRepo->getBySearch('book', $searchTerm, [], 10, $paginationAppends);
$chapters = $this->entityRepo->getBySearch('chapter', $searchTerm, [], 10, $paginationAppends);
// $paginationAppends = $request->only('term'); TODO - Check pagination
$this->setPageTitle(trans('entities.search_for_term', ['term' => $searchTerm]));
$entities = $this->searchService->searchEntities($searchTerm);
return view('search/all', [
'pages' => $pages,
'books' => $books,
'chapters' => $chapters,
'entities' => $entities,
'searchTerm' => $searchTerm
]);
}

13
app/Page.php
View File

@ -8,8 +8,7 @@ class Page extends Entity
protected $simpleAttributes = ['name', 'id', 'slug'];
protected $with = ['book'];
protected $fieldsToSearch = ['name', 'text'];
protected $textField = 'text';
/**
* Converts this page into a simplified array.
@ -96,4 +95,14 @@ class Page extends Entity
return mb_convert_encoding($text, 'UTF-8');
}
/**
* Return a generalised, common raw query that can be 'unioned' across entities.
* @param bool $withContent
* @return string
*/
public function entityRawQuery($withContent = false)
{ $htmlQuery = $withContent ? 'html' : "'' as html";
return "'BookStack\\\\Page' as entity_type, id, id as entity_id, slug, name, {$this->textField} as text, {$htmlQuery}, book_id, priority, chapter_id, draft, created_by, updated_by, updated_at, created_at";
}
}

21
app/Repos/EntityRepo.php
View File

@ -8,6 +8,7 @@ use BookStack\Page;
use BookStack\PageRevision;
use BookStack\Services\AttachmentService;
use BookStack\Services\PermissionService;
use BookStack\Services\SearchService;
use BookStack\Services\ViewService;
use Carbon\Carbon;
use DOMDocument;
@ -58,6 +59,11 @@ class EntityRepo
*/
protected $tagRepo;
/**
* @var SearchService
*/
protected $searchService;
/**
* Acceptable operators to be used in a query
* @var array
@ -65,7 +71,7 @@ class EntityRepo
protected $queryOperators = ['<=', '>=', '=', '<', '>', 'like', '!='];
/**
* EntityService constructor.
* EntityRepo constructor.
* @param Book $book
* @param Chapter $chapter
* @param Page $page
@ -73,10 +79,12 @@ class EntityRepo
* @param ViewService $viewService
* @param PermissionService $permissionService
* @param TagRepo $tagRepo
* @param SearchService $searchService
*/
public function __construct(
Book $book, Chapter $chapter, Page $page, PageRevision $pageRevision,
ViewService $viewService, PermissionService $permissionService, TagRepo $tagRepo
ViewService $viewService, PermissionService $permissionService,
TagRepo $tagRepo, SearchService $searchService
)
{
$this->book = $book;
@ -91,6 +99,7 @@ class EntityRepo
$this->viewService = $viewService;
$this->permissionService = $permissionService;
$this->tagRepo = $tagRepo;
$this->searchService = $searchService;
}
/**
@ -608,6 +617,7 @@ class EntityRepo
$entity->updated_by = user()->id;
$isChapter ? $book->chapters()->save($entity) : $entity->save();
$this->permissionService->buildJointPermissionsForEntity($entity);
$this->searchService->indexEntity($entity);
return $entity;
}
@ -628,6 +638,7 @@ class EntityRepo
$entityModel->updated_by = user()->id;
$entityModel->save();
$this->permissionService->buildJointPermissionsForEntity($entityModel);
$this->searchService->indexEntity($entityModel);
return $entityModel;
}
@ -961,6 +972,8 @@ class EntityRepo
$this->savePageRevision($page, $input['summary']);
}
$this->searchService->indexEntity($page);
return $page;
}
@ -1064,6 +1077,7 @@ class EntityRepo
$page->text = strip_tags($page->html);
$page->updated_by = user()->id;
$page->save();
$this->searchService->indexEntity($page);
return $page;
}
@ -1156,6 +1170,7 @@ class EntityRepo
$book->views()->delete();
$book->permissions()->delete();
$this->permissionService->deleteJointPermissionsForEntity($book);
$this->searchService->deleteEntityTerms($book);
$book->delete();
}
@ -1175,6 +1190,7 @@ class EntityRepo
$chapter->views()->delete();
$chapter->permissions()->delete();
$this->permissionService->deleteJointPermissionsForEntity($chapter);
$this->searchService->deleteEntityTerms($chapter);
$chapter->delete();
}
@ -1190,6 +1206,7 @@ class EntityRepo
$page->revisions()->delete();
$page->permissions()->delete();
$this->permissionService->deleteJointPermissionsForEntity($page);
$this->searchService->deleteEntityTerms($page);
// Delete Attached Files
$attachmentService = app(AttachmentService::class);

20
app/SearchTerm.php Normal file
View File

@ -0,0 +1,20 @@
<?php namespace BookStack;
use Illuminate\Database\Eloquent\Model;
class SearchTerm extends Model
{
protected $fillable = ['term', 'entity_id', 'entity_type', 'score'];
public $timestamps = false;
/**
* Get the entity that this term belongs to
* @return \Illuminate\Database\Eloquent\Relations\MorphTo
*/
public function entity()
{
return $this->morphTo('entity');
}
}

7
app/Services/PermissionService.php
View File

@ -479,8 +479,7 @@ class PermissionService
* @return \Illuminate\Database\Query\Builder
*/
public function bookChildrenQuery($book_id, $filterDrafts = false, $fetchPageContent = false) {
$pageContentSelect = $fetchPageContent ? 'html' : "''";
$pageSelect = $this->db->table('pages')->selectRaw("'BookStack\\\\Page' as entity_type, id, slug, name, text, {$pageContentSelect} as description, book_id, priority, chapter_id, draft")->where('book_id', '=', $book_id)->where(function($query) use ($filterDrafts) {
$pageSelect = $this->db->table('pages')->selectRaw($this->page->entityRawQuery($fetchPageContent))->where('book_id', '=', $book_id)->where(function($query) use ($filterDrafts) {
$query->where('draft', '=', 0);
if (!$filterDrafts) {
$query->orWhere(function($query) {
@ -488,7 +487,7 @@ class PermissionService
});
}
});
$chapterSelect = $this->db->table('chapters')->selectRaw("'BookStack\\\\Chapter' as entity_type, id, slug, name, '' as text, description, book_id, priority, 0 as chapter_id, 0 as draft")->where('book_id', '=', $book_id);
$chapterSelect = $this->db->table('chapters')->selectRaw($this->chapter->entityRawQuery())->where('book_id', '=', $book_id);
$query = $this->db->query()->select('*')->from($this->db->raw("({$pageSelect->toSql()} UNION {$chapterSelect->toSql()}) AS U"))
->mergeBindings($pageSelect)->mergeBindings($chapterSelect);
@ -540,7 +539,7 @@ class PermissionService
}
/**
* Filter items that have entities set a a polymorphic relation.
* Filter items that have entities set as a polymorphic relation.
* @param $query
* @param string $tableName
* @param string $entityIdColumn

150
app/Services/SearchService.php Normal file
View File

@ -0,0 +1,150 @@
<?php namespace BookStack\Services;
use BookStack\Book;
use BookStack\Chapter;
use BookStack\Entity;
use BookStack\Page;
use BookStack\SearchTerm;
use Illuminate\Database\Connection;
use Illuminate\Database\Query\JoinClause;
class SearchService
{
protected $searchTerm;
protected $book;
protected $chapter;
protected $page;
protected $db;
/**
* SearchService constructor.
* @param SearchTerm $searchTerm
* @param Book $book
* @param Chapter $chapter
* @param Page $page
* @param Connection $db
*/
public function __construct(SearchTerm $searchTerm, Book $book, Chapter $chapter, Page $page, Connection $db)
{
$this->searchTerm = $searchTerm;
$this->book = $book;
$this->chapter = $chapter;
$this->page = $page;
$this->db = $db;
}
public function searchEntities($searchString, $entityType = 'all')
{
// TODO - Add Tag Searches
// TODO - Add advanced custom column searches
// TODO - Add exact match searches ("")
$termArray = explode(' ', $searchString);
$subQuery = $this->db->table('search_terms')->select('entity_id', 'entity_type', \DB::raw('SUM(score) as score'));
$subQuery->where(function($query) use ($termArray) {
foreach ($termArray as $inputTerm) {
$query->orWhere('term', 'like', $inputTerm .'%');
}
});
$subQuery = $subQuery->groupBy('entity_type', 'entity_id');
$pageSelect = $this->db->table('pages as e')->join(\DB::raw('(' . $subQuery->toSql() . ') as s'), function(JoinClause $join) {
$join->on('e.id', '=', 's.entity_id');
})->selectRaw('e.*, s.score')->orderBy('score', 'desc');
$pageSelect->mergeBindings($subQuery);
dd($pageSelect->toSql());
// TODO - Continue from here
}
/**
* Index the given entity.
* @param Entity $entity
*/
public function indexEntity(Entity $entity)
{
$this->deleteEntityTerms($entity);
$nameTerms = $this->generateTermArrayFromText($entity->name, 5);
$bodyTerms = $this->generateTermArrayFromText($entity->getText(), 1);
$terms = array_merge($nameTerms, $bodyTerms);
$entity->searchTerms()->createMany($terms);
}
/**
* Index multiple Entities at once
* @param Entity[] $entities
*/
protected function indexEntities($entities) {
$terms = [];
foreach ($entities as $entity) {
$nameTerms = $this->generateTermArrayFromText($entity->name, 5);
$bodyTerms = $this->generateTermArrayFromText($entity->getText(), 1);
foreach (array_merge($nameTerms, $bodyTerms) as $term) {
$term['entity_id'] = $entity->id;
$term['entity_type'] = $entity->getMorphClass();
$terms[] = $term;
}
}
$this->searchTerm->insert($terms);
}
/**
* Delete and re-index the terms for all entities in the system.
*/
public function indexAllEntities()
{
$this->searchTerm->truncate();
// Chunk through all books
$this->book->chunk(500, function ($books) {
$this->indexEntities($books);
});
// Chunk through all chapters
$this->chapter->chunk(500, function ($chapters) {
$this->indexEntities($chapters);
});
// Chunk through all pages
$this->page->chunk(500, function ($pages) {
$this->indexEntities($pages);
});
}
/**
* Delete related Entity search terms.
* @param Entity $entity
*/
public function deleteEntityTerms(Entity $entity)
{
$entity->searchTerms()->delete();
}
/**
* Create a scored term array from the given text.
* @param $text
* @param float|int $scoreAdjustment
* @return array
*/
protected function generateTermArrayFromText($text, $scoreAdjustment = 1)
{
$tokenMap = []; // {TextToken => OccurrenceCount}
$splitText = explode(' ', $text);
foreach ($splitText as $token) {
if ($token === '') continue;
if (!isset($tokenMap[$token])) $tokenMap[$token] = 0;
$tokenMap[$token]++;
}
$terms = [];
foreach ($tokenMap as $token => $count) {
$terms[] = [
'term' => $token,
'score' => $count * $scoreAdjustment
];
}
return $terms;
}
}

43
database/migrations/2017_03_19_091553_create_search_index_table.php Normal file
View File

@ -0,0 +1,43 @@
<?php
use Illuminate\Support\Facades\Schema;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Database\Migrations\Migration;
class CreateSearchIndexTable extends Migration
{
/**
* Run the migrations.
*
* @return void
*/
public function up()
{
Schema::create('search_terms', function (Blueprint $table) {
$table->increments('id');
$table->string('term', 200);
$table->string('entity_type', 100);
$table->integer('entity_id');
$table->integer('score');
$table->index('term');
$table->index('entity_type');
$table->index(['entity_type', 'entity_id']);
$table->index('score');
});
// TODO - Drop old fulltext indexes
app(\BookStack\Services\SearchService::class)->indexAllEntities();
}
/**
* Reverse the migrations.
*
* @return void
*/
public function down()
{
Schema::dropIfExists('search_terms');
}
}

36
resources/views/search/all.blade.php
View File

@ -20,36 +20,30 @@
<h1>{{ trans('entities.search_results') }}</h1>
<p>
@if(count($pages) > 0)
<a href="{{ baseUrl("/search/pages?term={$searchTerm}") }}" class="text-page"><i class="zmdi zmdi-file-text"></i>{{ trans('entities.search_view_pages') }}</a>
@endif
{{--TODO - Remove these pages--}}
Remove these links (Commented out)
{{--@if(count($pages) > 0)--}}
{{--<a href="{{ baseUrl("/search/pages?term={$searchTerm}") }}" class="text-page"><i class="zmdi zmdi-file-text"></i>{{ trans('entities.search_view_pages') }}</a>--}}
{{--@endif--}}
@if(count($chapters) > 0)
&nbsp; &nbsp;&nbsp;
<a href="{{ baseUrl("/search/chapters?term={$searchTerm}") }}" class="text-chapter"><i class="zmdi zmdi-collection-bookmark"></i>{{ trans('entities.search_view_chapters') }}</a>
@endif
{{--@if(count($chapters) > 0)--}}
{{--&nbsp; &nbsp;&nbsp;--}}
{{--<a href="{{ baseUrl("/search/chapters?term={$searchTerm}") }}" class="text-chapter"><i class="zmdi zmdi-collection-bookmark"></i>{{ trans('entities.search_view_chapters') }}</a>--}}
{{--@endif--}}
@if(count($books) > 0)
&nbsp; &nbsp;&nbsp;
<a href="{{ baseUrl("/search/books?term={$searchTerm}") }}" class="text-book"><i class="zmdi zmdi-book"></i>{{ trans('entities.search_view_books') }}</a>
@endif
{{--@if(count($books) > 0)--}}
{{--&nbsp; &nbsp;&nbsp;--}}
{{--<a href="{{ baseUrl("/search/books?term={$searchTerm}") }}" class="text-book"><i class="zmdi zmdi-book"></i>{{ trans('entities.search_view_books') }}</a>--}}
{{--@endif--}}
</p>
<div class="row">
<div class="col-md-6">
<h3><a href="{{ baseUrl("/search/pages?term={$searchTerm}") }}" class="no-color">{{ trans('entities.pages') }}</a></h3>
@include('partials/entity-list', ['entities' => $pages, 'style' => 'detailed'])
@include('partials/entity-list', ['entities' => $entities, 'style' => 'detailed'])
</div>
<div class="col-md-5 col-md-offset-1">
@if(count($books) > 0)
<h3><a href="{{ baseUrl("/search/books?term={$searchTerm}") }}" class="no-color">{{ trans('entities.books') }}</a></h3>
@include('partials/entity-list', ['entities' => $books])
@endif
@if(count($chapters) > 0)
<h3><a href="{{ baseUrl("/search/chapters?term={$searchTerm}") }}" class="no-color">{{ trans('entities.chapters') }}</a></h3>
@include('partials/entity-list', ['entities' => $chapters])
@endif
Sidebar filter controls
</div>
</div>