Input WYSIWYG: Added reference store & fetch handling

For book, shelves and chapters.
Made much of the existing handling generic to entity types.
Added new MixedEntityListLoader to help load lists somewhat efficiently.
Only manually tested so far.

app/Console/Commands/RegenerateReferencesCommand.php

@@ -34,7 +34,7 @@ class RegenerateReferencesCommand extends Command
             DB::setDefaultConnection($this->option('database'));
         }
 
-        $references->updateForAllPages();
+        $references->updateForAll();
 
         DB::setDefaultConnection($connection);
 

app/Entities/Controllers/BookController.php

@@ -138,7 +138,7 @@ class BookController extends Controller
             'bookParentShelves' => $bookParentShelves,
             'watchOptions'      => new UserEntityWatchOptions(user(), $book),
             'activity'          => $activities->entityActivity($book, 20, 1),
-            'referenceCount'    => $this->referenceFetcher->getPageReferenceCountToEntity($book),
+            'referenceCount'    => $this->referenceFetcher->getReferenceCountToEntity($book),
         ]);
     }
 

app/Entities/Controllers/BookshelfController.php

@@ -125,7 +125,7 @@ class BookshelfController extends Controller
             'view'                    => $view,
             'activity'                => $activities->entityActivity($shelf, 20, 1),
             'listOptions'             => $listOptions,
-            'referenceCount'          => $this->referenceFetcher->getPageReferenceCountToEntity($shelf),
+            'referenceCount'          => $this->referenceFetcher->getReferenceCountToEntity($shelf),
         ]);
     }
 

app/Entities/Controllers/ChapterController.php

@@ -86,7 +86,7 @@ class ChapterController extends Controller
             'pages'          => $pages,
             'next'           => $nextPreviousLocator->getNext(),
             'previous'       => $nextPreviousLocator->getPrevious(),
-            'referenceCount' => $this->referenceFetcher->getPageReferenceCountToEntity($chapter),
+            'referenceCount' => $this->referenceFetcher->getReferenceCountToEntity($chapter),
         ]);
     }
 

app/Entities/Controllers/PageController.php

@@ -155,7 +155,7 @@ class PageController extends Controller
             'watchOptions'    => new UserEntityWatchOptions(user(), $page),
             'next'            => $nextPreviousLocator->getNext(),
             'previous'        => $nextPreviousLocator->getPrevious(),
-            'referenceCount'  => $this->referenceFetcher->getPageReferenceCountToEntity($page),
+            'referenceCount'  => $this->referenceFetcher->getReferenceCountToEntity($page),
         ]);
     }
 

app/Entities/Models/Book.php

@@ -28,7 +28,7 @@ class Book extends Entity implements HasCoverImage
     use HasFactory;
     use HasHtmlDescription;
 
-    public $searchFactor = 1.2;
+    public float $searchFactor = 1.2;
 
     protected $fillable = ['name'];
     protected $hidden = ['pivot', 'image_id', 'deleted_at'];

app/Entities/Models/Bookshelf.php

@@ -15,7 +15,7 @@ class Bookshelf extends Entity implements HasCoverImage
 
     protected $table = 'bookshelves';
 
-    public $searchFactor = 1.2;
+    public float $searchFactor = 1.2;
 
     protected $fillable = ['name', 'description', 'image_id'];
 

app/Entities/Models/Chapter.php

@@ -17,7 +17,7 @@ class Chapter extends BookChild
     use HasFactory;
     use HasHtmlDescription;
 
-    public $searchFactor = 1.2;
+    public float $searchFactor = 1.2;
 
     protected $fillable = ['name', 'description', 'priority'];
     protected $hidden = ['pivot', 'deleted_at'];

app/Entities/Models/Entity.php

@@ -57,12 +57,17 @@ abstract class Entity extends Model implements Sluggable, Favouritable, Viewable
     /**
      * @var string - Name of property where the main text content is found
      */
-    public $textField = 'description';
+    public string $textField = 'description';
+
+    /**
+     * @var string - Name of the property where the main HTML content is found
+     */
+    public string $htmlField = 'description_html';
 
     /**
      * @var float - Multiplier for search indexing.
      */
-    public $searchFactor = 1.0;
+    public float $searchFactor = 1.0;
 
     /**
      * Get the entities that are visible to the current user.

app/Entities/Models/Page.php

@@ -37,7 +37,8 @@ class Page extends BookChild
 
     protected $fillable = ['name', 'priority'];
 
-    public $textField = 'text';
+    public string $textField = 'text';
+    public string $htmlField = 'html';
 
     protected $hidden = ['html', 'markdown', 'text', 'pivot', 'deleted_at'];
 

app/Entities/Repos/BaseRepo.php

@@ -7,6 +7,7 @@ use BookStack\Entities\Models\Entity;
 use BookStack\Entities\Models\HasCoverImage;
 use BookStack\Entities\Models\HasHtmlDescription;
 use BookStack\Exceptions\ImageUploadException;
+use BookStack\References\ReferenceStore;
 use BookStack\References\ReferenceUpdater;
 use BookStack\Uploads\ImageRepo;
 use Illuminate\Http\UploadedFile;
@@ -16,7 +17,8 @@ class BaseRepo
     public function __construct(
         protected TagRepo $tagRepo,
         protected ImageRepo $imageRepo,
-        protected ReferenceUpdater $referenceUpdater
+        protected ReferenceUpdater $referenceUpdater,
+        protected ReferenceStore $referenceStore,
     ) {
     }
 
@@ -42,6 +44,7 @@ class BaseRepo
         $entity->refresh();
         $entity->rebuildPermissions();
         $entity->indexForSearch();
+        $this->referenceStore->updateForEntity($entity);
     }
 
     /**
@@ -68,6 +71,7 @@ class BaseRepo
 
         $entity->rebuildPermissions();
         $entity->indexForSearch();
+        $this->referenceStore->updateForEntity($entity);
 
         if ($oldUrl !== $entity->getUrl()) {
             $this->referenceUpdater->updateEntityPageReferences($entity, $oldUrl);

app/Entities/Repos/PageRepo.php

@@ -162,7 +162,6 @@ class PageRepo
         $this->baseRepo->update($draft, $input);
 
         $this->revisionRepo->storeNewForPage($draft, trans('entities.pages_initial_revision'));
-        $this->referenceStore->updateForPage($draft);
         $draft->refresh();
 
         Activity::add(ActivityType::PAGE_CREATE, $draft);
@@ -182,7 +181,6 @@ class PageRepo
 
         $this->updateTemplateStatusAndContentFromInput($page, $input);
         $this->baseRepo->update($page, $input);
-        $this->referenceStore->updateForPage($page);
 
         // Update with new details
         $page->revision_count++;
@@ -301,7 +299,7 @@ class PageRepo
         $page->refreshSlug();
         $page->save();
         $page->indexForSearch();
-        $this->referenceStore->updateForPage($page);
+        $this->referenceStore->updateForEntity($page);
 
         $summary = trans('entities.pages_revision_restored_from', ['id' => strval($revisionId), 'summary' => $revision->summary]);
         $this->revisionRepo->storeNewForPage($page, $summary);

app/Entities/Tools/MixedEntityListLoader.php

@@ -0,0 +1,103 @@
+<?php
+
+namespace BookStack\Entities\Tools;
+
+use BookStack\App\Model;
+use BookStack\Entities\EntityProvider;
+use Illuminate\Database\Eloquent\Relations\Relation;
+
+class MixedEntityListLoader
+{
+    protected array $listAttributes = [
+        'page'      => ['id', 'name', 'slug', 'book_id', 'chapter_id', 'text', 'draft'],
+        'chapter'   => ['id', 'name', 'slug', 'book_id', 'description'],
+        'book'      => ['id', 'name', 'slug', 'description'],
+        'bookshelf' => ['id', 'name', 'slug', 'description'],
+    ];
+
+    public function __construct(
+        protected EntityProvider $entityProvider
+    ) {
+    }
+
+    /**
+     * Efficiently load in entities for listing onto the given list
+     * where entities are set as a relation via the given name.
+     * This will look for a model id and type via 'name_id' and 'name_type'.
+     * @param Model[] $relations
+     */
+    public function loadIntoRelations(array $relations, string $relationName): void
+    {
+        $idsByType = [];
+        foreach ($relations as $relation) {
+            $type = $relation->getAttribute($relationName . '_type');
+            $id = $relation->getAttribute($relationName . '_id');
+
+            if (!isset($idsByType[$type])) {
+                $idsByType[$type] = [];
+            }
+
+            $idsByType[$type][] = $id;
+        }
+
+        $modelMap = $this->idsByTypeToModelMap($idsByType);
+
+        foreach ($relations as $relation) {
+            $type = $relation->getAttribute($relationName . '_type');
+            $id = $relation->getAttribute($relationName . '_id');
+            $related = $modelMap[$type][strval($id)] ?? null;
+            if ($related) {
+                $relation->setRelation($relationName, $related);
+            }
+        }
+    }
+
+    /**
+     * @param array<string, int[]> $idsByType
+     * @return array<string, array<int, Model>>
+     */
+    protected function idsByTypeToModelMap(array $idsByType): array
+    {
+        $modelMap = [];
+
+        foreach ($idsByType as $type => $ids) {
+            if (!isset($this->listAttributes[$type])) {
+                continue;
+            }
+
+            $instance = $this->entityProvider->get($type);
+            $models = $instance->newQuery()
+                ->select($this->listAttributes[$type])
+                ->scopes('visible')
+                ->whereIn('id', $ids)
+                ->with($this->getRelationsToEagerLoad($type))
+                ->get();
+
+            if (count($models) > 0) {
+                $modelMap[$type] = [];
+            }
+
+            foreach ($models as $model) {
+                $modelMap[$type][strval($model->id)] = $model;
+            }
+        }
+
+        return $modelMap;
+    }
+
+    protected function getRelationsToEagerLoad(string $type): array
+    {
+        $toLoad = [];
+        $loadVisible = fn (Relation $query) => $query->scopes('visible');
+
+        if ($type === 'chapter' || $type === 'page') {
+            $toLoad['book'] = $loadVisible;
+        }
+
+        if ($type === 'page') {
+            $toLoad['chapter'] = $loadVisible;
+        }
+
+        return $toLoad;
+    }
+}

app/References/ReferenceController.php

@@ -10,11 +10,9 @@ use BookStack\Http\Controller;
 
 class ReferenceController extends Controller
 {
-    protected ReferenceFetcher $referenceFetcher;
+    public function __construct(
-
+        protected ReferenceFetcher $referenceFetcher
-    public function __construct(ReferenceFetcher $referenceFetcher)
+    ) {
-    {
-        $this->referenceFetcher = $referenceFetcher;
     }
 
     /**
@@ -23,7 +21,7 @@ class ReferenceController extends Controller
     public function page(string $bookSlug, string $pageSlug)
     {
         $page = Page::getBySlugs($bookSlug, $pageSlug);
-        $references = $this->referenceFetcher->getPageReferencesToEntity($page);
+        $references = $this->referenceFetcher->getReferencesToEntity($page);
 
         return view('pages.references', [
             'page'       => $page,
@@ -37,7 +35,7 @@ class ReferenceController extends Controller
     public function chapter(string $bookSlug, string $chapterSlug)
     {
         $chapter = Chapter::getBySlugs($bookSlug, $chapterSlug);
-        $references = $this->referenceFetcher->getPageReferencesToEntity($chapter);
+        $references = $this->referenceFetcher->getReferencesToEntity($chapter);
 
         return view('chapters.references', [
             'chapter'    => $chapter,
@@ -51,7 +49,7 @@ class ReferenceController extends Controller
     public function book(string $slug)
     {
         $book = Book::getBySlug($slug);
-        $references = $this->referenceFetcher->getPageReferencesToEntity($book);
+        $references = $this->referenceFetcher->getReferencesToEntity($book);
 
         return view('books.references', [
             'book'       => $book,
@@ -65,7 +63,7 @@ class ReferenceController extends Controller
     public function shelf(string $slug)
     {
         $shelf = Bookshelf::getBySlug($slug);
-        $references = $this->referenceFetcher->getPageReferencesToEntity($shelf);
+        $references = $this->referenceFetcher->getReferencesToEntity($shelf);
 
         return view('shelves.references', [
             'shelf'      => $shelf,

app/References/ReferenceFetcher.php

@@ -3,65 +3,51 @@
 namespace BookStack\References;
 
 use BookStack\Entities\Models\Entity;
-use BookStack\Entities\Models\Page;
+use BookStack\Entities\Tools\MixedEntityListLoader;
 use BookStack\Permissions\PermissionApplicator;
 use Illuminate\Database\Eloquent\Builder;
 use Illuminate\Database\Eloquent\Collection;
-use Illuminate\Database\Eloquent\Relations\Relation;
 
 class ReferenceFetcher
 {
-    protected PermissionApplicator $permissions;
+    public function __construct(
-
+        protected PermissionApplicator $permissions,
-    public function __construct(PermissionApplicator $permissions)
+        protected MixedEntityListLoader $mixedEntityListLoader,
-    {
+    ) {
-        $this->permissions = $permissions;
     }
 
     /**
-     * Query and return the page references pointing to the given entity.
+     * Query and return the references pointing to the given entity.
      * Loads the commonly required relations while taking permissions into account.
      */
-    public function getPageReferencesToEntity(Entity $entity): Collection
+    public function getReferencesToEntity(Entity $entity): Collection
     {
-        $baseQuery = $this->queryPageReferencesToEntity($entity)
+        $references = $this->queryReferencesToEntity($entity)->get();
-            ->with([
+        $this->mixedEntityListLoader->loadIntoRelations($references->all(), 'from');
-                'from'         => fn (Relation $query) => $query->select(Page::$listAttributes),
-                'from.book'    => fn (Relation $query) => $query->scopes('visible'),
-                'from.chapter' => fn (Relation $query) => $query->scopes('visible'),
-            ]);
-
-        $references = $this->permissions->restrictEntityRelationQuery(
-            $baseQuery,
-            'references',
-            'from_id',
-            'from_type'
-        )->get();
 
         return $references;
     }
 
     /**
-     * Returns the count of page references pointing to the given entity.
+     * Returns the count of references pointing to the given entity.
      * Takes permissions into account.
      */
-    public function getPageReferenceCountToEntity(Entity $entity): int
+    public function getReferenceCountToEntity(Entity $entity): int
     {
-        $count = $this->permissions->restrictEntityRelationQuery(
+        return $this->queryReferencesToEntity($entity)->count();
-            $this->queryPageReferencesToEntity($entity),
+    }
+
+    protected function queryReferencesToEntity(Entity $entity): Builder
+    {
+        $baseQuery = Reference::query()
+            ->where('to_type', '=', $entity->getMorphClass())
+            ->where('to_id', '=', $entity->id);
+
+        return $this->permissions->restrictEntityRelationQuery(
+            $baseQuery,
             'references',
             'from_id',
             'from_type'
-        )->count();
+        );
-
-        return $count;
-    }
-
-    protected function queryPageReferencesToEntity(Entity $entity): Builder
-    {
-        return Reference::query()
-            ->where('to_type', '=', $entity->getMorphClass())
-            ->where('to_id', '=', $entity->id)
-            ->where('from_type', '=', (new Page())->getMorphClass());
     }
 }

app/References/ReferenceStore.php

@@ -2,60 +2,62 @@
 
 namespace BookStack\References;
 
-use BookStack\Entities\Models\Page;
+use BookStack\Entities\EntityProvider;
+use BookStack\Entities\Models\Entity;
 use Illuminate\Database\Eloquent\Collection;
 
 class ReferenceStore
 {
-    /**
+    public function __construct(
-     * Update the outgoing references for the given page.
+        protected EntityProvider $entityProvider
-     */
+    ) {
-    public function updateForPage(Page $page): void
-    {
-        $this->updateForPages([$page]);
     }
 
     /**
-     * Update the outgoing references for all pages in the system.
+     * Update the outgoing references for the given entity.
      */
-    public function updateForAllPages(): void
+    public function updateForEntity(Entity $entity): void
     {
-        Reference::query()
+        $this->updateForEntities([$entity]);
-            ->where('from_type', '=', (new Page())->getMorphClass())
-            ->delete();
-
-        Page::query()->select(['id', 'html'])->chunk(100, function (Collection $pages) {
-            $this->updateForPages($pages->all());
-        });
     }
 
     /**
-     * Update the outgoing references for the pages in the given array.
+     * Update the outgoing references for all entities in the system.
+     */
+    public function updateForAll(): void
+    {
+        Reference::query()->delete();
+
+        foreach ($this->entityProvider->all() as $entity) {
+            $entity->newQuery()->select(['id', $entity->htmlField])->chunk(100, function (Collection $entities) {
+                $this->updateForEntities($entities->all());
+            });
+        }
+    }
+
+    /**
+     * Update the outgoing references for the entities in the given array.
      *
-     * @param Page[] $pages
+     * @param Entity[] $entities
      */
-    protected function updateForPages(array $pages): void
+    protected function updateForEntities(array $entities): void
     {
-        if (count($pages) === 0) {
+        if (count($entities) === 0) {
             return;
         }
 
         $parser = CrossLinkParser::createWithEntityResolvers();
         $references = [];
 
-        $pageIds = array_map(fn (Page $page) => $page->id, $pages);
+        $this->dropReferencesFromEntities($entities);
-        Reference::query()
-            ->where('from_type', '=', $pages[0]->getMorphClass())
-            ->whereIn('from_id', $pageIds)
-            ->delete();
 
-        foreach ($pages as $page) {
+        foreach ($entities as $entity) {
-            $models = $parser->extractLinkedModels($page->html);
+            $models = $parser->extractLinkedModels($entity->getAttribute($entity->htmlField));
 
             foreach ($models as $model) {
                 $references[] = [
-                    'from_id'   => $page->id,
+                    'from_id'   => $entity->id,
-                    'from_type' => $page->getMorphClass(),
+                    'from_type' => $entity->getMorphClass(),
                     'to_id'     => $model->id,
                     'to_type'   => $model->getMorphClass(),
                 ];
@@ -66,4 +68,29 @@ class ReferenceStore
             Reference::query()->insert($referenceDataChunk);
         }
     }
+
+    /**
+     * Delete all the existing references originating from the given entities.
+     * @param Entity[] $entities
+     */
+    protected function dropReferencesFromEntities(array $entities): void
+    {
+        $IdsByType = [];
+
+        foreach ($entities as $entity) {
+            $type = $entity->getMorphClass();
+            if (!isset($IdsByType[$type])) {
+                $IdsByType[$type] = [];
+            }
+
+            $IdsByType[$type][] = $entity->id;
+        }
+
+        foreach ($IdsByType as $type => $entityIds) {
+            Reference::query()
+                ->where('from_type', '=', $type)
+                ->whereIn('from_id', $entityIds)
+                ->delete();
+        }
+    }
 }

app/References/ReferenceUpdater.php

@@ -35,7 +35,7 @@ class ReferenceUpdater
     protected function getReferencesToUpdate(Entity $entity): array
     {
         /** @var Reference[] $references */
-        $references = $this->referenceFetcher->getPageReferencesToEntity($entity)->values()->all();
+        $references = $this->referenceFetcher->getReferencesToEntity($entity)->values()->all();
 
         if ($entity instanceof Book) {
             $pages = $entity->pages()->get(['id']);
@@ -43,7 +43,7 @@ class ReferenceUpdater
             $children = $pages->concat($chapters);
             foreach ($children as $bookChild) {
                 /** @var Reference[] $childRefs */
-                $childRefs = $this->referenceFetcher->getPageReferencesToEntity($bookChild)->values()->all();
+                $childRefs = $this->referenceFetcher->getReferencesToEntity($bookChild)->values()->all();
                 array_push($references, ...$childRefs);
             }
         }

app/Settings/MaintenanceController.php

@@ -87,7 +87,7 @@ class MaintenanceController extends Controller
         $this->logActivity(ActivityType::MAINTENANCE_ACTION_RUN, 'regenerate-references');
 
         try {
-            $referenceStore->updateForAllPages();
+            $referenceStore->updateForAll();
             $this->showSuccessNotification(trans('settings.maint_regen_references_success'));
         } catch (\Exception $exception) {
             $this->showErrorNotification($exception->getMessage());

lang/en/entities.php

@@ -23,7 +23,7 @@ return [
     'meta_updated' => 'Updated :timeLength',
     'meta_updated_name' => 'Updated :timeLength by :user',
     'meta_owned_name' => 'Owned by :user',
-    'meta_reference_page_count' => 'Referenced on :count page|Referenced on :count pages',
+    'meta_reference_count' => 'Referenced by :count item|Referenced by :count items',
     'entity_select' => 'Entity Select',
     'entity_select_lack_permission' => 'You don\'t have the required permissions to select this item',
     'images' => 'Images',
@@ -409,7 +409,7 @@ return [
     // References
     'references' => 'References',
     'references_none' => 'There are no tracked references to this item.',
-    'references_to_desc' => 'Shown below are all the known pages in the system that link to this item.',
+    'references_to_desc' => 'Listed below is all the known content in the system that links to this item.',
 
     // Watch Options
     'watch' => 'Watch',

resources/views/entities/meta.blade.php

@@ -64,7 +64,7 @@
         <a href="{{ $entity->getUrl('/references') }}" class="entity-meta-item">
             @icon('reference')
             <div>
-                {!! trans_choice('entities.meta_reference_page_count', $referenceCount, ['count' => $referenceCount]) !!}
+                {{ trans_choice('entities.meta_reference_count', $referenceCount, ['count' => $referenceCount]) }}
             </div>
         </a>
     @endif

resources/views/shelves/show.blade.php

@@ -28,7 +28,7 @@
         </div>
 
         <div class="book-content">
-            <p class="text-muted">{!! nl2br(e($shelf->description)) !!}</p>
+            <p class="text-muted">{!! $shelf->descriptionHtml() !!}</p>
             @if(count($sortedVisibleShelfBooks) > 0)
                 @if($view === 'list')
                     <div class="entity-list">