Refactor cache handling

Signed-off-by: Julius Härtl <jus@bitgrid.net>

build/psalm-baseline.xml

@@ -2417,6 +2417,13 @@
       <code>bool|mixed</code>
     </LessSpecificImplementedReturnType>
   </file>
+  <file src="lib/private/Collaboration/Reference/File/FileReferenceEventListener.php">
+    <InvalidArgument occurrences="3">
+      <code>addServiceListener</code>
+      <code>addServiceListener</code>
+      <code>addServiceListener</code>
+    </InvalidArgument>
+  </file>
   <file src="lib/private/Command/CallableJob.php">
     <ParamNameMismatch occurrences="1">
       <code>$serializedCallable</code>

config/config.sample.php

@@ -2245,4 +2245,11 @@ $CONFIG = [
  * Defaults to ``true``
  */
 'bulkupload.enabled' => true,
+
+/**
+ * Enables fetching open graph metadata from remote urls
+ *
+ * Defaults to ``true``
+ */
+'reference_opengraph' => true,
 ];

core/Controller/ReferenceApiController.php

@@ -45,9 +45,11 @@ class ReferenceApiController extends \OCP\AppFramework\OCSController {
 		$result = [];
 		$index = 0;
 		foreach ($references as $reference) {
-			if ($index++ < $limit) {
-				$result[$reference] = $resolve ? $this->referenceManager->resolveReference($reference) : null;
+			if ($index++ >= $limit) {
+				break;
 			}
+
+			$result[$reference] = $resolve ? $this->referenceManager->resolveReference($reference) : null;
 		}
 
 		return new DataResponse([

lib/base.php

@@ -750,6 +750,7 @@ class OC {
 		self::registerEncryptionWrapperAndHooks();
 		self::registerAccountHooks();
 		self::registerResourceCollectionHooks();
+		self::registerFileReferenceEventListener();
 		self::registerAppRestrictionsHooks();
 
 		// Make sure that the application class is not loaded before the database is setup
@@ -912,6 +913,10 @@ class OC {
 		\OC\Collaboration\Resources\Listener::register(Server::get(SymfonyAdapter::class), Server::get(IEventDispatcher::class));
 	}
 
+	private static function registerFileReferenceEventListener() {
+		\OC\Collaboration\Reference\File\FileReferenceEventListener::register(Server::get(IEventDispatcher::class));
+	}
+
 	/**
 	 * register hooks for the filesystem
 	 */

lib/composer/composer/autoload_classmap.php

@@ -826,7 +826,8 @@ return array(
     'OC\\Collaboration\\Collaborators\\Search' => $baseDir . '/lib/private/Collaboration/Collaborators/Search.php',
     'OC\\Collaboration\\Collaborators\\SearchResult' => $baseDir . '/lib/private/Collaboration/Collaborators/SearchResult.php',
     'OC\\Collaboration\\Collaborators\\UserPlugin' => $baseDir . '/lib/private/Collaboration/Collaborators/UserPlugin.php',
-    'OC\\Collaboration\\Reference\\FileReferenceProvider' => $baseDir . '/lib/private/Collaboration/Reference/FileReferenceProvider.php',
+    'OC\\Collaboration\\Reference\\File\\FileReferenceEventListener' => $baseDir . '/lib/private/Collaboration/Reference/File/FileReferenceEventListener.php',
+    'OC\\Collaboration\\Reference\\File\\FileReferenceProvider' => $baseDir . '/lib/private/Collaboration/Reference/File/FileReferenceProvider.php',
     'OC\\Collaboration\\Reference\\LinkReferenceProvider' => $baseDir . '/lib/private/Collaboration/Reference/LinkReferenceProvider.php',
     'OC\\Collaboration\\Reference\\Reference' => $baseDir . '/lib/private/Collaboration/Reference/Reference.php',
     'OC\\Collaboration\\Reference\\ReferenceManager' => $baseDir . '/lib/private/Collaboration/Reference/ReferenceManager.php',

lib/composer/composer/autoload_static.php

@@ -859,7 +859,8 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2
         'OC\\Collaboration\\Collaborators\\Search' => __DIR__ . '/../../..' . '/lib/private/Collaboration/Collaborators/Search.php',
         'OC\\Collaboration\\Collaborators\\SearchResult' => __DIR__ . '/../../..' . '/lib/private/Collaboration/Collaborators/SearchResult.php',
         'OC\\Collaboration\\Collaborators\\UserPlugin' => __DIR__ . '/../../..' . '/lib/private/Collaboration/Collaborators/UserPlugin.php',
-        'OC\\Collaboration\\Reference\\FileReferenceProvider' => __DIR__ . '/../../..' . '/lib/private/Collaboration/Reference/FileReferenceProvider.php',
+        'OC\\Collaboration\\Reference\\File\\FileReferenceEventListener' => __DIR__ . '/../../..' . '/lib/private/Collaboration/Reference/File/FileReferenceEventListener.php',
+        'OC\\Collaboration\\Reference\\File\\FileReferenceProvider' => __DIR__ . '/../../..' . '/lib/private/Collaboration/Reference/File/FileReferenceProvider.php',
         'OC\\Collaboration\\Reference\\LinkReferenceProvider' => __DIR__ . '/../../..' . '/lib/private/Collaboration/Reference/LinkReferenceProvider.php',
         'OC\\Collaboration\\Reference\\Reference' => __DIR__ . '/../../..' . '/lib/private/Collaboration/Reference/Reference.php',
         'OC\\Collaboration\\Reference\\ReferenceManager' => __DIR__ . '/../../..' . '/lib/private/Collaboration/Reference/ReferenceManager.php',

lib/private/Collaboration/Reference/File/FileReferenceEventListener.php

@@ -0,0 +1,61 @@
+<?php
+
+declare(strict_types=1);
+/**
+ * @copyright Copyright (c) 2022 Julius Härtl <jus@bitgrid.net>
+ *
+ * @author Julius Härtl <jus@bitgrid.net>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+namespace OC\Collaboration\Reference\File;
+
+use OCP\Collaboration\Reference\IReferenceManager;
+use OCP\EventDispatcher\Event;
+use OCP\EventDispatcher\IEventDispatcher;
+use OCP\Files\Events\Node\NodeDeletedEvent;
+use OCP\Share\Events\ShareCreatedEvent;
+use OCP\Share\Events\ShareDeletedEvent;
+
+class FileReferenceEventListener implements \OCP\EventDispatcher\IEventListener {
+	private IReferenceManager $manager;
+
+	public function __construct(IReferenceManager $manager) {
+		$this->manager = $manager;
+	}
+
+	public static function register(IEventDispatcher $eventDispatcher): void {
+		$eventDispatcher->addServiceListener(NodeDeletedEvent::class, FileReferenceEventListener::class);
+		$eventDispatcher->addServiceListener(ShareDeletedEvent::class, FileReferenceEventListener::class);
+		$eventDispatcher->addServiceListener(ShareCreatedEvent::class, FileReferenceEventListener::class);
+	}
+
+	/**
+	 * @inheritDoc
+	 */
+	public function handle(Event $event): void {
+		if ($event instanceof NodeDeletedEvent) {
+			$this->manager->invalidateCache((string)$event->getNode()->getId());
+		}
+		if ($event instanceof ShareDeletedEvent) {
+			$this->manager->invalidateCache((string)$event->getShare()->getNodeId());
+		}
+		if ($event instanceof ShareCreatedEvent) {
+			$this->manager->invalidateCache((string)$event->getShare()->getNodeId());
+		}
+	}
+}

lib/private/Collaboration/Reference/FileReferenceProvider.php → lib/private/Collaboration/Reference/File/FileReferenceProvider.php

@@ -22,8 +22,9 @@ declare(strict_types=1);
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-namespace OC\Collaboration\Reference;
+namespace OC\Collaboration\Reference\File;
 
+use OC\Collaboration\Reference\Reference;
 use OC\User\NoUserException;
 use OCP\Collaboration\Reference\IReference;
 use OCP\Collaboration\Reference\IReferenceProvider;
@@ -50,8 +51,38 @@ class FileReferenceProvider implements IReferenceProvider {
 	}
 
 	public function matchReference(string $referenceText): bool {
-		return str_starts_with($referenceText, $this->urlGenerator->getAbsoluteURL('/index.php/f/'))
-			|| str_starts_with($referenceText, $this->urlGenerator->getAbsoluteURL('/f/'));
+		return $this->getFilesAppLinkId($referenceText) !== null;
+	}
+
+	private function getFilesAppLinkId(string $referenceText): ?int {
+		$start = $this->urlGenerator->getAbsoluteURL('/apps/files');
+		$startIndex = $this->urlGenerator->getAbsoluteURL('/index.php/apps/files');
+
+		$fileId = null;
+
+		if (mb_strpos($referenceText, $start) === 0) {
+			$parts = parse_url($referenceText);
+			parse_str($parts['query'], $query);
+			$fileId = isset($query['fileid']) ? (int)$query['fileid'] : $fileId;
+			$fileId = isset($query['openfile']) ? (int)$query['openfile'] : $fileId;
+		}
+
+		if (mb_strpos($referenceText, $startIndex) === 0) {
+			$parts = parse_url($referenceText);
+			parse_str($parts['query'], $query);
+			$fileId = isset($query['fileid']) ? (int)$query['fileid'] : $fileId;
+			$fileId = isset($query['openfile']) ? (int)$query['openfile'] : $fileId;
+		}
+
+		if (mb_strpos($referenceText, $this->urlGenerator->getAbsoluteURL('/index.php/f/')) === 0) {
+			$fileId = str_replace($this->urlGenerator->getAbsoluteURL('/index.php/f/'), '', $referenceText);
+		}
+
+		if (mb_strpos($referenceText, $this->urlGenerator->getAbsoluteURL('/f/')) === 0) {
+			$fileId = str_replace($this->urlGenerator->getAbsoluteURL('/f/'), '', $referenceText);
+		}
+
+		return $fileId !== null ? (int)$fileId : null;
 	}
 
 	public function resolveReference(string $referenceText): ?IReference {
@@ -77,12 +108,14 @@ class FileReferenceProvider implements IReferenceProvider {
 			throw new NotFoundException();
 		}
 
-		$fileId = str_replace($this->urlGenerator->getAbsoluteURL('/index.php/f/'), '', $reference->getId());
-		$fileId = str_replace($this->urlGenerator->getAbsoluteURL('/f/'), '', $fileId);
+		$fileId = $this->getFilesAppLinkId($reference->getId());
+		if ($fileId === null) {
+			throw new NotFoundException();
+		}
 
 		try {
 			$userFolder = $this->rootFolder->getUserFolder($this->userId);
-			$files = $userFolder->getById((int)$fileId);
+			$files = $userFolder->getById($fileId);
 
 			if (empty($files)) {
 				throw new NotFoundException();
@@ -110,11 +143,11 @@ class FileReferenceProvider implements IReferenceProvider {
 		}
 	}
 
-	public function isGloballyCacheable(): bool {
-		return false;
+	public function getCachePrefix(string $referenceId): string {
+		return (string)$this->getFilesAppLinkId($referenceId);
 	}
 
-	public function getCacheKey(string $referenceId): string {
+	public function getCacheKey(string $referenceId): ?string {
 		return $this->userId ?? '';
 	}
 }

lib/private/Collaboration/Reference/LinkReferenceProvider.php

@@ -106,12 +106,13 @@ class LinkReferenceProvider implements IReferenceProvider {
 
 		$responseBody = (string)$response->getBody();
 
-		$reference->setUrl($reference->getId());
-
 		// OpenGraph handling
 		$consumer = new Consumer();
+		$consumer->useFallbackMode = true;
 		$object = $consumer->loadHtml($responseBody);
 
+		$reference->setUrl($reference->getId());
+
 		if ($object->title) {
 			$reference->setTitle($object->title);
 		}
@@ -145,11 +146,11 @@ class LinkReferenceProvider implements IReferenceProvider {
 		}
 	}
 
-	public function isGloballyCacheable(): bool {
-		return true;
+	public function getCachePrefix(string $referenceId): string {
+		return $referenceId;
 	}
 
-	public function getCacheKey(string $referenceId): string {
-		return '';
+	public function getCacheKey(string $referenceId): ?string {
+		return null;
 	}
 }

lib/private/Collaboration/Reference/ReferenceManager.php

@@ -25,6 +25,7 @@ declare(strict_types=1);
 namespace OC\Collaboration\Reference;
 
 use OC\AppFramework\Bootstrap\Coordinator;
+use OC\Collaboration\Reference\File\FileReferenceProvider;
 use OCP\Collaboration\Reference\IReference;
 use OCP\Collaboration\Reference\IReferenceManager;
 use OCP\Collaboration\Reference\IReferenceProvider;
@@ -62,6 +63,17 @@ class ReferenceManager implements IReferenceManager {
 		}, $references);
 	}
 
+	public function getReferenceFromCache(string $referenceId): ?IReference {
+		$matchedProvider = $this->getMatchedProvider($referenceId);
+
+		if ($matchedProvider === null) {
+			return null;
+		}
+
+		$cacheKey = $this->getFullCacheKey($matchedProvider, $referenceId);
+		return $this->getReferenceByCacheKey($cacheKey);
+	}
+
 	public function getReferenceByCacheKey(string $cacheKey): ?IReference {
 		$cached = $this->cache->get($cacheKey);
 		if ($cached) {
@@ -78,7 +90,7 @@ class ReferenceManager implements IReferenceManager {
 			return null;
 		}
 
-		$cacheKey = $this->getCacheKey($matchedProvider, $referenceId);
+		$cacheKey = $this->getFullCacheKey($matchedProvider, $referenceId);
 		$cached = $this->cache->get($cacheKey);
 		if ($cached) {
 			return Reference::fromCache($cached);
@@ -109,27 +121,20 @@ class ReferenceManager implements IReferenceManager {
 		return $matchedProvider;
 	}
 
-	private function getCacheKey(IReferenceProvider $provider, string $referenceId): string {
-		return md5($referenceId) . (
-			$provider->isGloballyCacheable()
-				? ''
-				: '-' . md5($provider->getCacheKey($referenceId))
+	private function getFullCacheKey(IReferenceProvider $provider, string $referenceId): string {
+		$cacheKey = $provider->getCacheKey($referenceId);
+		return md5($provider->getCachePrefix($referenceId)) . (
+			$cacheKey !== null ? ('-' . md5($cacheKey)) : ''
 		);
 	}
 
-	public function invalidateCache(string $referenceId, ?string $providerCacheKey = null): void {
-		$matchedProvider = $this->getMatchedProvider($referenceId);
-
-		if ($matchedProvider === null) {
-			return;
-		}
-
-		if ($providerCacheKey === null) {
-			$this->cache->clear(md5($referenceId));
+	public function invalidateCache(string $cachePrefix, ?string $cacheKey = null): void {
+		if ($cacheKey === null) {
+			$this->cache->clear(md5($cachePrefix));
 			return;
 		}
 
-		$this->cache->remove($this->getCacheKey($matchedProvider, $referenceId));
+		$this->cache->remove(md5($cachePrefix) . '-' . md5($cacheKey));
 	}
 
 	/**

lib/public/Collaboration/Reference/IReferenceManager.php

@@ -31,6 +31,7 @@ interface IReferenceManager {
 	/**
 	 * Return all reference identifiers within a string as an array
 	 *
+	 * @return string[] Array of found references (urls)
 	 * @since 25.0.0
 	 */
 	public function extractReferences(string $text): array;
@@ -44,4 +45,26 @@ interface IReferenceManager {
 	 * @since 25.0.0
 	 */
 	public function resolveReference(string $referenceId): ?IReference;
+
+	/**
+	 * Get a reference by its cache key
+	 *
+	 * @since 25.0.0
+	 */
+	public function getReferenceByCacheKey(string $cacheKey): ?IReference;
+
+	/**
+	 * Explicitly get a reference from the cache to avoid heavy fetches for cases
+	 * the cache can then be filled with a separate request from the frontend
+	 *
+	 * @since 25.0.0
+	 */
+	public function getReferenceFromCache(string $referenceId): ?IReference;
+
+	/**
+	 * Invalidate all cache entries with a prefix or just one if the cache key is provided
+	 *
+	 * @since 25.0.0
+	 */
+	public function invalidateCache(string $cachePrefix, ?string $cacheKey = null): void;
 }

lib/public/Collaboration/Reference/IReferenceProvider.php

@@ -47,14 +47,17 @@ interface IReferenceProvider {
 	 *
 	 * @since 25.0.0
 	 */
-	public function isGloballyCacheable(): bool;
+	public function getCachePrefix(string $referenceId): string;
 
 	/**
 	 * Return a custom cache key to be used for caching the metadata
 	 * This could be for example the current user id if the reference
 	 * access permissions are different for each user
 	 *
+	 * Should return null, if the cache is only related to the
+	 * reference id and has no further dependency
+	 *
 	 * @since 25.0.0
 	 */
-	public function getCacheKey(string $referenceId): string;
+	public function getCacheKey(string $referenceId): ?string;
 }