Merge pull request #44130 from nextcloud/fix/dav/occ-fix-caldav-sync-tokens

fix(dav): Add occ command to fix missing caldav sync tokens

apps/dav/appinfo/info.xml

@@ -50,6 +50,7 @@
 		<command>OCA\DAV\Command\CreateAddressBook</command>
 		<command>OCA\DAV\Command\CreateCalendar</command>
 		<command>OCA\DAV\Command\DeleteCalendar</command>
+		<command>OCA\DAV\Command\FixCalendarSyncCommand</command>
 		<command>OCA\DAV\Command\MoveCalendar</command>
 		<command>OCA\DAV\Command\ListCalendars</command>
 		<command>OCA\DAV\Command\RetentionCleanupCommand</command>

apps/dav/composer/composer/autoload_classmap.php

@@ -141,6 +141,7 @@ return array(
     'OCA\\DAV\\Command\\CreateAddressBook' => $baseDir . '/../lib/Command/CreateAddressBook.php',
     'OCA\\DAV\\Command\\CreateCalendar' => $baseDir . '/../lib/Command/CreateCalendar.php',
     'OCA\\DAV\\Command\\DeleteCalendar' => $baseDir . '/../lib/Command/DeleteCalendar.php',
+    'OCA\\DAV\\Command\\FixCalendarSyncCommand' => $baseDir . '/../lib/Command/FixCalendarSyncCommand.php',
     'OCA\\DAV\\Command\\ListCalendars' => $baseDir . '/../lib/Command/ListCalendars.php',
     'OCA\\DAV\\Command\\MoveCalendar' => $baseDir . '/../lib/Command/MoveCalendar.php',
     'OCA\\DAV\\Command\\RemoveInvalidShares' => $baseDir . '/../lib/Command/RemoveInvalidShares.php',

apps/dav/composer/composer/autoload_static.php

@@ -156,6 +156,7 @@ class ComposerStaticInitDAV
         'OCA\\DAV\\Command\\CreateAddressBook' => __DIR__ . '/..' . '/../lib/Command/CreateAddressBook.php',
         'OCA\\DAV\\Command\\CreateCalendar' => __DIR__ . '/..' . '/../lib/Command/CreateCalendar.php',
         'OCA\\DAV\\Command\\DeleteCalendar' => __DIR__ . '/..' . '/../lib/Command/DeleteCalendar.php',
+        'OCA\\DAV\\Command\\FixCalendarSyncCommand' => __DIR__ . '/..' . '/../lib/Command/FixCalendarSyncCommand.php',
         'OCA\\DAV\\Command\\ListCalendars' => __DIR__ . '/..' . '/../lib/Command/ListCalendars.php',
         'OCA\\DAV\\Command\\MoveCalendar' => __DIR__ . '/..' . '/../lib/Command/MoveCalendar.php',
         'OCA\\DAV\\Command\\RemoveInvalidShares' => __DIR__ . '/..' . '/../lib/Command/RemoveInvalidShares.php',

apps/dav/lib/CalDAV/CalDavBackend.php

@@ -98,6 +98,7 @@ use Sabre\VObject\Property;
 use Sabre\VObject\Reader;
 use Sabre\VObject\Recur\EventIterator;
 use function array_column;
+use function array_map;
 use function array_merge;
 use function array_values;
 use function explode;
@@ -874,7 +875,7 @@ class CalDavBackend extends AbstractBackend implements SyncSupport, Subscription
 				$query->where($query->expr()->eq('id', $query->createNamedParameter($calendarId)));
 				$query->executeStatement();
 
-				$this->addChange($calendarId, "", 2);
+				$this->addChanges($calendarId, [""], 2);
 
 				$calendarData = $this->getCalendarById($calendarId);
 				$shares = $this->getShares($calendarId);
@@ -1295,7 +1296,7 @@ class CalDavBackend extends AbstractBackend implements SyncSupport, Subscription
 				->executeStatement();
 
 			$this->updateProperties($calendarId, $objectUri, $calendarData, $calendarType);
-			$this->addChange($calendarId, $objectUri, 1, $calendarType);
+			$this->addChanges($calendarId, [$objectUri], 1, $calendarType);
 
 			$objectRow = $this->getCalendarObject($calendarId, $objectUri, $calendarType);
 			assert($objectRow !== null);
@@ -1356,7 +1357,7 @@ class CalDavBackend extends AbstractBackend implements SyncSupport, Subscription
 				->executeStatement();
 
 			$this->updateProperties($calendarId, $objectUri, $calendarData, $calendarType);
-			$this->addChange($calendarId, $objectUri, 2, $calendarType);
+			$this->addChanges($calendarId, [$objectUri], 2, $calendarType);
 
 			$objectRow = $this->getCalendarObject($calendarId, $objectUri, $calendarType);
 			if (is_array($objectRow)) {
@@ -1406,8 +1407,8 @@ class CalDavBackend extends AbstractBackend implements SyncSupport, Subscription
 			$this->purgeProperties($sourceCalendarId, $objectId);
 			$this->updateProperties($targetCalendarId, $object['uri'], $object['calendardata'], $calendarType);
 
-			$this->addChange($sourceCalendarId, $object['uri'], 3, $calendarType);
-			$this->addChange($targetCalendarId, $object['uri'], 1, $calendarType);
+			$this->addChanges($sourceCalendarId, [$object['uri']], 3, $calendarType);
+			$this->addChanges($targetCalendarId, [$object['uri']], 1, $calendarType);
 
 			$object = $this->getCalendarObjectById($newPrincipalUri, $objectId);
 			// Calendar Object wasn't found - possibly because it was deleted in the meantime by a different client
@@ -1534,7 +1535,7 @@ class CalDavBackend extends AbstractBackend implements SyncSupport, Subscription
 				}
 			}
 
-			$this->addChange($calendarId, $objectUri, 3, $calendarType);
+			$this->addChanges($calendarId, [$objectUri], 3, $calendarType);
 		}, $this->db);
 	}
 
@@ -1576,7 +1577,7 @@ class CalDavBackend extends AbstractBackend implements SyncSupport, Subscription
 				// Welp, this should possibly not have happened, but let's ignore
 				return;
 			}
-			$this->addChange($row['calendarid'], $row['uri'], 1, (int) $row['calendartype']);
+			$this->addChanges($row['calendarid'], [$row['uri']], 1, (int) $row['calendartype']);
 
 			$calendarRow = $this->getCalendarById((int) $row['calendarid']);
 			if ($calendarRow === null) {
@@ -2758,16 +2759,16 @@ class CalDavBackend extends AbstractBackend implements SyncSupport, Subscription
 	 * Adds a change record to the calendarchanges table.
 	 *
 	 * @param mixed $calendarId
-	 * @param string $objectUri
+	 * @param string[] $objectUris
 	 * @param int $operation 1 = add, 2 = modify, 3 = delete.
 	 * @param int $calendarType
 	 * @return void
 	 */
-	protected function addChange(int $calendarId, string $objectUri, int $operation, int $calendarType = self::CALENDAR_TYPE_CALENDAR): void {
+	protected function addChanges(int $calendarId, array $objectUris, int $operation, int $calendarType = self::CALENDAR_TYPE_CALENDAR): void {
 		$this->cachedObjects = [];
 		$table = $calendarType === self::CALENDAR_TYPE_CALENDAR ? 'calendars': 'calendarsubscriptions';
 
-		$this->atomic(function () use ($calendarId, $objectUri, $operation, $calendarType, $table) {
+		$this->atomic(function () use ($calendarId, $objectUris, $operation, $calendarType, $table) {
 			$query = $this->db->getQueryBuilder();
 			$query->select('synctoken')
 				->from($table)
@@ -2779,13 +2780,16 @@ class CalDavBackend extends AbstractBackend implements SyncSupport, Subscription
 			$query = $this->db->getQueryBuilder();
 			$query->insert('calendarchanges')
 				->values([
-					'uri' => $query->createNamedParameter($objectUri),
+					'uri' => $query->createParameter('uri'),
 					'synctoken' => $query->createNamedParameter($syncToken),
 					'calendarid' => $query->createNamedParameter($calendarId),
 					'operation' => $query->createNamedParameter($operation),
 					'calendartype' => $query->createNamedParameter($calendarType),
-				])
-				->executeStatement();
+				]);
+			foreach ($objectUris as $uri) {
+				$query->setParameter('uri', $uri);
+				$query->executeStatement();
+			}
 
 			$query = $this->db->getQueryBuilder();
 			$query->update($table)
@@ -2795,6 +2799,47 @@ class CalDavBackend extends AbstractBackend implements SyncSupport, Subscription
 		}, $this->db);
 	}
 
+	public function restoreChanges(int $calendarId, int $calendarType = self::CALENDAR_TYPE_CALENDAR): void {
+		$this->cachedObjects = [];
+
+		$this->atomic(function () use ($calendarId, $calendarType) {
+			$qbAdded = $this->db->getQueryBuilder();
+			$qbAdded->select('uri')
+				->from('calendarobjects')
+				->where(
+					$qbAdded->expr()->andX(
+						$qbAdded->expr()->eq('calendarid', $qbAdded->createNamedParameter($calendarId)),
+						$qbAdded->expr()->eq('calendartype', $qbAdded->createNamedParameter($calendarType)),
+						$qbAdded->expr()->isNull('deleted_at'),
+					)
+				);
+			$resultAdded = $qbAdded->executeQuery();
+			$addedUris = $resultAdded->fetchAll(\PDO::FETCH_COLUMN);
+			$resultAdded->closeCursor();
+			// Track everything as changed
+			// Tracking the creation is not necessary because \OCA\DAV\CalDAV\CalDavBackend::getChangesForCalendar
+			// only returns the last change per object.
+			$this->addChanges($calendarId, $addedUris, 2, $calendarType);
+
+			$qbDeleted = $this->db->getQueryBuilder();
+			$qbDeleted->select('uri')
+				->from('calendarobjects')
+				->where(
+					$qbDeleted->expr()->andX(
+						$qbDeleted->expr()->eq('calendarid', $qbDeleted->createNamedParameter($calendarId)),
+						$qbDeleted->expr()->eq('calendartype', $qbDeleted->createNamedParameter($calendarType)),
+						$qbDeleted->expr()->isNotNull('deleted_at'),
+					)
+				);
+			$resultDeleted = $qbDeleted->executeQuery();
+			$deletedUris = array_map(function (string $uri) {
+				return str_replace("-deleted.ics", ".ics", $uri);
+			}, $resultDeleted->fetchAll(\PDO::FETCH_COLUMN));
+			$resultDeleted->closeCursor();
+			$this->addChanges($calendarId, $deletedUris, 3, $calendarType);
+		}, $this->db);
+	}
+
 	/**
 	 * Parses some information from calendar objects, used for optimized
 	 * calendar-queries.
@@ -3137,9 +3182,7 @@ class CalDavBackend extends AbstractBackend implements SyncSupport, Subscription
 				->andWhere($query->expr()->eq('calendartype', $query->createNamedParameter(self::CALENDAR_TYPE_SUBSCRIPTION)))
 				->executeStatement();
 
-			foreach ($uris as $uri) {
-				$this->addChange($subscriptionId, $uri, 3, self::CALENDAR_TYPE_SUBSCRIPTION);
-			}
+			$this->addChanges($subscriptionId, $uris, 3, self::CALENDAR_TYPE_SUBSCRIPTION);
 		}, $this->db);
 	}
 

apps/dav/lib/Command/FixCalendarSyncCommand.php

@@ -0,0 +1,86 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * @copyright 2024 Christoph Wurst <christoph@winzerhof-wurst.at>
+ *
+ * @author 2024 Christoph Wurst <christoph@winzerhof-wurst.at>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+namespace OCA\DAV\Command;
+
+use OCA\DAV\CalDAV\CalDavBackend;
+use OCP\IUser;
+use OCP\IUserManager;
+use Symfony\Component\Console\Command\Command;
+use Symfony\Component\Console\Helper\ProgressBar;
+use Symfony\Component\Console\Input\InputArgument;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Output\OutputInterface;
+
+class FixCalendarSyncCommand extends Command {
+
+	public function __construct(private IUserManager $userManager,
+		private CalDavBackend $calDavBackend) {
+		parent::__construct('dav:fix-missing-caldav-changes');
+	}
+
+	protected function configure(): void {
+		$this->setDescription('Insert missing calendarchanges rows for existing events');
+		$this->addArgument(
+			'user',
+			InputArgument::OPTIONAL,
+			'User to fix calendar sync tokens for, if omitted all users will be fixed',
+			null,
+		);
+	}
+
+	public function execute(InputInterface $input, OutputInterface $output): int {
+		$userArg = $input->getArgument('user');
+		if ($userArg !== null) {
+			$user = $this->userManager->get($userArg);
+			if ($user === null) {
+				$output->writeln("<error>User $userArg does not exist</error>");
+				return 1;
+			}
+
+			$this->fixUserCalendars($user);
+		} else {
+			$progress = new ProgressBar($output);
+			$this->userManager->callForSeenUsers(function (IUser $user) use ($progress) {
+				$this->fixUserCalendars($user, $progress);
+			});
+			$progress->finish();
+		}
+		return 0;
+	}
+
+	private function fixUserCalendars(IUser $user, ?ProgressBar $progress = null): void {
+		$calendars = $this->calDavBackend->getCalendarsForUser("principals/users/" . $user->getUID());
+
+		foreach ($calendars as $calendar) {
+			$this->calDavBackend->restoreChanges($calendar['id']);
+		}
+
+		if ($progress !== null) {
+			$progress->advance();
+		}
+	}
+
+}

apps/dav/tests/unit/CalDAV/AbstractCalDavBackend.php

@@ -39,6 +39,7 @@ use OCP\App\IAppManager;
 use OCP\EventDispatcher\IEventDispatcher;
 use OCP\ICacheFactory;
 use OCP\IConfig;
+use OCP\IDBConnection;
 use OCP\IGroupManager;
 use OCP\IUserManager;
 use OCP\IUserSession;
@@ -70,6 +71,7 @@ abstract class AbstractCalDavBackend extends TestCase {
 	private IConfig|MockObject $config;
 	private ISecureRandom $random;
 	protected SharingBackend $sharingBackend;
+	protected IDBConnection $db;
 	public const UNIT_TEST_USER = 'principals/users/caldav-unit-test';
 	public const UNIT_TEST_USER1 = 'principals/users/caldav-unit-test1';
 	public const UNIT_TEST_GROUP = 'principals/groups/caldav-unit-test-group';
@@ -105,7 +107,7 @@ abstract class AbstractCalDavBackend extends TestCase {
 			->withAnyParameters()
 			->willReturn([self::UNIT_TEST_GROUP, self::UNIT_TEST_GROUP2]);
 
-		$db = \OC::$server->getDatabaseConnection();
+		$this->db = \OC::$server->getDatabaseConnection();
 		$this->random = \OC::$server->getSecureRandom();
 		$this->logger = $this->createMock(LoggerInterface::class);
 		$this->config = $this->createMock(IConfig::class);
@@ -114,10 +116,10 @@ abstract class AbstractCalDavBackend extends TestCase {
 			$this->groupManager,
 			$this->principal,
 			$this->createMock(ICacheFactory::class),
-			new Service(new SharingMapper($db)),
+			new Service(new SharingMapper($this->db)),
 			$this->logger);
 		$this->backend = new CalDavBackend(
-			$db,
+			$this->db,
 			$this->principal,
 			$this->userManager,
 			$this->random,

apps/dav/tests/unit/CalDAV/CalDavBackendTest.php

@@ -1502,4 +1502,93 @@ EOD;
 			}
 		}
 	}
+
+	public function testRestoreChanges(): void {
+		$calendarId = $this->createTestCalendar();
+		$uri1 = static::getUniqueID('calobj1') . '.ics';
+		$calData = <<<EOD
+BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:Nextcloud Calendar
+BEGIN:VEVENT
+CREATED;VALUE=DATE-TIME:20130910T125139Z
+UID:47d15e3ec8
+LAST-MODIFIED;VALUE=DATE-TIME:20130910T125139Z
+DTSTAMP;VALUE=DATE-TIME:20130910T125139Z
+SUMMARY:Test Event
+DTSTART;VALUE=DATE-TIME:20130912T130000Z
+DTEND;VALUE=DATE-TIME:20130912T140000Z
+CLASS:PUBLIC
+END:VEVENT
+END:VCALENDAR
+EOD;
+		$this->backend->createCalendarObject($calendarId, $uri1, $calData);
+		$calData = <<<EOD
+BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:Nextcloud Calendar
+BEGIN:VEVENT
+CREATED;VALUE=DATE-TIME:20130910T125139Z
+UID:47d15e3ec8
+LAST-MODIFIED;VALUE=DATE-TIME:20130910T125139Z
+DTSTAMP;VALUE=DATE-TIME:20130910T125139Z
+SUMMARY:Test Event – UPDATED
+DTSTART;VALUE=DATE-TIME:20130912T130000Z
+DTEND;VALUE=DATE-TIME:20130912T140000Z
+CLASS:PUBLIC
+SEQUENCE:1
+END:VEVENT
+END:VCALENDAR
+EOD;
+		$this->backend->updateCalendarObject($calendarId, $uri1, $calData);
+		$uri2 = static::getUniqueID('calobj2') . '.ics';
+		$calData = <<<EOD
+BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:Nextcloud Calendar
+BEGIN:VEVENT
+CREATED;VALUE=DATE-TIME:20130910T125139Z
+UID:47d15e3ec9
+LAST-MODIFIED;VALUE=DATE-TIME:20130910T125139Z
+DTSTAMP;VALUE=DATE-TIME:20130910T125139Z
+SUMMARY:Test Event
+DTSTART;VALUE=DATE-TIME:20130912T130000Z
+DTEND;VALUE=DATE-TIME:20130912T140000Z
+CLASS:PUBLIC
+END:VEVENT
+END:VCALENDAR
+EOD;
+		$this->backend->createCalendarObject($calendarId, $uri2, $calData);
+		$changesBefore = $this->backend->getChangesForCalendar($calendarId, null, 1);
+		$this->backend->deleteCalendarObject($calendarId, $uri2);
+		$uri3 = static::getUniqueID('calobj3') . '.ics';
+		$calData = <<<EOD
+BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:Nextcloud Calendar
+BEGIN:VEVENT
+CREATED;VALUE=DATE-TIME:20130910T125139Z
+UID:47d15e3e10
+LAST-MODIFIED;VALUE=DATE-TIME:20130910T125139Z
+DTSTAMP;VALUE=DATE-TIME:20130910T125139Z
+SUMMARY:Test Event
+DTSTART;VALUE=DATE-TIME:20130912T130000Z
+DTEND;VALUE=DATE-TIME:20130912T140000Z
+CLASS:PUBLIC
+END:VEVENT
+END:VCALENDAR
+EOD;
+		$this->backend->createCalendarObject($calendarId, $uri3, $calData);
+		$deleteChanges = $this->db->getQueryBuilder();
+		$deleteChanges->delete('calendarchanges')
+			->where($deleteChanges->expr()->eq('calendarid', $deleteChanges->createNamedParameter($calendarId)));
+		$deleteChanges->executeStatement();
+
+		$this->backend->restoreChanges($calendarId);
+
+		$changesAfter = $this->backend->getChangesForCalendar($calendarId, $changesBefore['syncToken'], 1);
+		self::assertEquals([], $changesAfter['added']);
+		self::assertEqualsCanonicalizing([$uri1, $uri3], $changesAfter['modified']);
+		self::assertEquals([$uri2], $changesAfter['deleted']);
+	}
 }