Browse Source

Store storage availability in database

Storage status is saved in the database. Failed storages are rechecked every
10 minutes, while working storages are rechecked every request.

Using the files_external app will recheck all external storages when the
settings page is viewed, or whenever an external storage is saved.
remotes/origin/handlebars-approach
Robin McCorkell 12 years ago
parent
commit
df19cabb44
  1. 12
      apps/files_external/lib/config.php
  2. 12
      db_structure.xml
  3. 48
      lib/private/files/cache/storage.php
  4. 19
      lib/private/files/storage/common.php
  5. 462
      lib/private/files/storage/wrapper/availability.php
  6. 18
      lib/private/files/storage/wrapper/wrapper.php
  7. 8
      lib/private/util.php
  8. 20
      lib/public/files/storage.php
  9. 149
      tests/lib/files/storage/wrapper/availability.php
  10. 2
      version.php

12
apps/files_external/lib/config.php

@ -494,8 +494,16 @@ class OC_Mount_Config {
if (class_exists($class)) {
try {
$storage = new $class($options);
if ($storage->test($isPersonal)) {
return self::STATUS_SUCCESS;
try {
$result = $storage->test($isPersonal);
$storage->setAvailability($result);
if ($result) {
return self::STATUS_SUCCESS;
}
} catch (\Exception $e) {
$storage->setAvailability(false);
throw $e;
}
} catch (Exception $exception) {
\OCP\Util::logException('files_external', $exception);

12
db_structure.xml

@ -102,6 +102,18 @@
<length>4</length>
</field>
<field>
<name>available</name>
<type>boolean</type>
<default>true</default>
<notnull>true</notnull>
</field>
<field>
<name>last_checked</name>
<type>integer</type>
</field>
<index>
<name>storages_id_index</name>
<unique>true</unique>

48
lib/private/files/cache/storage.php

@ -43,9 +43,10 @@ class Storage {
/**
* @param \OC\Files\Storage\Storage|string $storage
* @param bool $isAvailable
* @throws \RuntimeException
*/
public function __construct($storage) {
public function __construct($storage, $isAvailable = true) {
if ($storage instanceof \OC\Files\Storage\Storage) {
$this->storageId = $storage->getId();
} else {
@ -53,17 +54,14 @@ class Storage {
}
$this->storageId = self::adjustStorageId($this->storageId);
$sql = 'SELECT `numeric_id` FROM `*PREFIX*storages` WHERE `id` = ?';
$result = \OC_DB::executeAudited($sql, array($this->storageId));
if ($row = $result->fetchRow()) {
if ($row = self::getStorageById($this->storageId)) {
$this->numericId = $row['numeric_id'];
} else {
$connection = \OC_DB::getConnection();
if ($connection->insertIfNotExist('*PREFIX*storages', ['id' => $this->storageId])) {
if ($connection->insertIfNotExist('*PREFIX*storages', ['id' => $this->storageId, 'available' => $isAvailable])) {
$this->numericId = \OC_DB::insertid('*PREFIX*storages');
} else {
$result = \OC_DB::executeAudited($sql, array($this->storageId));
if ($row = $result->fetchRow()) {
if ($row = self::getStorageById($this->storageId)) {
$this->numericId = $row['numeric_id'];
} else {
throw new \RuntimeException('Storage could neither be inserted nor be selected from the database');
@ -72,6 +70,16 @@ class Storage {
}
}
/**
* @param string $storageId
* @return array|null
*/
public static function getStorageById($storageId) {
$sql = 'SELECT * FROM `*PREFIX*storages` WHERE `id` = ?';
$result = \OC_DB::executeAudited($sql, array($storageId));
return $result->fetchRow();
}
/**
* Adjusts the storage id to use md5 if too long
* @param string $storageId storage id
@ -120,15 +128,35 @@ class Storage {
public static function getNumericStorageId($storageId) {
$storageId = self::adjustStorageId($storageId);
$sql = 'SELECT `numeric_id` FROM `*PREFIX*storages` WHERE `id` = ?';
$result = \OC_DB::executeAudited($sql, array($storageId));
if ($row = $result->fetchRow()) {
if ($row = self::getStorageById($storageId)) {
return $row['numeric_id'];
} else {
return null;
}
}
/**
* @return array|null [ available, last_checked ]
*/
public function getAvailability() {
if ($row = self::getStorageById($this->storageId)) {
return [
'available' => $row['available'],
'last_checked' => $row['last_checked']
];
} else {
return null;
}
}
/**
* @param bool $isAvailable
*/
public function setAvailability($isAvailable) {
$sql = 'UPDATE `*PREFIX*storages` SET `available` = ?, `last_checked` = ? WHERE `id` = ?';
\OC_DB::executeAudited($sql, array($isAvailable, time(), $this->storageId));
}
/**
* Check if a string storage id is known
*

19
lib/private/files/storage/common.php

@ -404,6 +404,11 @@ abstract class Common implements Storage {
return implode('/', $output);
}
/**
* Test a storage for availability
*
* @return bool
*/
public function test() {
if ($this->stat('')) {
return true;
@ -650,4 +655,18 @@ abstract class Common implements Storage {
public function changeLock($path, $type, ILockingProvider $provider) {
$provider->changeLock('files/' . md5($this->getId() . '::' . trim($path, '/')), $type);
}
/**
* @return array [ available, last_checked ]
*/
public function getAvailability() {
return $this->getStorageCache()->getAvailability();
}
/**
* @param bool $isAvailable
*/
public function setAvailability($isAvailable) {
$this->getStorageCache()->setAvailability($isAvailable);
}
}

462
lib/private/files/storage/wrapper/availability.php

@ -0,0 +1,462 @@
<?php
/**
* @author Robin McCorkell <rmccorkell@karoshi.org.uk>
*
* @copyright Copyright (c) 2015, ownCloud, Inc.
* @license AGPL-3.0
*
* This code is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License, version 3,
* as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License, version 3,
* along with this program. If not, see <http://www.gnu.org/licenses/>
*
*/
namespace OC\Files\Storage\Wrapper;
/**
* Availability checker for storages
*
* Throws a StorageNotAvailableException for storages with known failures
*/
class Availability extends Wrapper {
const RECHECK_TTL_SEC = 600; // 10 minutes
/**
* @return bool
*/
private function updateAvailability() {
try {
$result = $this->test();
} catch (\Exception $e) {
$result = false;
}
$this->setAvailability($result);
return $result;
}
/**
* @return bool
*/
private function isAvailable() {
$availability = $this->getAvailability();
if (!$availability['available']) {
// trigger a recheck if TTL reached
if ((time() - $availability['last_checked']) > self::RECHECK_TTL_SEC) {
return $this->updateAvailability();
}
}
return $availability['available'];
}
/**
* @throws \OCP\Files\StorageNotAvailableException
*/
private function checkAvailability() {
if (!$this->isAvailable()) {
throw new \OCP\Files\StorageNotAvailableException();
}
}
/** {@inheritdoc} */
public function mkdir($path) {
$this->checkAvailability();
try {
return parent::mkdir($path);
} catch (\OCP\Files\StorageNotAvailableException $e) {
$this->setAvailability(false);
throw $e;
}
}
/** {@inheritdoc} */
public function rmdir($path) {
$this->checkAvailability();
try {
return parent::rmdir($path);
} catch (\OCP\Files\StorageNotAvailableException $e) {
$this->setAvailability(false);
throw $e;
}
}
/** {@inheritdoc} */
public function opendir($path) {
$this->checkAvailability();
try {
return parent::opendir($path);
} catch (\OCP\Files\StorageNotAvailableException $e) {
$this->setAvailability(false);
throw $e;
}
}
/** {@inheritdoc} */
public function is_dir($path) {
$this->checkAvailability();
try {
return parent::is_dir($path);
} catch (\OCP\Files\StorageNotAvailableException $e) {
$this->setAvailability(false);
throw $e;
}
}
/** {@inheritdoc} */
public function is_file($path) {
$this->checkAvailability();
try {
return parent::is_file($path);
} catch (\OCP\Files\StorageNotAvailableException $e) {
$this->setAvailability(false);
throw $e;
}
}
/** {@inheritdoc} */
public function stat($path) {
$this->checkAvailability();
try {
return parent::stat($path);
} catch (\OCP\Files\StorageNotAvailableException $e) {
$this->setAvailability(false);
throw $e;
}
}
/** {@inheritdoc} */
public function filetype($path) {
$this->checkAvailability();
try {
return parent::filetype($path);
} catch (\OCP\Files\StorageNotAvailableException $e) {
$this->setAvailability(false);
throw $e;
}
}
/** {@inheritdoc} */
public function filesize($path) {
$this->checkAvailability();
try {
return parent::filesize($path);
} catch (\OCP\Files\StorageNotAvailableException $e) {
$this->setAvailability(false);
throw $e;
}
}
/** {@inheritdoc} */
public function isCreatable($path) {
$this->checkAvailability();
try {
return parent::isCreatable($path);
} catch (\OCP\Files\StorageNotAvailableException $e) {
$this->setAvailability(false);
throw $e;
}
}
/** {@inheritdoc} */
public function isReadable($path) {
$this->checkAvailability();
try {
return parent::isReadable($path);
} catch (\OCP\Files\StorageNotAvailableException $e) {
$this->setAvailability(false);
throw $e;
}
}
/** {@inheritdoc} */
public function isUpdatable($path) {
$this->checkAvailability();
try {
return parent::isUpdatable($path);
} catch (\OCP\Files\StorageNotAvailableException $e) {
$this->setAvailability(false);
throw $e;
}
}
/** {@inheritdoc} */
public function isDeletable($path) {
$this->checkAvailability();
try {
return parent::isDeletable($path);
} catch (\OCP\Files\StorageNotAvailableException $e) {
$this->setAvailability(false);
throw $e;
}
}
/** {@inheritdoc} */
public function isSharable($path) {
$this->checkAvailability();
try {
return parent::isSharable($path);
} catch (\OCP\Files\StorageNotAvailableException $e) {
$this->setAvailability(false);
throw $e;
}
}
/** {@inheritdoc} */
public function getPermissions($path) {
$this->checkAvailability();
try {
return parent::getPermissions($path);
} catch (\OCP\Files\StorageNotAvailableException $e) {
$this->setAvailability(false);
throw $e;
}
}
/** {@inheritdoc} */
public function file_exists($path) {
$this->checkAvailability();
try {
return parent::file_exists($path);
} catch (\OCP\Files\StorageNotAvailableException $e) {
$this->setAvailability(false);
throw $e;
}
}
/** {@inheritdoc} */
public function filemtime($path) {
$this->checkAvailability();
try {
return parent::filemtime($path);
} catch (\OCP\Files\StorageNotAvailableException $e) {
$this->setAvailability(false);
throw $e;
}
}
/** {@inheritdoc} */
public function file_get_contents($path) {
$this->checkAvailability();
try {
return parent::file_get_contents($path);
} catch (\OCP\Files\StorageNotAvailableException $e) {
$this->setAvailability(false);
throw $e;
}
}
/** {@inheritdoc} */
public function file_put_contents($path, $data) {
$this->checkAvailability();
try {
return parent::file_put_contents($path, $data);
} catch (\OCP\Files\StorageNotAvailableException $e) {
$this->setAvailability(false);
throw $e;
}
}
/** {@inheritdoc} */
public function unlink($path) {
$this->checkAvailability();
try {
return parent::unlink($path);
} catch (\OCP\Files\StorageNotAvailableException $e) {
$this->setAvailability(false);
throw $e;
}
}
/** {@inheritdoc} */
public function rename($path1, $path2) {
$this->checkAvailability();
try {
return parent::rename($path1, $path2);
} catch (\OCP\Files\StorageNotAvailableException $e) {
$this->setAvailability(false);
throw $e;
}
}
/** {@inheritdoc} */
public function copy($path1, $path2) {
$this->checkAvailability();
try {
return parent::copy($path1, $path2);
} catch (\OCP\Files\StorageNotAvailableException $e) {
$this->setAvailability(false);
throw $e;
}
}
/** {@inheritdoc} */
public function fopen($path, $mode) {
$this->checkAvailability();
try {
return parent::fopen($path, $mode);
} catch (\OCP\Files\StorageNotAvailableException $e) {
$this->setAvailability(false);
throw $e;
}
}
/** {@inheritdoc} */
public function getMimeType($path) {
$this->checkAvailability();
try {
return parent::getMimeType($path);
} catch (\OCP\Files\StorageNotAvailableException $e) {
$this->setAvailability(false);
throw $e;
}
}
/** {@inheritdoc} */
public function hash($type, $path, $raw = false) {
$this->checkAvailability();
try {
return parent::hash($type, $path, $raw);
} catch (\OCP\Files\StorageNotAvailableException $e) {
$this->setAvailability(false);
throw $e;
}
}
/** {@inheritdoc} */
public function free_space($path) {
$this->checkAvailability();
try {
return parent::free_space($path);
} catch (\OCP\Files\StorageNotAvailableException $e) {
$this->setAvailability(false);
throw $e;
}
}
/** {@inheritdoc} */
public function search($query) {
$this->checkAvailability();
try {
return parent::search($query);
} catch (\OCP\Files\StorageNotAvailableException $e) {
$this->setAvailability(false);
throw $e;
}
}
/** {@inheritdoc} */
public function touch($path, $mtime = null) {
$this->checkAvailability();
try {
return parent::touch($path, $mtime);
} catch (\OCP\Files\StorageNotAvailableException $e) {
$this->setAvailability(false);
throw $e;
}
}
/** {@inheritdoc} */
public function getLocalFile($path) {
$this->checkAvailability();
try {
return parent::getLocalFile($path);
} catch (\OCP\Files\StorageNotAvailableException $e) {
$this->setAvailability(false);
throw $e;
}
}
/** {@inheritdoc} */
public function getLocalFolder($path) {
$this->checkAvailability();
try {
return parent::getLocalFolder($path);
} catch (\OCP\Files\StorageNotAvailableException $e) {
$this->setAvailability(false);
throw $e;
}
}
/** {@inheritdoc} */
public function hasUpdated($path, $time) {
$this->checkAvailability();
try {
return parent::hasUpdated($path, $time);
} catch (\OCP\Files\StorageNotAvailableException $e) {
$this->setAvailability(false);
throw $e;
}
}
/** {@inheritdoc} */
public function getOwner($path) {
$this->checkAvailability();
try {
return parent::getOwner($path);
} catch (\OCP\Files\StorageNotAvailableException $e) {
$this->setAvailability(false);
throw $e;
}
}
/** {@inheritdoc} */
public function getETag($path) {
$this->checkAvailability();
try {
return parent::getETag($path);
} catch (\OCP\Files\StorageNotAvailableException $e) {
$this->setAvailability(false);
throw $e;
}
}
/** {@inheritdoc} */
public function getDirectDownload($path) {
$this->checkAvailability();
try {
return parent::getDirectDownload($path);
} catch (\OCP\Files\StorageNotAvailableException $e) {
$this->setAvailability(false);
throw $e;
}
}
/** {@inheritdoc} */
public function copyFromStorage(\OCP\Files\Storage $sourceStorage, $sourceInternalPath, $targetInternalPath) {
$this->checkAvailability();
try {
return parent::copyFromStorage($sourceStorage, $sourceInternalPath, $targetInternalPath);
} catch (\OCP\Files\StorageNotAvailableException $e) {
$this->setAvailability(false);
throw $e;
}
}
/** {@inheritdoc} */
public function moveFromStorage(\OCP\Files\Storage $sourceStorage, $sourceInternalPath, $targetInternalPath) {
$this->checkAvailability();
try {
return parent::moveFromStorage($sourceStorage, $sourceInternalPath, $targetInternalPath);
} catch (\OCP\Files\StorageNotAvailableException $e) {
$this->setAvailability(false);
throw $e;
}
}
/** {@inheritdoc} */
public function getMetaData($path) {
$this->checkAvailability();
try {
return parent::getMetaData($path);
} catch (\OCP\Files\StorageNotAvailableException $e) {
$this->setAvailability(false);
throw $e;
}
}
}

18
lib/private/files/storage/wrapper/wrapper.php

@ -497,6 +497,24 @@ class Wrapper implements \OC\Files\Storage\Storage {
return $this->storage->getDirectDownload($path);
}
/**
* Get availability of the storage
*
* @return array [ available, last_checked ]
*/
public function getAvailability() {
return $this->storage->getAvailability();
}
/**
* Set availability of the storage
*
* @param bool $isAvailable
*/
public function setAvailability($isAvailable) {
$this->storage->setAvailability($isAvailable);
}
/**
* @param string $path the path of the target folder
* @param string $fileName the name of the file itself

8
lib/private/util.php

@ -143,6 +143,14 @@ class OC_Util {
return $storage;
});
// install storage availability wrapper, before most other wrappers
\OC\Files\Filesystem::addStorageWrapper('oc_availability', function ($mountPoint, $storage) {
if (!$storage->isLocal()) {
return new \OC\Files\Storage\Wrapper\Availability(['storage' => $storage]);
}
return $storage;
});
\OC\Files\Filesystem::addStorageWrapper('oc_quota', function ($mountPoint, $storage) {
// set up quota for home storages, even for other users
// which can happen when using sharing

20
lib/public/files/storage.php

@ -439,4 +439,24 @@ interface Storage {
* @since 8.1.0
*/
public function changeLock($path, $type, ILockingProvider $provider);
/**
* Test a storage for availability
*
* @since 8.2.0
* @return bool
*/
public function test();
/**
* @since 8.2.0
* @return array [ available, last_checked ]
*/
public function getAvailability();
/**
* @since 8.2.0
* @param bool $isAvailable
*/
public function setAvailability($isAvailable);
}

149
tests/lib/files/storage/wrapper/availability.php

@ -0,0 +1,149 @@
<?php
/**
* @author Robin McCorkell <rmccorkell@karoshi.org.uk>
*
* @copyright Copyright (c) 2015, ownCloud, Inc.
* @license AGPL-3.0
*
* This code is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License, version 3,
* as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License, version 3,
* along with this program. If not, see <http://www.gnu.org/licenses/>
*
*/
namespace Test\Files\Storage\Wrapper;
class Availability extends \Test\TestCase {
protected function getWrapperInstance() {
$storage = $this->getMockBuilder('\OC\Files\Storage\Temporary')
->disableOriginalConstructor()
->getMock();
$wrapper = new \OC\Files\Storage\Wrapper\Availability(['storage' => $storage]);
return [$storage, $wrapper];
}
/**
* Storage is available
*/
public function testAvailable() {
list($storage, $wrapper) = $this->getWrapperInstance();
$storage->expects($this->once())
->method('getAvailability')
->willReturn(['available' => true, 'last_checked' => 0]);
$storage->expects($this->never())
->method('test');
$storage->expects($this->once())
->method('mkdir');
$wrapper->mkdir('foobar');
}
/**
* Storage marked unavailable, TTL not expired
*
* @expectedException \OCP\Files\StorageNotAvailableException
*/
public function testUnavailable() {
list($storage, $wrapper) = $this->getWrapperInstance();
$storage->expects($this->once())
->method('getAvailability')
->willReturn(['available' => false, 'last_checked' => time()]);
$storage->expects($this->never())
->method('test');
$storage->expects($this->never())
->method('mkdir');
$wrapper->mkdir('foobar');
}
/**
* Storage marked unavailable, TTL expired
*/
public function testUnavailableRecheck() {
list($storage, $wrapper) = $this->getWrapperInstance();
$storage->expects($this->once())
->method('getAvailability')
->willReturn(['available' => false, 'last_checked' => 0]);
$storage->expects($this->once())
->method('test')
->willReturn(true);
$storage->expects($this->once())
->method('setAvailability')
->with($this->equalTo(true));
$storage->expects($this->once())
->method('mkdir');
$wrapper->mkdir('foobar');
}
/**
* Storage marked available, but throws StorageNotAvailableException
*
* @expectedException \OCP\Files\StorageNotAvailableException
*/
public function testAvailableThrowStorageNotAvailable() {
list($storage, $wrapper) = $this->getWrapperInstance();
$storage->expects($this->once())
->method('getAvailability')
->willReturn(['available' => true, 'last_checked' => 0]);
$storage->expects($this->never())
->method('test');
$storage->expects($this->once())
->method('mkdir')
->will($this->throwException(new \OCP\Files\StorageNotAvailableException()));
$storage->expects($this->once())
->method('setAvailability')
->with($this->equalTo(false));
$wrapper->mkdir('foobar');
}
/**
* Storage available, but call fails
* Method failure does not indicate storage unavailability
*/
public function testAvailableFailure() {
list($storage, $wrapper) = $this->getWrapperInstance();
$storage->expects($this->once())
->method('getAvailability')
->willReturn(['available' => true, 'last_checked' => 0]);
$storage->expects($this->never())
->method('test');
$storage->expects($this->once())
->method('mkdir')
->willReturn(false);
$storage->expects($this->never())
->method('setAvailability');
$wrapper->mkdir('foobar');
}
/**
* Storage available, but throws exception
* Standard exception does not indicate storage unavailability
*
* @expectedException \Exception
*/
public function testAvailableThrow() {
list($storage, $wrapper) = $this->getWrapperInstance();
$storage->expects($this->once())
->method('getAvailability')
->willReturn(['available' => true, 'last_checked' => 0]);
$storage->expects($this->never())
->method('test');
$storage->expects($this->once())
->method('mkdir')
->will($this->throwException(new \Exception()));
$storage->expects($this->never())
->method('setAvailability');
$wrapper->mkdir('foobar');
}
}

2
version.php

@ -22,7 +22,7 @@
// We only can count up. The 4. digit is only for the internal patchlevel to trigger DB upgrades
// between betas, final and RCs. This is _not_ the public version number. Reset minor/patchlevel
// when updating major/minor version number.
$OC_Version=array(8, 2, 0, 2);
$OC_Version=array(8, 2, 0, 3);
// The human readable string
$OC_VersionString='8.2 pre alpha';

Loading…
Cancel
Save