#35825【機関ストレージ】 タイムスタンプ処理性能の向上

.travis.yml

@@ -1,7 +1,7 @@
 language: php
 
 php:
-  - '7.4'
+  - '8.1'
 
 env:
   global:
@@ -19,7 +19,7 @@ before_script:
   - git clone https://github.com/nextcloud/server.git
   - mv server/ nextcloud
   - cd nextcloud
-  - git checkout origin/stable19 -b stable19
+  - git checkout origin/stable25 -b stable25
   - mkdir custom_apps
   - cp -pi ${TRAVIS_BUILD_DIR}/travis/apps.config.php config
   - cd 3rdparty
@@ -28,6 +28,10 @@ before_script:
   - php occ maintenance:install --database ${NEXTCLOUD_DB} --database-name ${NEXTCLOUD_DB_NAME} --database-user ${NEXTCLOUD_DB_USER} --database-pass ${NEXTCLOUD_DB_PASSWD} --admin-user ${NEXTCLOUD_ADMIN_NAME} --admin-pass ${NEXTCLOUD_ADMIN_PASSWD}
   - mkdir custom_apps/${NEXTCLOUD_APP}
   - cp -r ${TRAVIS_BUILD_DIR}/{appinfo,lib,tests} custom_apps/${NEXTCLOUD_APP}
+  - cp ${TRAVIS_BUILD_DIR}/composer.json custom_apps/${NEXTCLOUD_APP}
+  - cd custom_apps/${NEXTCLOUD_APP} 
+  - composer install
+  - cd ../../
   - php occ app:enable ${NEXTCLOUD_APP}
 
 script:

README.md

@@ -10,6 +10,11 @@ query "hash" tag,sets MD5/SHA256/SHA512.
 
 # Usage
 
+- cd apps/checksum_api
+- Composer update
+- Composer dump-autoload
+
+
 ## Method 
 
  GET 

appinfo/info.xml

@@ -16,6 +16,6 @@
     <category>tools</category>
     <bugs>https://github.com/RCOSDP/nextcloud-checksum_api/issues</bugs>
     <dependencies>
-        <nextcloud min-version="15" max-version="19" />
+        <nextcloud min-version="25" max-version="26" />
     </dependencies>
 </info>

composer.json

@@ -0,0 +1,16 @@
+{
+  "autoload-dev": {
+    "psr-4": {
+      "OCA\\ChecksumAPI\\": "lib/"
+    }
+  },
+  "config": {
+    "optimize-autoloader": true
+  },
+  "require-dev": {
+    "phpunit/phpunit": "^9"
+  },
+  "require": {
+    "amphp/parallel": "^1.4"
+  }
+}

lib/Controller/ChecksumAPIController.php

@@ -1,38 +1,47 @@
 <?php
 
+
+
 declare(strict_types=1);
 
 namespace OCA\ChecksumAPI\Controller;
 
-use OC\Files\Filesystem;
+require __DIR__ . '../../../vendor/autoload.php';
+
 use OCP\AppFramework\Http;
 use OCP\AppFramework\Http\DataResponse;
 use OCP\AppFramework\OCSController;
 use OCP\Files\IRootFolder;
 use OCP\Files\NotFoundException;
-use OCP\IDBConnection;
-use OCP\ILogger;
 use OCP\IRequest;
 use OCP\IUserSession;
-
 use OCA\ChecksumAPI\Db\Hash;
 use OCA\ChecksumAPI\Db\HashMapper;
 
+use Amp\Parallel\Worker\DefaultPool;
+use Amp;
+use OCA\ChecksumAPI\Jobs\HashableTask;
+use Psr\Log\LoggerInterface;
+
 class ChecksumAPIController extends OCSController {
 
     private $rootFolder;
     private $userSession;
     private $mapper;
+    private $appManager;
     private $logger;
+    private $minFileSizeToExcuteParallel = 20971520;
     private $hashTypes = ['md5', 'sha256', 'sha512'];
     private $versionAppId = 'files_versions';
 
-    public function __construct($appName,
-                                IRequest $request,
-                                IRootFolder $rootFolder,
-                                IUserSession $userSession,
-                                HashMapper $mapper,
-                                ILogger $logger) {
+    public function __construct(
+        $appName,
+        IRequest $request,
+        IRootFolder $rootFolder,
+        IUserSession $userSession,
+        HashMapper $mapper,
+        LoggerInterface $logger
+    ) {
         parent::__construct($appName, $request);
         $this->rootFolder = $rootFolder;
         $this->userSession = $userSession;
@@ -41,7 +50,7 @@ public function __construct($appName,
     }
 
     private function isValidHash(string $hash) {
-        foreach($this->hashTypes as $hashType) {
+        foreach ($this->hashTypes as $hashType) {
             if ($hashType === $hash) {
                 return true;
             }
@@ -62,7 +71,7 @@ private function getMatchedVersion(string $path, string $revision) {
         return null;
     }
 
-    private function saveRecord(int $fileid, int $revision, string $hashType, string $hash) {
+    function saveRecord(int $fileid, int $revision, string $hashType, string $hash) {
         $entity = new Hash();
         $entity->setFileid($fileid);
         $entity->setRevision($revision);
@@ -94,7 +103,7 @@ public function checksum($hash, $path, $revision) {
         }
 
         $hashTypes = explode(',', $hash);
-        foreach($hashTypes as $hashType) {
+        foreach ($hashTypes as $hashType) {
             if (!$this->isValidHash($hashType)) {
                 $this->logger->error('query parameter hash is invalid.');
                 return new DataResponse(
@@ -139,16 +148,14 @@ public function checksum($hash, $path, $revision) {
             if ($revision === strval($latestRevision)) {
                 $this->logger->info('latest version matches');
             } else {
-                // check if version function is enabled
-                if (!\OCP\App::isEnabled($this->versionAppId)) {
+                if (!\OC::$server->getAppManager()->isEnabledForUser($this->versionAppId)) {
                     $this->logger->error('version function is not enabled');
                     return new DataResponse(
                         'version function is not enabled',
                         Http::STATUS_NOT_IMPLEMENTED
                     );
                 }
                 $this->logger->info($this->versionAppId . ' is enabled');
-
                 $version = $this->getMatchedVersion($path, $revision);
                 if (is_null($version)) {
                     $this->logger->error('specified revision is not found');
@@ -163,6 +170,7 @@ public function checksum($hash, $path, $revision) {
         }
 
         $entities = [];
+        $tasks = [];
         foreach ($hashTypes as $hashType) {
             $entity = $this->mapper->find($fileid, $targetRevision, $hashType);
             if (is_null($entity)) {
@@ -175,13 +183,48 @@ public function checksum($hash, $path, $revision) {
                     $view = new \OC\Files\View('/');
                     $info = $view->getLocalFile($targetFile);
                 }
-                $hash = hash_file($hashType, $info);
-                $this->logger->debug('hash: ' . $hash);
-                $entity = $this->saveRecord($fileid, $targetRevision, $hashType, $hash);
+                // check file size 20MB
+                if (fileSize($info) <= $this->minFileSizeToExcuteParallel || count($hashTypes) === 1) {
+                    $hash = hash_file($hashType, $info);
+                    $this->logger->debug('hash: ' . $hash);
+                    $entity = $this->saveRecord($fileid, $targetRevision, $hashType, $hash);
+                    array_push($entities, $entity);
+                } else {
+                    array_push(
+                        $tasks,
+                        new HashableTask(
+                            'hashCalculator',
+                            [
+                                "hashType" => $hashType,
+                                "fileid" => $fileid,
+                                "revision" => $targetRevision,
+                                "info" => $info
+                            ],
+                        ),
+                    );
+                }
+            } else {
+                array_push($entities, $entity);
             }
-            array_push($entities, $entity);
         }
-
+        if (!empty($tasks)) {
+            Amp\Loop::run(function () use (&$entities, $tasks) {
+                $pool = new DefaultPool;
+                $coroutines = [];
+                foreach ($tasks as $task) {
+                    $coroutines[] = Amp\call(function () use ($pool, $task) {
+                        $entity = yield $pool->enqueue($task);
+                        $this->mapper->insert($entity);
+                        return $entity;
+                    });
+                }
+                $completedTask = yield Amp\Promise\all($coroutines);
+                foreach ($completedTask as $task) {
+                    array_push($entities, $task);
+                }
+                return yield $pool->shutdown();
+            });
+        }
         $res = [];
         $hashes = [];
         foreach ($entities as $entity) {

lib/Db/Hash.php

@@ -4,6 +4,8 @@
 
 namespace OCA\ChecksumAPI\Db;
 
+require __DIR__ . '../../../../../lib/composer/autoload.php';
+
 use OCP\AppFramework\Db\Entity;
 
 class Hash extends Entity {

lib/Db/HashMapper.php

@@ -4,6 +4,8 @@
 
 namespace OCA\ChecksumAPI\Db;
 
+require __DIR__ . '../../../vendor/autoload.php';
+
 use OCP\AppFramework\Db\DoesNotExistException;
 use OCP\AppFramework\Db\QBMapper;
 use OCP\DB\QueryBuilder\IQueryBuilder;

lib/Jobs/HashableTask.php

@@ -0,0 +1,43 @@
+<?php
+
+declare(strict_types=1);
+
+namespace OCA\ChecksumAPI\Jobs;
+
+require __DIR__ . '../../../../../lib/composer/autoload.php';
+
+use Amp\Parallel\Worker\Environment;
+use Amp\Parallel\Worker\Task;
+use OCA\ChecksumAPI\Db\Hash;
+
+class HashableTask implements Task {
+    /**
+     * @var callable
+     */
+    private $function;
+
+    private $args;
+
+    public function __construct($function, $args) {
+        $this->function = $function;
+        $this->args = $args;
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    public function run(Environment $environment) {
+        $function = $this->function;
+        return $this->$function($this->args);
+    }
+
+    public function hashCalculator($params) {
+        $hash = hash_file($params["hashType"], $params["info"]);
+        $entity = new Hash();
+        $entity->setFileid($params["fileid"]);
+        $entity->setRevision($params["revision"]);
+        $entity->setType($params["hashType"]);
+        $entity->setHash($hash);
+        return $entity;
+    }
+}

lib/Migration/Version00001Date20201016095257.php

@@ -59,7 +59,7 @@ public function changeSchema(IOutput $output, Closure $schemaClosure, array $opt
 			]);
 			$table->addColumn('hash', 'string', [
 				'notnull' => true,
-				'length' => 64,
+				'length' => 128,
 				'default' => '',
 			]);
 			$table->setPrimaryKey(['id']);