diff --git a/classes/local/object_manipulator/candidates/candidates_factory.php b/classes/local/object_manipulator/candidates/candidates_factory.php index de72292f..8d757fdc 100644 --- a/classes/local/object_manipulator/candidates/candidates_factory.php +++ b/classes/local/object_manipulator/candidates/candidates_factory.php @@ -31,6 +31,7 @@ use tool_objectfs\local\object_manipulator\puller; use tool_objectfs\local\object_manipulator\pusher; use tool_objectfs\local\object_manipulator\recoverer; +use tool_objectfs\local\object_manipulator\orphaner; defined('MOODLE_INTERNAL') || die(); @@ -43,6 +44,7 @@ class candidates_factory { puller::class => puller_candidates::class, pusher::class => pusher_candidates::class, recoverer::class => recoverer_candidates::class, + orphaner::class => orphaner_candidates::class, ]; /** diff --git a/classes/local/object_manipulator/candidates/orphaner_candidates.php b/classes/local/object_manipulator/candidates/orphaner_candidates.php new file mode 100644 index 00000000..67fb364c --- /dev/null +++ b/classes/local/object_manipulator/candidates/orphaner_candidates.php @@ -0,0 +1,55 @@ +. + +/** + * Class orphaner_candidates + * @package tool_objectfs + * @author Nathan Mares + * @copyright Catalyst IT + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +namespace tool_objectfs\local\object_manipulator\candidates; + +defined('MOODLE_INTERNAL') || die(); + +class orphaner_candidates extends manipulator_candidates_base { + + /** @var string $queryname */ + protected $queryname = 'get_orphan_candidates'; + + /** + * @inheritDoc + * @return string + */ + public function get_candidates_sql() { + return 'SELECT o.id, o.contenthash, o.location + FROM {tool_objectfs_objects} o + LEFT JOIN {files} f ON o.contenthash = f.contenthash + WHERE f.id is null + AND o.location != :location'; + } + + /** + * @inheritDoc + * @return array + */ + public function get_candidates_sql_params() { + return [ + 'location' => OBJECT_LOCATION_ORPHANED + ]; + } +} diff --git a/classes/local/object_manipulator/candidates/pusher_candidates.php b/classes/local/object_manipulator/candidates/pusher_candidates.php index 1fd1938f..52f7b31c 100644 --- a/classes/local/object_manipulator/candidates/pusher_candidates.php +++ b/classes/local/object_manipulator/candidates/pusher_candidates.php @@ -45,7 +45,7 @@ public function get_candidates_sql() { JOIN {tool_objectfs_objects} o ON f.contenthash = o.contenthash WHERE f.filesize > :threshold AND f.filesize < :maximum_file_size - AND f.timecreated <= :maxcreatedtimstamp + AND f.timecreated <= :maxcreatedtimestamp AND o.location = :object_location GROUP BY f.contenthash, o.location'; } @@ -57,7 +57,7 @@ public function get_candidates_sql() { public function get_candidates_sql_params() { $filesystem = new $this->config->filesystem; return [ - 'maxcreatedtimstamp' => time() - $this->config->minimumage, + 'maxcreatedtimestamp' => time() - $this->config->minimumage, 'threshold' => $this->config->sizethreshold, 'maximum_file_size' => $filesystem->get_maximum_upload_filesize(), 'object_location' => OBJECT_LOCATION_LOCAL, diff --git a/classes/local/object_manipulator/manipulator.php b/classes/local/object_manipulator/manipulator.php index cf1cc41f..a0cadaf8 100644 --- a/classes/local/object_manipulator/manipulator.php +++ b/classes/local/object_manipulator/manipulator.php @@ -119,12 +119,19 @@ public function execute(array $objectrecords) { } /** + * Given an object record, the class implementing this will be able to manipulate + * the object, and return the new location of the object. + * @see examples in lib.php (OBJECT_LOCATION_*) + * * @param stdClass $objectrecord - * @return int + * @return int OBJECT_LOCATION_* */ abstract public function manipulate_object(stdClass $objectrecord); /** + * Returns whether or not the particular manipulator will manipulate the + * object when execute is called. + * * @return bool */ protected function manipulator_can_execute() { diff --git a/classes/local/object_manipulator/manipulator_builder.php b/classes/local/object_manipulator/manipulator_builder.php index 31f77c6b..66e5ba41 100644 --- a/classes/local/object_manipulator/manipulator_builder.php +++ b/classes/local/object_manipulator/manipulator_builder.php @@ -45,6 +45,7 @@ class manipulator_builder { pusher::class, recoverer::class, checker::class, + orphaner::class ]; /** @var string $manipulatorclass */ diff --git a/classes/local/object_manipulator/orphaner.php b/classes/local/object_manipulator/orphaner.php new file mode 100644 index 00000000..66dc776c --- /dev/null +++ b/classes/local/object_manipulator/orphaner.php @@ -0,0 +1,46 @@ +. + +/** + * Orphans {tool_objectfs_objects} records for files that have been + * deleted from the core {files} table. + * + * @package tool_objectfs + * @author Nathan Mares + * @author Kevin Pham + * @copyright Catalyst IT + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +namespace tool_objectfs\local\object_manipulator; + +use stdClass; + +defined('MOODLE_INTERNAL') || die(); + +class orphaner extends manipulator { + + /** + * Updates the location of {tool_objectfs_objects} records for files that + * have been deleted from the core {files} table. + * + * @param \stdClass $objectrecord + * @return int + */ + public function manipulate_object(stdClass $objectrecord): int { + return OBJECT_LOCATION_ORPHANED; + } +} diff --git a/classes/local/report/location_report_builder.php b/classes/local/report/location_report_builder.php index 4c8cef64..3ff2bc46 100644 --- a/classes/local/report/location_report_builder.php +++ b/classes/local/report/location_report_builder.php @@ -43,6 +43,7 @@ public function build_report($reportid) { OBJECT_LOCATION_LOCAL, OBJECT_LOCATION_DUPLICATED, OBJECT_LOCATION_EXTERNAL, + OBJECT_LOCATION_ORPHANED, OBJECT_LOCATION_ERROR ]; @@ -65,7 +66,20 @@ public function build_report($reportid) { HAVING o.location = ?' . $localsql .') AS sub WHERE sub.filesize > 0'; - $result = $DB->get_record_sql($sql, array($location)); + if ($location !== OBJECT_LOCATION_ORPHANED) { + // Process the query normally. + $result = $DB->get_record_sql($sql, array($location)); + } else if ($location === OBJECT_LOCATION_ORPHANED) { + // Start the query from objectfs, for ORPHANED objects, they are not located in the files table. + $sql = 'SELECT COALESCE(count(sub.contenthash) ,0) AS objectcount + FROM (SELECT o.contenthash + FROM {tool_objectfs_objects} o + LEFT JOIN {files} f on f.contenthash = o.contenthash + GROUP BY o.contenthash, f.filesize, o.location + HAVING o.location = ?' . $localsql .') AS sub'; + $result = $DB->get_record_sql($sql, array($location)); + $result->objectsum = 0; + } $result->datakey = $location; diff --git a/classes/local/report/object_location_history_table.php b/classes/local/report/object_location_history_table.php index 07fc80f7..2a2a96a3 100644 --- a/classes/local/report/object_location_history_table.php +++ b/classes/local/report/object_location_history_table.php @@ -49,6 +49,8 @@ public function __construct() { 'local_size' => get_string('object_status:location:localsize', 'tool_objectfs'), 'duplicated_count' => get_string('object_status:location:duplicatedcount', 'tool_objectfs'), 'duplicated_size' => get_string('object_status:location:duplicatedsize', 'tool_objectfs'), + 'orphaned_count' => get_string('object_status:location:orphanedcount', 'tool_objectfs'), + 'orphaned_size' => get_string('object_status:location:orphanedsize', 'tool_objectfs'), 'external_count' => get_string('object_status:location:externalcount', 'tool_objectfs'), 'external_size' => get_string('object_status:location:externalsize', 'tool_objectfs'), 'missing_count' => get_string('object_status:location:missingcount', 'tool_objectfs'), @@ -84,40 +86,59 @@ public function query_db($pagesize, $useinitialsbar = true) { $rawrecords = $DB->get_records('tool_objectfs_report_data', $conditions, 'reportid', $fields); $reports = objectfs_report::get_report_ids(); + // Used to fallback to when the expected record is not there. + // NOTE: This avoids the need to null coalesce on a non-existing count/size. + $emptyrecord = (object)[ + 'count' => 0, + 'size' => 0 + ]; foreach ($reports as $id => $timecreated) { - $localcount = $rawrecords[$id.OBJECT_LOCATION_LOCAL]->count + $rawrecords[$id.OBJECT_LOCATION_DUPLICATED]->count; - $deltacount = abs($rawrecords[$id.'filedir']->count - $localcount); - $localsize = $rawrecords[$id.OBJECT_LOCATION_LOCAL]->size + $rawrecords[$id.OBJECT_LOCATION_DUPLICATED]->size; - $deltasize = abs($rawrecords[$id.'filedir']->size - $localsize); + // Initialises the records to be used, and fallback to an empty one if not found. + $localrecord = $rawrecords[$id.OBJECT_LOCATION_LOCAL] ?? $emptyrecord; + $duplicatedrecord = $rawrecords[$id.OBJECT_LOCATION_DUPLICATED] ?? $emptyrecord; + $orphanedrecord = $rawrecords[$id.OBJECT_LOCATION_ORPHANED] ?? $emptyrecord; + $externalrecord = $rawrecords[$id.OBJECT_LOCATION_EXTERNAL] ?? $emptyrecord; + $errorrecord = $rawrecords[$id.OBJECT_LOCATION_ERROR] ?? $emptyrecord; + $filedir = $rawrecords[$id.'filedir'] ?? $emptyrecord; + $total = $rawrecords[$id.'total'] ?? $emptyrecord; + + $localcount = $localrecord->count + $duplicatedrecord->count; + $deltacount = abs($filedir->count - $localcount); + $localsize = $localrecord->size + $duplicatedrecord->size; + $deltasize = abs($filedir->size - $localsize); $row['date'] = userdate($timecreated, get_string('strftimedaydatetime')); if ($this->is_downloading() && in_array($this->download, ['csv', 'excel', 'json', 'ods'])) { - $row['local_count'] = $rawrecords[$id.OBJECT_LOCATION_LOCAL]->count; - $row['local_size'] = $rawrecords[$id.OBJECT_LOCATION_LOCAL]->size; - $row['duplicated_count'] = $rawrecords[$id.OBJECT_LOCATION_DUPLICATED]->count; - $row['duplicated_size'] = $rawrecords[$id.OBJECT_LOCATION_DUPLICATED]->size; - $row['external_count'] = $rawrecords[$id.OBJECT_LOCATION_EXTERNAL]->count; - $row['external_size'] = $rawrecords[$id.OBJECT_LOCATION_EXTERNAL]->size; - $row['missing_count'] = $rawrecords[$id.OBJECT_LOCATION_ERROR]->count; - $row['missing_size'] = $rawrecords[$id.OBJECT_LOCATION_ERROR]->size; - $row['total_count'] = $rawrecords[$id.'total']->count; - $row['total_size'] = $rawrecords[$id.'total']->size; - $row['filedir_count'] = $rawrecords[$id.'filedir']->count; - $row['filedir_size'] = $rawrecords[$id.'filedir']->size; + $row['local_count'] = $localrecord->count; + $row['local_size'] = $localrecord->size; + $row['duplicated_count'] = $duplicatedrecord->count; + $row['duplicated_size'] = $duplicatedrecord->size; + $row['orphaned_count'] = $orphanedrecord->count; + $row['orphaned_size'] = get_string('object_status:location:orphanedsizeunknown', 'tool_objectfs'); + $row['external_count'] = $externalrecord->count; + $row['external_size'] = $externalrecord->size; + $row['missing_count'] = $errorrecord->count; + $row['missing_size'] = $errorrecord->size; + $row['total_count'] = $total->count; + $row['total_size'] = $total->size; + $row['filedir_count'] = $filedir->count; + $row['filedir_size'] = $filedir->size; $row['delta_count'] = $deltacount; $row['delta_size'] = $deltasize; } else { - $row['local_count'] = number_format($rawrecords[$id.OBJECT_LOCATION_LOCAL]->count); - $row['local_size'] = display_size($rawrecords[$id.OBJECT_LOCATION_LOCAL]->size); - $row['duplicated_count'] = number_format($rawrecords[$id.OBJECT_LOCATION_DUPLICATED]->count); - $row['duplicated_size'] = display_size($rawrecords[$id.OBJECT_LOCATION_DUPLICATED]->size); - $row['external_count'] = number_format($rawrecords[$id.OBJECT_LOCATION_EXTERNAL]->count); - $row['external_size'] = display_size($rawrecords[$id.OBJECT_LOCATION_EXTERNAL]->size); - $row['missing_count'] = number_format($rawrecords[$id.OBJECT_LOCATION_ERROR]->count); - $row['missing_size'] = display_size($rawrecords[$id.OBJECT_LOCATION_ERROR]->size); - $row['total_count'] = number_format($rawrecords[$id.'total']->count); - $row['total_size'] = display_size($rawrecords[$id.'total']->size); - $row['filedir_count'] = number_format($rawrecords[$id.'filedir']->count); - $row['filedir_size'] = display_size($rawrecords[$id.'filedir']->size); + $row['local_count'] = number_format($localrecord->count); + $row['local_size'] = display_size($localrecord->size); + $row['duplicated_count'] = number_format($duplicatedrecord->count); + $row['duplicated_size'] = display_size($duplicatedrecord->size); + $row['orphaned_count'] = number_format($orphanedrecord->count); + $row['orphaned_size'] = get_string('object_status:location:orphanedsizeunknown', 'tool_objectfs'); + $row['external_count'] = number_format($externalrecord->count); + $row['external_size'] = display_size($externalrecord->size); + $row['missing_count'] = number_format($errorrecord->count); + $row['missing_size'] = display_size($errorrecord->size); + $row['total_count'] = number_format($total->count); + $row['total_size'] = display_size($total->size); + $row['filedir_count'] = number_format($filedir->count); + $row['filedir_size'] = display_size($filedir->size); $row['delta_count'] = number_format($deltacount); $row['delta_size'] = display_size($deltasize); } diff --git a/classes/local/report/object_status_history_table.php b/classes/local/report/object_status_history_table.php index 81f1dc2d..be843e45 100644 --- a/classes/local/report/object_status_history_table.php +++ b/classes/local/report/object_status_history_table.php @@ -154,6 +154,10 @@ public function col_count(\stdClass $row) { * @return string */ public function col_size(\stdClass $row) { + // For orphaned entries, the filesize is N/A or Unknown. Note: non-strict check as the heading is a string. + if ($row->heading == OBJECT_LOCATION_ORPHANED) { + return get_string('object_status:location:orphanedsizeunknown', 'tool_objectfs'); + } return $this->add_barchart($row->size, $this->maxsize, 'size'); } @@ -227,6 +231,7 @@ public function get_file_location_string($filelocation) { OBJECT_LOCATION_LOCAL => 'object_status:location:local', OBJECT_LOCATION_DUPLICATED => 'object_status:location:duplicated', OBJECT_LOCATION_EXTERNAL => 'object_status:location:external', + OBJECT_LOCATION_ORPHANED => 'object_status:location:orphaned', ]; if (isset($locationstringmap[$filelocation])) { return get_string($locationstringmap[$filelocation], 'tool_objectfs'); diff --git a/classes/log/aggregate_logger.php b/classes/log/aggregate_logger.php index ed9fa190..bdd23d78 100644 --- a/classes/log/aggregate_logger.php +++ b/classes/log/aggregate_logger.php @@ -106,6 +106,8 @@ public function location_to_string($location) { return 'duplicated'; case OBJECT_LOCATION_EXTERNAL: return 'remote'; + case OBJECT_LOCATION_ORPHANED: + return 'orphaned'; default: return $location; } diff --git a/classes/task/delete_orphaned_object_metadata.php b/classes/task/delete_orphaned_object_metadata.php new file mode 100644 index 00000000..259a2688 --- /dev/null +++ b/classes/task/delete_orphaned_object_metadata.php @@ -0,0 +1,61 @@ +. + +/** + * Task that checks for old orphaned objects, and removes their metadata + * (record) as it is no longer useful/relevant. + * + * @package tool_objectfs + * @author Kevin Pham + * @copyright Catalyst IT + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +namespace tool_objectfs\task; + +defined('MOODLE_INTERNAL') || die(); + +require_once(__DIR__ . '/../../lib.php'); + +class delete_orphaned_object_metadata extends task { + + /** @var string $stringname */ + protected $stringname = 'delete_orphaned_object_metadata_task'; + + /** + * Execute task + */ + public function execute() { + global $DB; + + $wheresql = 'location = :location and timeduplicated < :ageforremoval'; + $ageforremoval = $this->config->maxorphanedage; + if (empty($ageforremoval)) { + mtrace('Skipping deletion of orphaned object metadata as maxorphanedage is set to an empty value.'); + return; + } + + $params = [ + 'location' => OBJECT_LOCATION_ORPHANED, + 'ageforremoval' => time() - $ageforremoval + ]; + $count = $DB->count_records_select('tool_objectfs_objects', $wheresql, $params); + if (!empty($count)) { + mtrace("Deleting $count records with orphaned metadata (orphaned tool_objectfs_objects)"); + $DB->delete_records_select('tool_objectfs_objects', $wheresql, $params); + } + } +} diff --git a/classes/task/orphan_objects.php b/classes/task/orphan_objects.php new file mode 100644 index 00000000..97d9efdc --- /dev/null +++ b/classes/task/orphan_objects.php @@ -0,0 +1,41 @@ +. + +/** + * Task that orphans {tool_objectfs_object} records for deleted + * {files} records. + * + * @package tool_objectfs + * @author Nathan Mares + * @copyright Catalyst IT + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +namespace tool_objectfs\task; + +use tool_objectfs\local\object_manipulator\orphaner; + +defined('MOODLE_INTERNAL') || die(); + + +class orphan_objects extends task { + + /** @var string $manipulator */ + protected $manipulator = orphaner::class; + + /** @var string $stringname */ + protected $stringname = 'orphan_objects_task'; +} diff --git a/db/tasks.php b/db/tasks.php index e8c2a183..5125d734 100644 --- a/db/tasks.php +++ b/db/tasks.php @@ -53,6 +53,24 @@ 'dayofweek' => '*', 'month' => '*' ), + array( + 'classname' => 'tool_objectfs\task\orphan_objects', + 'blocking' => 0, + 'minute' => 'R', + 'hour' => 'R', + 'day' => '*', + 'dayofweek' => '*', + 'month' => '*' + ), + array( + 'classname' => 'tool_objectfs\task\delete_orphaned_object_metadata', + 'blocking' => 0, + 'minute' => 'R', + 'hour' => 'R', + 'day' => '*', + 'dayofweek' => '*', + 'month' => '*' + ), array( 'classname' => 'tool_objectfs\task\delete_local_empty_directories', 'blocking' => 0, diff --git a/lang/en/tool_objectfs.php b/lang/en/tool_objectfs.php index 7d46cbc6..7227d277 100644 --- a/lang/en/tool_objectfs.php +++ b/lang/en/tool_objectfs.php @@ -28,11 +28,13 @@ $string['pluginsettings'] = 'Plugin Settings'; $string['privacy:metadata'] = 'The tool objectfs plugin does not store any personal data.'; $string['push_objects_to_storage_task'] = 'Object file system upload task'; +$string['delete_orphaned_object_metadata_task'] = 'Object file system delete orphaned metadata task'; $string['delete_local_objects_task'] = 'Object file system delete local objects task'; $string['delete_local_empty_directories_task'] = 'Object file system delete local empty directories task'; $string['pull_objects_from_storage_task'] = 'Object file system download objects task'; $string['recover_error_objects_task'] = 'Object error recovery task'; $string['check_objects_location_task'] = 'Object file system check objects location task'; +$string['orphan_objects_task'] = 'Object file system orphan objects task'; $string['generate_status_report_task'] = 'Object status report generator task'; $string['not_enabled'] = 'The object file system background tasks are not enabled. No objects will move location until you do.'; @@ -55,6 +57,7 @@ $string['object_status:location:error'] = 'Missing from filedir and external storage (view files)'; $string['object_status:location:duplicated'] = 'Duplicated in filedir and external storage'; $string['object_status:location:local'] = 'Marked as only in filedir'; +$string['object_status:location:orphaned'] = 'Marked as orphaned (not in the {files} table)'; $string['object_status:location:external'] = 'Only in external storage'; $string['object_status:location:unknown'] = 'Unknown object location'; $string['object_status:location:total'] = 'Total'; @@ -62,6 +65,9 @@ $string['object_status:location:localsize'] = 'Local (size)'; $string['object_status:location:duplicatedcount'] = 'Duplicated (count)'; $string['object_status:location:duplicatedsize'] = 'Duplicated (size)'; +$string['object_status:location:orphanedcount'] = 'Orphaned (count)'; +$string['object_status:location:orphanedsize'] = 'Orphaned (size)'; +$string['object_status:location:orphanedsizeunknown'] = 'Unknown'; $string['object_status:location:externalcount'] = 'External (count)'; $string['object_status:location:externalsize'] = 'External (size)'; $string['object_status:location:missingcount'] = 'Error (count)'; @@ -85,7 +91,7 @@ $string['rangerequestfailed'] = 'URL: {$a->url}
HTTP code: {$a->httpcode}
Details: {$a->details}'; $string['settings'] = 'Settings'; -$string['settings:enabletasks'] = 'Enable transfer tasks'; +$string['settings:enabletasks'] = 'Enable background transfer tasks'; $string['settings:enabletasks_help'] = 'Enable or disable the object file system tasks which move files between the filedir and external object storage.'; $string['settings:enablelogging'] = 'Enable real time logging'; $string['settings:enablelogging_help'] = 'Enable or disable file system logging. Will output diagnostic information to the php error log. '; @@ -161,6 +167,8 @@ $string['settings:sizethreshold_help'] = 'Minimum size threshold for transfering objects to external object storage. If objects are over this size they will be transfered.'; $string['settings:batchsize'] = 'Number files in one batch'; $string['settings:batchsize_help'] = 'Number of files to be transferred in one cron run'; +$string['settings:maxorphanedage'] = 'Max orphaned object age'; +$string['settings:maxorphanedage_help'] = 'If set to zero, this will not delete old orphaned metadata for objects. Otherwise, it will remove these records as they are no longer relevant. An orphaned object is one where the metadata exists on the {tool_objectfs_objects} table but referenced file no longer exists.'; $string['settings:minimumage'] = 'Minimum age'; $string['settings:minimumage_help'] = 'Minimum age that a object must exist on the local filedir before it will be considered for transfer.'; $string['settings:deleteexternal'] = 'Delete external objects'; diff --git a/lib.php b/lib.php index 08dbb12a..0c370c46 100644 --- a/lib.php +++ b/lib.php @@ -28,9 +28,37 @@ defined('MOODLE_INTERNAL') || die; define('OBJECTFS_PLUGIN_NAME', 'tool_objectfs'); + +/** + * Location enum of the object + * ORPHANED is when the {objectfs_objects} table contains a record linking to a + * moodle {files} record which is no longer present. + */ +define('OBJECT_LOCATION_ORPHANED', -2); + +/** + * Location enum of the object + * ERROR is when the file is missing when it is expected to be there. + * @see tests/object_file_system_test.php for examples. + */ define('OBJECT_LOCATION_ERROR', -1); + +/** + * Location enum of the object + * LOCAL is when the object exists locally only. + */ define('OBJECT_LOCATION_LOCAL', 0); + +/** + * Location enum of the object + * DUPLICATED is when the object exists both locally, and remotely. + */ define('OBJECT_LOCATION_DUPLICATED', 1); + +/** + * Location enum of the object + * EXTERNAL is when when the object lives remotely only. + */ define('OBJECT_LOCATION_EXTERNAL', 2); define('OBJECTFS_REPORT_OBJECT_LOCATION', 0); diff --git a/settings.php b/settings.php index 2163e4e0..67f71805 100644 --- a/settings.php +++ b/settings.php @@ -93,6 +93,10 @@ new lang_string('settings:deleteexternal', 'tool_objectfs'), new lang_string('settings:deleteexternal_help', 'tool_objectfs'), TOOL_OBJECTFS_DELETE_EXTERNAL_NO, $options)); + $settings->add(new admin_setting_configduration('tool_objectfs/maxorphanedage', + new lang_string('settings:maxorphanedage', 'tool_objectfs'), + new lang_string('settings:maxorphanedage_help', 'tool_objectfs'), 0, DAYSECS)); + $settings->add(new admin_setting_configcheckbox('tool_objectfs/enablelogging', new lang_string('settings:enablelogging', 'tool_objectfs'), '', '')); diff --git a/tests/object_status_test.php b/tests/object_status_test.php index b3c6292e..de4c5da9 100644 --- a/tests/object_status_test.php +++ b/tests/object_status_test.php @@ -85,8 +85,8 @@ public function test_object_status_history_table_location() { $table->define_baseurl($CFG->wwwroot); $table->setup(); $table->query_db(100, false); - // 7 is expected number of rows for location section of Object status report. - $this->assertEquals(7, count($table->rawdata)); + // 8 is expected number of rows for location section of Object status report. + $this->assertEquals(8, count($table->rawdata)); } /** diff --git a/tests/orphaner_test.php b/tests/orphaner_test.php new file mode 100644 index 00000000..e51b6224 --- /dev/null +++ b/tests/orphaner_test.php @@ -0,0 +1,116 @@ +. + +namespace tool_objectfs\tests; + +defined('MOODLE_INTERNAL') || die(); + +use tool_objectfs\local\manager; +use tool_objectfs\local\object_manipulator\candidates\candidates_finder; +use tool_objectfs\local\object_manipulator\orphaner; + +require_once(__DIR__ . '/classes/test_client.php'); +require_once(__DIR__ . '/tool_objectfs_testcase.php'); + +class orphaner_testcase extends tool_objectfs_testcase { + + /** @var string $manipulator */ + protected $manipulator = orphaner::class; + + protected function setUp(): void { + parent::setUp(); + $config = manager::get_objectfs_config(); + $config->sizethreshold = 0; + $config->minimumage = 0; + manager::set_objectfs_config($config); + $this->logger = new \tool_objectfs\log\aggregate_logger(); + $this->orphaner = new orphaner($this->filesystem, $config, $this->logger); + ob_start(); + } + + protected function tearDown(): void { + ob_end_clean(); + } + + protected function set_orphaner_config($key, $value) { + $config = manager::get_objectfs_config(); + $config->$key = $value; + manager::set_objectfs_config($config); + $this->orphaner = new orphaner($this->filesystem, $config, $this->logger); + } + + public function test_orphaner_can_orphan_files() { + global $DB; + $this->orphaner->execute([ + $object1 = $this->create_local_object(), + $object2 = $this->create_duplicated_object(), + $object3 = $this->create_remote_object(), + ]); + $location1 = $DB->get_field('tool_objectfs_objects', 'location', ['contenthash' => $object1->contenthash]); + $location2 = $DB->get_field('tool_objectfs_objects', 'location', ['contenthash' => $object2->contenthash]); + $location3 = $DB->get_field('tool_objectfs_objects', 'location', ['contenthash' => $object3->contenthash]); + $this->assertEquals(OBJECT_LOCATION_ORPHANED, $location1); + $this->assertEquals(OBJECT_LOCATION_ORPHANED, $location2); + $this->assertEquals(OBJECT_LOCATION_ORPHANED, $location3); + } + + public function test_orphaner_finds_correct_candidates() { + global $DB; + + // Initialise the candidate finder. + $config = manager::get_objectfs_config(); + $config->filesystem = get_class($this->filesystem); + $finder = new candidates_finder($this->manipulator, $config); + $objects = $finder->get(); + $this->assertCount(0, $objects); // No candidates. + + // Create an object. + $object = $this->create_local_object(); + + // Still no candidates - object created but nothing is missing from {files} table. + $objects = $finder->get(); + $this->assertCount(0, $objects); + + // Update that object to have a different hash, to mock a non-existent + // mdl_file with an objectfs record (orphaned). + $DB->set_field('files', 'contenthash', 'different', array('contenthash' => $object->contenthash)); + + // Expect one candidate - no matching contenthash in {files}. + $objects = $finder->get(); + $this->assertCount(1, $objects); + + // Ensure it ignores orphaned records during the find. + $DB->set_field('tool_objectfs_objects', 'location', OBJECT_LOCATION_ORPHANED, ['contenthash' => $object->contenthash]); + $objects = $finder->get(); + $this->assertCount(0, $objects); // No candidates - only candidate has been orphaned. + } + + public function test_orphaner_correctly_orphans_provided_files() { + global $DB; + $this->orphaner->execute([ + $object1 = $this->create_local_object(), + $object2 = $this->create_duplicated_object(), + $object3 = $this->create_remote_object(), + ]); + $location1 = $DB->get_field('tool_objectfs_objects', 'location', ['contenthash' => $object1->contenthash]); + $location2 = $DB->get_field('tool_objectfs_objects', 'location', ['contenthash' => $object2->contenthash]); + $location3 = $DB->get_field('tool_objectfs_objects', 'location', ['contenthash' => $object3->contenthash]); + $this->assertEquals(OBJECT_LOCATION_ORPHANED, $location1); + $this->assertEquals(OBJECT_LOCATION_ORPHANED, $location2); + $this->assertEquals(OBJECT_LOCATION_ORPHANED, $location3); + } + +} diff --git a/tests/tasks_test.php b/tests/tasks_test.php index 011051b4..24547d58 100644 --- a/tests/tasks_test.php +++ b/tests/tasks_test.php @@ -63,6 +63,7 @@ public function test_run_scheduled_tasks() { 'push_objects_to_storage', 'recover_error_objects', 'check_objects_location', + 'delete_orphaned_object_metadata', ]; foreach ($scheduledtasknames as $taskname) {