Skip to content

Commit

Permalink
Merge pull request #451 from keevan/440-add-archiver-task-to-handle-o…
Browse files Browse the repository at this point in the history
…rphaned-objects

Add archiver task to handle orphaned objects
  • Loading branch information
keevan authored Dec 23, 2021
2 parents 329e8ee + 9afe7bf commit caa0d12
Show file tree
Hide file tree
Showing 19 changed files with 465 additions and 35 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
use tool_objectfs\local\object_manipulator\puller;
use tool_objectfs\local\object_manipulator\pusher;
use tool_objectfs\local\object_manipulator\recoverer;
use tool_objectfs\local\object_manipulator\orphaner;

defined('MOODLE_INTERNAL') || die();

Expand All @@ -43,6 +44,7 @@ class candidates_factory {
puller::class => puller_candidates::class,
pusher::class => pusher_candidates::class,
recoverer::class => recoverer_candidates::class,
orphaner::class => orphaner_candidates::class,
];

/**
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
<?php
// This file is part of Moodle - http://moodle.org/
//
// Moodle is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// Moodle is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with Moodle. If not, see <http://www.gnu.org/licenses/>.

/**
* Class orphaner_candidates
* @package tool_objectfs
* @author Nathan Mares <[email protected]>
* @copyright Catalyst IT
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/

namespace tool_objectfs\local\object_manipulator\candidates;

defined('MOODLE_INTERNAL') || die();

class orphaner_candidates extends manipulator_candidates_base {

/** @var string $queryname */
protected $queryname = 'get_orphan_candidates';

/**
* @inheritDoc
* @return string
*/
public function get_candidates_sql() {
return 'SELECT o.id, o.contenthash, o.location
FROM {tool_objectfs_objects} o
LEFT JOIN {files} f ON o.contenthash = f.contenthash
WHERE f.id is null
AND o.location != :location';
}

/**
* @inheritDoc
* @return array
*/
public function get_candidates_sql_params() {
return [
'location' => OBJECT_LOCATION_ORPHANED
];
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ public function get_candidates_sql() {
JOIN {tool_objectfs_objects} o ON f.contenthash = o.contenthash
WHERE f.filesize > :threshold
AND f.filesize < :maximum_file_size
AND f.timecreated <= :maxcreatedtimstamp
AND f.timecreated <= :maxcreatedtimestamp
AND o.location = :object_location
GROUP BY f.contenthash, o.location';
}
Expand All @@ -57,7 +57,7 @@ public function get_candidates_sql() {
public function get_candidates_sql_params() {
$filesystem = new $this->config->filesystem;
return [
'maxcreatedtimstamp' => time() - $this->config->minimumage,
'maxcreatedtimestamp' => time() - $this->config->minimumage,
'threshold' => $this->config->sizethreshold,
'maximum_file_size' => $filesystem->get_maximum_upload_filesize(),
'object_location' => OBJECT_LOCATION_LOCAL,
Expand Down
9 changes: 8 additions & 1 deletion classes/local/object_manipulator/manipulator.php
Original file line number Diff line number Diff line change
Expand Up @@ -119,12 +119,19 @@ public function execute(array $objectrecords) {
}

/**
* Given an object record, the class implementing this will be able to manipulate
* the object, and return the new location of the object.
* @see examples in lib.php (OBJECT_LOCATION_*)
*
* @param stdClass $objectrecord
* @return int
* @return int OBJECT_LOCATION_*
*/
abstract public function manipulate_object(stdClass $objectrecord);

/**
* Returns whether or not the particular manipulator will manipulate the
* object when execute is called.
*
* @return bool
*/
protected function manipulator_can_execute() {
Expand Down
1 change: 1 addition & 0 deletions classes/local/object_manipulator/manipulator_builder.php
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ class manipulator_builder {
pusher::class,
recoverer::class,
checker::class,
orphaner::class
];

/** @var string $manipulatorclass */
Expand Down
46 changes: 46 additions & 0 deletions classes/local/object_manipulator/orphaner.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
<?php
// This file is part of Moodle - http://moodle.org/
//
// Moodle is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// Moodle is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with Moodle. If not, see <http://www.gnu.org/licenses/>.

/**
* Orphans {tool_objectfs_objects} records for files that have been
* deleted from the core {files} table.
*
* @package tool_objectfs
* @author Nathan Mares <[email protected]>
* @author Kevin Pham <[email protected]>
* @copyright Catalyst IT
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/

namespace tool_objectfs\local\object_manipulator;

use stdClass;

defined('MOODLE_INTERNAL') || die();

class orphaner extends manipulator {

/**
* Updates the location of {tool_objectfs_objects} records for files that
* have been deleted from the core {files} table.
*
* @param \stdClass $objectrecord
* @return int
*/
public function manipulate_object(stdClass $objectrecord): int {
return OBJECT_LOCATION_ORPHANED;
}
}
16 changes: 15 additions & 1 deletion classes/local/report/location_report_builder.php
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ public function build_report($reportid) {
OBJECT_LOCATION_LOCAL,
OBJECT_LOCATION_DUPLICATED,
OBJECT_LOCATION_EXTERNAL,
OBJECT_LOCATION_ORPHANED,
OBJECT_LOCATION_ERROR
];

Expand All @@ -65,7 +66,20 @@ public function build_report($reportid) {
HAVING o.location = ?' . $localsql .') AS sub
WHERE sub.filesize > 0';

$result = $DB->get_record_sql($sql, array($location));
if ($location !== OBJECT_LOCATION_ORPHANED) {
// Process the query normally.
$result = $DB->get_record_sql($sql, array($location));
} else if ($location === OBJECT_LOCATION_ORPHANED) {
// Start the query from objectfs, for ORPHANED objects, they are not located in the files table.
$sql = 'SELECT COALESCE(count(sub.contenthash) ,0) AS objectcount
FROM (SELECT o.contenthash
FROM {tool_objectfs_objects} o
LEFT JOIN {files} f on f.contenthash = o.contenthash
GROUP BY o.contenthash, f.filesize, o.location
HAVING o.location = ?' . $localsql .') AS sub';
$result = $DB->get_record_sql($sql, array($location));
$result->objectsum = 0;
}

$result->datakey = $location;

Expand Down
77 changes: 49 additions & 28 deletions classes/local/report/object_location_history_table.php
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,8 @@ public function __construct() {
'local_size' => get_string('object_status:location:localsize', 'tool_objectfs'),
'duplicated_count' => get_string('object_status:location:duplicatedcount', 'tool_objectfs'),
'duplicated_size' => get_string('object_status:location:duplicatedsize', 'tool_objectfs'),
'orphaned_count' => get_string('object_status:location:orphanedcount', 'tool_objectfs'),
'orphaned_size' => get_string('object_status:location:orphanedsize', 'tool_objectfs'),
'external_count' => get_string('object_status:location:externalcount', 'tool_objectfs'),
'external_size' => get_string('object_status:location:externalsize', 'tool_objectfs'),
'missing_count' => get_string('object_status:location:missingcount', 'tool_objectfs'),
Expand Down Expand Up @@ -84,40 +86,59 @@ public function query_db($pagesize, $useinitialsbar = true) {
$rawrecords = $DB->get_records('tool_objectfs_report_data', $conditions, 'reportid', $fields);
$reports = objectfs_report::get_report_ids();

// Used to fallback to when the expected record is not there.
// NOTE: This avoids the need to null coalesce on a non-existing count/size.
$emptyrecord = (object)[
'count' => 0,
'size' => 0
];
foreach ($reports as $id => $timecreated) {
$localcount = $rawrecords[$id.OBJECT_LOCATION_LOCAL]->count + $rawrecords[$id.OBJECT_LOCATION_DUPLICATED]->count;
$deltacount = abs($rawrecords[$id.'filedir']->count - $localcount);
$localsize = $rawrecords[$id.OBJECT_LOCATION_LOCAL]->size + $rawrecords[$id.OBJECT_LOCATION_DUPLICATED]->size;
$deltasize = abs($rawrecords[$id.'filedir']->size - $localsize);
// Initialises the records to be used, and fallback to an empty one if not found.
$localrecord = $rawrecords[$id.OBJECT_LOCATION_LOCAL] ?? $emptyrecord;
$duplicatedrecord = $rawrecords[$id.OBJECT_LOCATION_DUPLICATED] ?? $emptyrecord;
$orphanedrecord = $rawrecords[$id.OBJECT_LOCATION_ORPHANED] ?? $emptyrecord;
$externalrecord = $rawrecords[$id.OBJECT_LOCATION_EXTERNAL] ?? $emptyrecord;
$errorrecord = $rawrecords[$id.OBJECT_LOCATION_ERROR] ?? $emptyrecord;
$filedir = $rawrecords[$id.'filedir'] ?? $emptyrecord;
$total = $rawrecords[$id.'total'] ?? $emptyrecord;

$localcount = $localrecord->count + $duplicatedrecord->count;
$deltacount = abs($filedir->count - $localcount);
$localsize = $localrecord->size + $duplicatedrecord->size;
$deltasize = abs($filedir->size - $localsize);
$row['date'] = userdate($timecreated, get_string('strftimedaydatetime'));
if ($this->is_downloading() && in_array($this->download, ['csv', 'excel', 'json', 'ods'])) {
$row['local_count'] = $rawrecords[$id.OBJECT_LOCATION_LOCAL]->count;
$row['local_size'] = $rawrecords[$id.OBJECT_LOCATION_LOCAL]->size;
$row['duplicated_count'] = $rawrecords[$id.OBJECT_LOCATION_DUPLICATED]->count;
$row['duplicated_size'] = $rawrecords[$id.OBJECT_LOCATION_DUPLICATED]->size;
$row['external_count'] = $rawrecords[$id.OBJECT_LOCATION_EXTERNAL]->count;
$row['external_size'] = $rawrecords[$id.OBJECT_LOCATION_EXTERNAL]->size;
$row['missing_count'] = $rawrecords[$id.OBJECT_LOCATION_ERROR]->count;
$row['missing_size'] = $rawrecords[$id.OBJECT_LOCATION_ERROR]->size;
$row['total_count'] = $rawrecords[$id.'total']->count;
$row['total_size'] = $rawrecords[$id.'total']->size;
$row['filedir_count'] = $rawrecords[$id.'filedir']->count;
$row['filedir_size'] = $rawrecords[$id.'filedir']->size;
$row['local_count'] = $localrecord->count;
$row['local_size'] = $localrecord->size;
$row['duplicated_count'] = $duplicatedrecord->count;
$row['duplicated_size'] = $duplicatedrecord->size;
$row['orphaned_count'] = $orphanedrecord->count;
$row['orphaned_size'] = get_string('object_status:location:orphanedsizeunknown', 'tool_objectfs');
$row['external_count'] = $externalrecord->count;
$row['external_size'] = $externalrecord->size;
$row['missing_count'] = $errorrecord->count;
$row['missing_size'] = $errorrecord->size;
$row['total_count'] = $total->count;
$row['total_size'] = $total->size;
$row['filedir_count'] = $filedir->count;
$row['filedir_size'] = $filedir->size;
$row['delta_count'] = $deltacount;
$row['delta_size'] = $deltasize;
} else {
$row['local_count'] = number_format($rawrecords[$id.OBJECT_LOCATION_LOCAL]->count);
$row['local_size'] = display_size($rawrecords[$id.OBJECT_LOCATION_LOCAL]->size);
$row['duplicated_count'] = number_format($rawrecords[$id.OBJECT_LOCATION_DUPLICATED]->count);
$row['duplicated_size'] = display_size($rawrecords[$id.OBJECT_LOCATION_DUPLICATED]->size);
$row['external_count'] = number_format($rawrecords[$id.OBJECT_LOCATION_EXTERNAL]->count);
$row['external_size'] = display_size($rawrecords[$id.OBJECT_LOCATION_EXTERNAL]->size);
$row['missing_count'] = number_format($rawrecords[$id.OBJECT_LOCATION_ERROR]->count);
$row['missing_size'] = display_size($rawrecords[$id.OBJECT_LOCATION_ERROR]->size);
$row['total_count'] = number_format($rawrecords[$id.'total']->count);
$row['total_size'] = display_size($rawrecords[$id.'total']->size);
$row['filedir_count'] = number_format($rawrecords[$id.'filedir']->count);
$row['filedir_size'] = display_size($rawrecords[$id.'filedir']->size);
$row['local_count'] = number_format($localrecord->count);
$row['local_size'] = display_size($localrecord->size);
$row['duplicated_count'] = number_format($duplicatedrecord->count);
$row['duplicated_size'] = display_size($duplicatedrecord->size);
$row['orphaned_count'] = number_format($orphanedrecord->count);
$row['orphaned_size'] = get_string('object_status:location:orphanedsizeunknown', 'tool_objectfs');
$row['external_count'] = number_format($externalrecord->count);
$row['external_size'] = display_size($externalrecord->size);
$row['missing_count'] = number_format($errorrecord->count);
$row['missing_size'] = display_size($errorrecord->size);
$row['total_count'] = number_format($total->count);
$row['total_size'] = display_size($total->size);
$row['filedir_count'] = number_format($filedir->count);
$row['filedir_size'] = display_size($filedir->size);
$row['delta_count'] = number_format($deltacount);
$row['delta_size'] = display_size($deltasize);
}
Expand Down
5 changes: 5 additions & 0 deletions classes/local/report/object_status_history_table.php
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,10 @@ public function col_count(\stdClass $row) {
* @return string
*/
public function col_size(\stdClass $row) {
// For orphaned entries, the filesize is N/A or Unknown. Note: non-strict check as the heading is a string.
if ($row->heading == OBJECT_LOCATION_ORPHANED) {
return get_string('object_status:location:orphanedsizeunknown', 'tool_objectfs');
}
return $this->add_barchart($row->size, $this->maxsize, 'size');
}

Expand Down Expand Up @@ -227,6 +231,7 @@ public function get_file_location_string($filelocation) {
OBJECT_LOCATION_LOCAL => 'object_status:location:local',
OBJECT_LOCATION_DUPLICATED => 'object_status:location:duplicated',
OBJECT_LOCATION_EXTERNAL => 'object_status:location:external',
OBJECT_LOCATION_ORPHANED => 'object_status:location:orphaned',
];
if (isset($locationstringmap[$filelocation])) {
return get_string($locationstringmap[$filelocation], 'tool_objectfs');
Expand Down
2 changes: 2 additions & 0 deletions classes/log/aggregate_logger.php
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,8 @@ public function location_to_string($location) {
return 'duplicated';
case OBJECT_LOCATION_EXTERNAL:
return 'remote';
case OBJECT_LOCATION_ORPHANED:
return 'orphaned';
default:
return $location;
}
Expand Down
61 changes: 61 additions & 0 deletions classes/task/delete_orphaned_object_metadata.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
<?php
// This file is part of Moodle - http://moodle.org/
//
// Moodle is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// Moodle is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with Moodle. If not, see <http://www.gnu.org/licenses/>.

/**
* Task that checks for old orphaned objects, and removes their metadata
* (record) as it is no longer useful/relevant.
*
* @package tool_objectfs
* @author Kevin Pham <[email protected]>
* @copyright Catalyst IT
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/

namespace tool_objectfs\task;

defined('MOODLE_INTERNAL') || die();

require_once(__DIR__ . '/../../lib.php');

class delete_orphaned_object_metadata extends task {

/** @var string $stringname */
protected $stringname = 'delete_orphaned_object_metadata_task';

/**
* Execute task
*/
public function execute() {
global $DB;

$wheresql = 'location = :location and timeduplicated < :ageforremoval';
$ageforremoval = $this->config->maxorphanedage;
if (empty($ageforremoval)) {
mtrace('Skipping deletion of orphaned object metadata as maxorphanedage is set to an empty value.');
return;
}

$params = [
'location' => OBJECT_LOCATION_ORPHANED,
'ageforremoval' => time() - $ageforremoval
];
$count = $DB->count_records_select('tool_objectfs_objects', $wheresql, $params);
if (!empty($count)) {
mtrace("Deleting $count records with orphaned metadata (orphaned tool_objectfs_objects)");
$DB->delete_records_select('tool_objectfs_objects', $wheresql, $params);
}
}
}
Loading

0 comments on commit caa0d12

Please sign in to comment.