File: //proc/self/cwd/wp-content/plugins/backup/src/JetBackup/Cron/Task/Reindex.php
<?php
namespace JetBackup\Cron\Task;
use Exception;
use JetBackup\Alert\Alert;
use JetBackup\BackupJob\BackupJob;
use JetBackup\Data\Engine;
use JetBackup\Destination\Destination;
use JetBackup\Destination\DestinationFile;
use JetBackup\Destination\Integration\DestinationDirIterator;
use JetBackup\Destination\Integration\DestinationFile as DestinationFileAlias;
use JetBackup\Entities\Util;
use JetBackup\Exception\DBException;
use JetBackup\Exception\DestinationException;
use JetBackup\Exception\ExecutionTimeException;
use JetBackup\Exception\IOException;
use JetBackup\Exception\JBException;
use JetBackup\Exception\JetBackupLinuxException;
use JetBackup\Exception\ReindexException;
use JetBackup\Exception\SnapshotMetaException;
use JetBackup\Exception\TaskException;
use JetBackup\Exception\ValidationException;
use JetBackup\Factory;
use JetBackup\JetBackup;
use JetBackup\JetBackupLinux\JetBackupLinux;
use JetBackup\License\License;
use JetBackup\Queue\Queue;
use JetBackup\Queue\QueueItem;
use JetBackup\Queue\QueueItemReindex;
use JetBackup\ResumableTask\ResumableTask;
use JetBackup\Snapshot\Snapshot;
use JetBackup\Snapshot\SnapshotItem;
use SleekDB\Exceptions\InvalidArgumentException;
if (!defined( '__JETBACKUP__')) die('Direct access is not allowed');
class Reindex extends Task {
const LOG_FILENAME = 'reindex';
private QueueItemReindex $_queue_item_reindex;
private ?Destination $_destination=null;
public function __construct() {
parent::__construct(self::LOG_FILENAME);
}
/**
* @return void
* @throws InvalidArgumentException
* @throws DBException
* @throws TaskException
* @throws \SleekDB\Exceptions\IOException
*/
public function execute():void {
parent::execute();
$this->_queue_item_reindex = $this->getQueueItem()->getItemData();
$this->_destination = $this->_queue_item_reindex->getDestinationId() ? new Destination($this->_queue_item_reindex->getDestinationId()) : null;
if($this->_destination && !License::isValid() && !in_array($this->_destination->getType(), Destination::LICENSE_EXCLUDED)) {
$this->getLogController()->logError("You can't reindex from {$this->_destination->getType()} destination without a license");
$this->getQueueItem()->updateStatus(Queue::STATUS_ABORTED);
$this->getQueueItem()->updateProgress('Reindex Aborted!', QueueItem::PROGRESS_LAST_STEP);
return;
}
if($this->getQueueItem()->getStatus() == Queue::STATUS_PENDING) {
if($this->_destination) $this->getLogController()->logMessage("Starting reindex for destination \"{$this->_destination->getName()}\"");
else $this->getLogController()->logMessage("Starting reindex for JetBackup Linux");
$this->getQueueItem()->getProgress()->setTotalItems(count(Queue::STATUS_REINDEX_NAMES)+3);
$this->getQueueItem()->save();
$this->getQueueItem()->updateProgress('Starting reindex');
} else if($this->getQueueItem()->getStatus() > Queue::STATUS_PENDING) {
$this->getLogController()->logMessage('Resumed Reindex');
}
try {
$this->func([$this, '_checkRequirements']);
$this->func([$this, '_markSnapshots']);
$this->func([$this, '_reindexSnapshots']);
$this->func([$this, '_deleteSnapshots']);
if($this->getQueueItem()->getStatus() < Queue::STATUS_DONE && !$this->getQueueItem()->getErrors()) $this->getQueueItem()->updateStatus(Queue::STATUS_DONE);
else $this->getQueueItem()->updateStatus(Queue::STATUS_PARTIALLY);
$this->getLogController()->logMessage('Completed!');
} catch(ReindexException $e) {
$this->getQueueItem()->updateStatus(Queue::STATUS_FAILED);
$this->getLogController()->logError($e->getMessage());
$this->getLogController()->logMessage('Failed!');
}
$this->getQueueItem()->updateProgress(
$this->getQueueItem()->getStatus() == Queue::STATUS_DONE
? 'Reindex Completed!'
: ($this->getQueueItem()->getStatus() == Queue::STATUS_PARTIALLY
? 'Completed with errors (see logs)'
: 'Reindex Failed!'),
QueueItem::PROGRESS_LAST_STEP
);
$this->getLogController()->logMessage('Total time: ' . $this->getExecutionTimeLimit());
}
public function _checkRequirements() {
if($this->_destination) return;
if(!Factory::getSettingsGeneral()->isJBIntegrationEnabled()) throw new ReindexException("JetBackup Linux integration isn't enabled");
if(!JetBackupLinux::isInstalled()) throw new ReindexException("JetBackup Linux isn't installed on this server");
try {
JetBackupLinux::checkRequirements();
} catch (JetBackupLinuxException $e) {
throw new ReindexException($e->getMessage());
}
}
/**
* @return void
* @throws ReindexException
*/
public function _deleteSnapshots() {
$this->getLogController()->logMessage('Execution time: ' . $this->getExecutionTimeElapsed());
$this->getLogController()->logMessage('TTL time: ' . $this->getExecutionTimeLimit());
$this->getQueueItem()->updateStatus(Queue::STATUS_REINDEX_DELETE_SNAPSHOTS);
$this->getLogController()->logMessage("Removing all unneeded snapshots");
try {
// remove all snapshots with reindex flag
if($this->_destination) {
$list = Snapshot::query()
->where([Snapshot::DESTINATION_ID, '=', $this->_destination->getId()])
->where([Engine::ENGINE, '=', Engine::ENGINE_WP])
->where([Snapshot::REINDEX, '=', true])
->getQuery()
->fetch();
} else {
$list = Snapshot::query()
->where([Engine::ENGINE, '=', Engine::ENGINE_JB])
->where([Snapshot::REINDEX, '=', true])
->getQuery()
->fetch();
}
if (empty($list)) return;
foreach ($list as $item) {
$snapshot = new Snapshot($item[JetBackup::ID_FIELD]);
if(!$snapshot->getId()) continue;
$snapshot->delete();
}
} catch( Exception $e) {
throw new ReindexException($e->getMessage());
}
}
/**
* @throws DBException
* @throws \SleekDB\Exceptions\IOException
* @throws InvalidArgumentException
*/
private function _handleSGBSnapshot($filename) {
$this->getLogController()->logDebug("[_handleSnapshot] Filename: $filename");
try {
$stat = $this->_destination->getInstance()->getFileStat($filename);
$this->getLogController()->logDebug("[_handleSnapshot] getFileStat: " . print_r($stat, true));
} catch( Exception $e) {
$this->getLogController()->logError("[_handleSnapshot] Failed getting snapshot stats. Error: " . $e->getMessage());
return;
}
$name = substr($stat->getName(), 0, -5);
$created = $stat->getModifyTime() ?? 0;
$size = $stat->getSize() ?? 0;
$this->getLogController()->logDebug("[_handleSnapshot] Name: $name");
$this->getLogController()->logDebug("[_handleSnapshot] Created: $created");
$this->getLogController()->logDebug("[_handleSnapshot] Size: $size");
$this->getLogController()->logMessage("");
$this->getLogController()->logMessage("\tLegacy Snapshot found \"$name\"");
try {
$details = Snapshot::query()
->where([Snapshot::NAME, '=', $name])
->where([ Engine::ENGINE, '=', Engine::ENGINE_SGB])
->where([Snapshot::DESTINATION_ID, '=', $this->_destination->getId()])
->getQuery()
->first();
} catch( Exception $e) {
$this->getLogController()->logError("[_handleSnapshot] Failed importing snapshot. Error: " . $e->getMessage());
return;
}
if($details) {
$snapshot = new Snapshot($details[JetBackup::ID_FIELD]);
$this->getLogController()->logMessage("\t[_handleSnapshot] Snapshot found on local database {$snapshot->getName()}");
$snapshot->setCreated($created);
$snapshot->setBackupType(BackupJob::TYPE_ACCOUNT);
$snapshot->setContains(BackupJob::BACKUP_ACCOUNT_CONTAINS_FULL);
$snapshot->setStructure(BackupJob::STRUCTURE_COMPRESSED);
$snapshot->setReindex(false);
$snapshot->save();
return;
}
$this->getLogController()->logMessage("\t[_handleSnapshot] Snapshot not exist on local database");
$snapshot = new Snapshot();
$snapshot->setBackupType(BackupJob::TYPE_ACCOUNT);
$snapshot->setContains(BackupJob::BACKUP_ACCOUNT_CONTAINS_FULL);
$snapshot->setStructure(BackupJob::STRUCTURE_COMPRESSED);
$snapshot->setDestinationId($this->_destination->getId());
$snapshot->setName($name);
$snapshot->setCreated($created);
$snapshot->setSize($size);
$snapshot->setEngine(Engine::ENGINE_SGB);
$snapshot->save();
$item = new SnapshotItem();
$item->setParentId($snapshot->getId());
$item->setBackupType(BackupJob::TYPE_ACCOUNT);
$item->setBackupContains(BackupJob::BACKUP_ACCOUNT_CONTAINS_FULL);
$item->setName($name);
$item->setCreated($created);
$item->setSize($size);
$item->setPath($filename);
$item->setEngine(Engine::ENGINE_SGB);
$item->save();
}
/**
* @param $identifier
* @param $name
*
* @return void
* @throws DBException
* @throws IOException
* @throws InvalidArgumentException
* @throws \SleekDB\Exceptions\IOException
*/
private function _handleSnapshot($identifier, $name) {
$this->getLogController()->logMessage("");
$this->getLogController()->logMessage("\tSnapshot found \"$name\"");
$meta_filepath = sprintf(Snapshot::META_FILEPATH, JetBackup::SEP . $identifier . JetBackup::SEP . $name);
$this->getLogController()->logDebug("[_handleSnapshot] Identifier: $identifier");
$this->getLogController()->logDebug("[_handleSnapshot] Meta path: $meta_filepath");
try {
if(!$this->_destination->fileExists($meta_filepath)) throw new DestinationException("can't find meta file $meta_filepath");
} catch(DestinationException $e) {
$this->getLogController()->logError("[_handleSnapshot] Failed importing snapshot. Error: " . $e->getMessage());
$this->getQueueItem()->addError();
return;
}
$meta_file = Factory::getLocations()->getTempDir() . JetBackup::SEP . 'dest_reindex_' . Util::generateUniqueId() . '.tmp';
try {
$this->_destination->copyFileToLocal($meta_filepath, $meta_file, $this->getQueueItem(), $this);
} catch(IOException|DestinationException $e) {
$this->getLogController()->logError("[_handleSnapshot] Failed importing snapshot. Error: failed downloading meta file ({$e->getMessage()})");
$this->getQueueItem()->addError();
return;
}
try {
$details = Snapshot::query()
->where([Snapshot::NAME, '=', $name])
->where([Engine::ENGINE, '=', Engine::ENGINE_WP])
->where([Snapshot::DESTINATION_ID, '=', $this->_destination->getId()])
->where([Snapshot::JOB_IDENTIFIER, '=', $identifier])
->getQuery()
->first();
} catch( Exception $e) {
$this->getLogController()->logError("[_handleSnapshot] Failed importing snapshot. Error: " . $e->getMessage());
$this->getQueueItem()->addError();
return;
}
if($details) {
$snapshot = new Snapshot($details[JetBackup::ID_FIELD]);
$this->getLogController()->logMessage("\t[_handleSnapshot] Snapshot found on local database {$snapshot->getName()}");
$snapshot->setReindex(false);
try {
$snapshot->removeItems();
} catch( Exception $e) {
$this->getLogController()->logError("[_handleSnapshot] Failed removing snapshot items. Error: " . $e->getMessage());
$this->getQueueItem()->addError();
return;
}
} else {
$this->getLogController()->logMessage("\t[_handleSnapshot] Snapshot not exist on local database");
$snapshot = new Snapshot();
$snapshot->setDestinationId($this->_destination->getId());
$snapshot->setJobIdentifier($identifier);
}
try {
$snapshot->importMeta($meta_file, $this->_queue_item_reindex->isCrossDomain());
} catch(SnapshotMetaException $e) {
// Can't import snapshot, there is a missing data in meta file
$this->getLogController()->logError("Failed importing snapshot. Error: " . $e->getMessage());
$this->getQueueItem()->addError();
unlink($meta_file);
return;
}
unlink($meta_file);
$snapshot->save();
$this->getLogController()->logMessage("\tSnapshot imported successfully");
$this->getLogController()->logDebug("Snapshot data: " . print_r($snapshot->getDisplay(), true));
}
/**
* @param $identifier
*
* @return array
* @throws ReindexException
*/
public function _fetchSnapshots($identifier): array {
$this->getLogController()->logDebug("[_fetchSnapshots] Identifier: /$identifier/");
try {
$list = $this->_destination->listDir("/$identifier/");
} catch(DestinationException $e) {
throw new ReindexException($e->getMessage());
}
$snapshots = [];
while($list->hasNext()) {
$name = $list->getNext()->getName();
$this->getLogController()->logDebug("[_fetchSnapshots] Name: $name");
if(!preg_match(Snapshot::SNAPSHOT_NAME_REGEX, $name)) continue;
$snapshots[] = $name;
}
return $snapshots;
}
/**
* @param int $pageSize
* @param int $skip
*
* @return array
* @throws JetBackupLinuxException
*/
public function _fetchJBBackupsPage(int $pageSize, int $skip):array {
$this->getLogController()->logDebug("[_fetchJBBackupsPage] Fetching backups (limit=$pageSize, skip=$skip)");
$this->getLogController()->logDebug("[_fetchJBBackupsPage] Memory before: " . Util::bytesToHumanReadable(memory_get_usage(true)));
$startTime = microtime(true);
$backups = JetBackupLinux::listBackups([], $pageSize, $skip);
$elapsed = round(microtime(true) - $startTime, 2);
$this->getLogController()->logDebug("[_fetchJBBackupsPage] Got " . count($backups) . " backups in {$elapsed}s");
$this->getLogController()->logDebug("[_fetchJBBackupsPage] Memory after: " . Util::bytesToHumanReadable(memory_get_usage(true)));
$this->getLogController()->logDebug("[_fetchJBBackupsPage] Response payload: " . Util::bytesToHumanReadable(strlen(serialize($backups))));
foreach($backups as $i => $backup) {
$this->getLogController()->logDebug("[_fetchJBBackupsPage] Backup[$i]: id=" . ($backup['_id'] ?? 'N/A')
. " created=" . ($backup['created'] ?? 'N/A')
. " structure=" . ($backup['backup_structure'] ?? 'N/A')
. " items=" . (isset($backup['items']) ? count($backup['items']) : 0)
);
}
return $backups;
}
public function _handleJBSnapshot($key, $backup) {
if(!$backup['items']) return;
$this->getLogController()->logDebug("[_handleJBSnapshot] Processing backup " . $backup['_id'] . " (" . count($backup['items']) . " items)");
if ($backup['backup_structure'] != JetBackupLinux::BACKUP_STRUCTURE_INCREMENTAL && $backup['backup_structure'] != JetBackupLinux::BACKUP_STRUCTURE_DEDUPLICATION) {
$this->getLogController()->logError("[_handleJBSnapshot] Backup ID" . $backup['_id'] . " is not incremental or deduplication, this type is not supported, skipping");
return;
}
try {
$details = Snapshot::query()
->where([Engine::ENGINE, '=', Engine::ENGINE_JB])
->where([Snapshot::UNIQUE_ID, '=', $backup['_id']])
->getQuery()
->first();
} catch( Exception $e) {
$this->getLogController()->logError("[_handleJBSnapshot] Failed importing snapshot. Error: " . $e->getMessage());
return;
}
if($details) {
$this->getLogController()->logMessage("\t[_handleJBSnapshot] Snapshot found on local database, skipping import");
$snapshot = new Snapshot($details[JetBackup::ID_FIELD]);
$snapshot->setReindex(false);
$snapshot->removeItems();
} else {
$this->getLogController()->logMessage("\t[_handleJBSnapshot] New snapshot found! Importing to local database...");
$snapshot = new Snapshot();
$snapshot->setNotes($backup['notes']);
}
$structure = $backup['backup_structure'] == JetBackupLinux::BACKUP_STRUCTURE_INCREMENTAL
? BackupJob::STRUCTURE_INCREMENTAL
: $backup['backup_structure'];
$snapshot->setEngine(Engine::ENGINE_JB);
$snapshot->setBackupType(BackupJob::TYPE_ACCOUNT);
$snapshot->setUniqueId($backup['_id']);
$snapshot->setCreated(strtotime($backup['created']));
$snapshot->setStructure($structure);
$snapshot->setName(sprintf(Snapshot::SNAPSHOT_NAME_PATTERN, Util::date('Y-m-d_His', strtotime($backup['created'])), $backup['_id']));
// Save snapshot early to get ID for items (reduces memory by saving items progressively)
$snapshot->save();
$size = 0;
$contains = 0;
$itemCount = 0;
foreach($backup['items'] as $item_details) {
if($item_details['disabled']) {
$this->getLogController()->logMessage("\t[_handleJBSnapshot] Skipping item " . $item_details['_id'] . " Remote destination disabled");
continue;
}
switch($item_details['backup_contains']) {
case JetBackupLinux::BACKUP_TYPE_ACCOUNT_HOMEDIR:
$item = new SnapshotItem();
$item->setParentId($snapshot->getId());
$item->setEngine(Engine::ENGINE_JB);
$item->setName($item_details['name']);
$item->setPath($item_details['path']);
$item->setUniqueId($item_details['_id']);
$item->setCreated(strtotime($item_details['created']));
$item->setBackupType(BackupJob::TYPE_ACCOUNT);
$item->setBackupContains(BackupJob::BACKUP_ACCOUNT_CONTAINS_HOMEDIR);
$item->save();
$size += intval($item_details['size']);
$contains |= BackupJob::BACKUP_ACCOUNT_CONTAINS_HOMEDIR;
$itemCount++;
break;
case JetBackupLinux::BACKUP_TYPE_ACCOUNT_DATABASES:
if($item_details['name'] != DB_NAME) continue 2;
$item = new SnapshotItem();
$item->setParentId($snapshot->getId());
$item->setEngine(Engine::ENGINE_JB);
$item->setName($item_details['name']);
$item->setPath($item_details['path']);
$item->setUniqueId($item_details['_id']);
$item->setCreated(strtotime($item_details['created']));
$item->setBackupType(BackupJob::TYPE_ACCOUNT);
$item->setBackupContains(BackupJob::BACKUP_ACCOUNT_CONTAINS_DATABASE);
$item->save();
$size += intval($item_details['size']);
$contains |= BackupJob::BACKUP_ACCOUNT_CONTAINS_DATABASE;
$itemCount++;
break;
}
}
if($itemCount === 0) {
$snapshot->delete();
return;
}
if($contains != BackupJob::BACKUP_ACCOUNT_CONTAINS_FULL) {
$item = new SnapshotItem();
$item->setParentId($snapshot->getId());
$item->setEngine(Engine::ENGINE_JB);
$item->setBackupType(BackupJob::TYPE_ACCOUNT);
$item->setBackupContains(BackupJob::BACKUP_ACCOUNT_CONTAINS_FULL);
$item->setCreated(strtotime($backup['created']));
$item->setName('');
$item->setPath('');
$item->save();
}
$snapshot->setContains($contains);
$snapshot->setSize($size);
$snapshot->save();
}
/**
* @return void
* @throws ReindexException
*/
public function _reindexSnapshots() : void {
$this->getLogController()->logMessage('Execution time: ' . $this->getExecutionTimeElapsed());
$this->getLogController()->logMessage('TTL time: ' . $this->getExecutionTimeLimit());
if($this->getQueueItem()->getStatus() < Queue::STATUS_REINDEX_INDEXING_SNAPSHOTS) {
$this->getQueueItem()->updateStatus(Queue::STATUS_REINDEX_INDEXING_SNAPSHOTS);
$this->getQueueItem()->updateProgress('Indexing Snapshots');
}
if($this->_destination) {
$this->foreachCallable([ $this, '_fetchRootDirectory' ], [], function($i, $filename) {
$this->getLogController()->logDebug("Inspecting $filename");
if(preg_match(BackupJob::IDENTIFIER_REGEX, $filename)) {
$this->getLogController()->logMessage("Reindexing identifier \"$filename\"");
$this->foreachCallable([ $this, '_fetchSnapshots'], [$filename], function($s, $name) use ($filename) {
$this->_handleSnapshot($filename, $name);
}, 'snapshots_' . $filename);
} elseif(str_ends_with($filename, BackupJob::SNAPSHOT_SGBP_SUFFIX)) {
$this->_handleSGBSnapshot($filename); // Anything *.sgbp
}
});
} else {
try {
$pageSize = 25;
$page = 0;
$totalProcessed = 0;
do {
// Check execution time before fetching next page
$this->checkExecutionTime();
$skip = $page * $pageSize;
$pageName = 'jb_backups_page_' . $page;
$this->foreachCallable(
[$this, '_fetchJBBackupsPage'],
[$pageSize, $skip],
[$this, '_handleJBSnapshot'],
$pageName
);
// Check how many items were in this page to determine if more pages exist
$resumable = $this->getQueueItem()->getResumableTask();
$item = $resumable->_getItem($pageName, ResumableTask::TYPE_FOREACH);
$data = $item->getData();
$pageCount = $data ? ($data['total'] ?? 0) : 0;
$totalProcessed += $pageCount;
// Update sub-progress for backup-level tracking within the reindex phase
$this->getQueueItem()->getProgress()->setCurrentSubItem($totalProcessed);
// Estimate total: if full page, assume at least one more page; otherwise this is the last page
$estimatedTotal = $pageCount >= $pageSize ? $totalProcessed + $pageSize : $totalProcessed;
$this->getQueueItem()->getProgress()->setTotalSubItems($estimatedTotal);
$this->getQueueItem()->getProgress()->setSubMessage("$totalProcessed backups processed");
$this->getQueueItem()->save();
$this->getLogController()->logMessage("[_reindexSnapshots] Page $page completed ($pageCount backups, $totalProcessed total)");
$page++;
} while($pageCount >= $pageSize);
} catch(JetBackupLinuxException $e) {
throw new ReindexException($e->getMessage());
} catch ( DBException|ExecutionTimeException|JBException|\SleekDB\Exceptions\IOException|InvalidArgumentException $e ) {
}
}
// Reset sub-progress after reindex phase completes
$this->getQueueItem()->getProgress()->resetSub();
$this->getQueueItem()->save();
}
/**
* @throws ReindexException
*/
public function _markSnapshots():void {
$this->getLogController()->logMessage('Execution time: ' . $this->getExecutionTimeElapsed());
$this->getLogController()->logMessage('TTL time: ' . $this->getExecutionTimeLimit());
$this->getQueueItem()->updateStatus(Queue::STATUS_REINDEX_MARKING_SNAPSHOTS);
$this->getQueueItem()->updateProgress('Hiding all destination snapshots');
$this->getLogController()->logMessage("Hiding all destination snapshots");
try {
// mark all destination snapshots with reindex flag
if($this->_destination) {
Snapshot::query()
->where([Snapshot::DESTINATION_ID, '=', $this->_destination->getId()])
->where([Engine::ENGINE, '=', Engine::ENGINE_WP])
->getQuery()
->update([Snapshot::REINDEX => true]);
} else {
Snapshot::query()
->where([Engine::ENGINE, '=', Engine::ENGINE_JB])
->getQuery()
->update([Snapshot::REINDEX => true]);
}
} catch( Exception $e) {
throw new ReindexException($e->getMessage());
}
}
/**
* @return array
* @throws ReindexException
*/
public function _fetchRootDirectory():array {
$this->getLogController()->logMessage("Fetching destination root directory");
try {
$this->_destination->setLogController($this->getLogController());
$this->_destination->validate();
$list = $this->_destination->listDir('/');
} catch(ValidationException|DestinationException $e) {
throw new ReindexException($e->getMessage());
}
$output = [];
while($list->hasNext()) {
$output[] = $list->getNext()->getName();
}
if(!sizeof($output)) $this->getLogController()->logMessage("No files and directories were found on the destination root directory");
return $output;
}
}