Add delete old CSV job for maintenance

This commit is contained in:
Jake Barnby
2025-11-14 15:55:41 +13:00
parent a11cf6b002
commit b9211d7141
4 changed files with 50 additions and 4 deletions
+3 -4
View File
@@ -1499,19 +1499,18 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/push')
throw new Exception(Exception::USER_UNAUTHORIZED);
}
// Check if this is an internal/platform file based on JWT flag
$isInternal = $decoded['internal'] ?? false;
$db = $isInternal ? $dbForPlatform : $dbForProject;
$dbForProject = $isInternal ? $dbForPlatform : $dbForProject;
$isAPIKey = Auth::isAppUser(Authorization::getRoles());
$isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles());
$bucket = Authorization::skip(fn () => $db->getDocument('buckets', $bucketId));
$bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId));
if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && !$isAPIKey && !$isPrivilegedUser)) {
throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND);
}
$file = Authorization::skip(fn () => $db->getDocument('bucket_' . $bucket->getSequence(), $fileId));
$file = Authorization::skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId));
if ($file->isEmpty()) {
throw new Exception(Exception::STORAGE_FILE_NOT_FOUND);
}
+1
View File
@@ -138,6 +138,7 @@ const DELETE_TYPE_TOPIC = 'topic';
const DELETE_TYPE_TARGET = 'target';
const DELETE_TYPE_EXPIRED_TARGETS = 'invalid_targets';
const DELETE_TYPE_SESSION_TARGETS = 'session_targets';
const DELETE_TYPE_CSV_EXPORTS = 'csv_exports';
const DELETE_TYPE_MAINTENANCE = 'maintenance';
// Message types
@@ -95,6 +95,7 @@ class Maintenance extends Action
$this->renewCertificates($dbForPlatform, $queueForCertificates);
$this->notifyDeleteCache($cacheRetention, $queueForDeletes);
$this->notifyDeleteSchedules($schedulesDeletionRetention, $queueForDeletes);
$this->notifyDeleteCSVExports($queueForDeletes);
}, $interval, $delay);
}
@@ -106,6 +107,13 @@ class Maintenance extends Action
->trigger();
}
private function notifyDeleteCSVExports(Delete $queueForDeletes): void
{
$queueForDeletes
->setType(DELETE_TYPE_CSV_EXPORTS)
->trigger();
}
private function renewCertificates(Database $dbForPlatform, Certificate $queueForCertificate): void
{
$time = DatabaseDateTime::now();
+38
View File
@@ -179,6 +179,9 @@ class Deletes extends Action
case DELETE_TYPE_SESSION_TARGETS:
$this->deleteSessionTargets($project, $getProjectDB, $document);
break;
case DELETE_TYPE_CSV_EXPORTS:
$this->deleteOldCSVExports($dbForPlatform, $deviceForFiles);
break;
case DELETE_TYPE_MAINTENANCE:
$this->deleteExpiredTargets($project, $getProjectDB);
$this->deleteExecutionLogs($project, $getProjectDB, $executionRetention);
@@ -720,6 +723,41 @@ class Deletes extends Action
], $dbForProject);
}
/**
* @param Database $dbForPlatform
* @param Device $deviceForFiles
* @return void
* @throws Exception|Throwable
*/
private function deleteOldCSVExports(Database $dbForPlatform, Device $deviceForFiles): void
{
$bucket = $dbForPlatform->getDocument('buckets', 'default');
if ($bucket->isEmpty()) {
Console::warning('Default bucket not found, skipping CSV export cleanup');
return;
}
$oneWeekAgo = DateTime::addSeconds(new \DateTime(), -1 * 60 * 60 * 24 * 7); // 1 week
Console::info("Deleting CSV export files older than " . $oneWeekAgo);
$this->deleteByGroup('bucket_' . $bucket->getSequence(), [
Query::select([...$this->selects, '$createdAt', 'filename', 'path']),
Query::equal('bucketId', ['default']),
Query::endsWith('filename', ['.csv']),
Query::createdBefore($oneWeekAgo),
Query::orderDesc('$createdAt'),
Query::orderDesc(),
], $dbForPlatform, function (Document $file) use ($deviceForFiles) {
$path = $file->getAttribute('path');
if ($deviceForFiles->exists($path)) {
$deviceForFiles->delete($path);
Console::success('Deleted CSV file: ' . $file->getAttribute('name'));
}
});
}
/**
* @param Database $dbForPlatform
* @param string $datetime