mirrors/cobalt
1
0
Fork 0
mirror of https://github.com/imputnet/cobalt.git synced 2025-06-28 01:18:27 +00:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

web/queue: replace pipelineResults array with object

Browse Source
This commit is contained in:
jj 2025-05-23 17:44:47 +00:00
parent 17bcfa3a03
commit 892c055d6a
No known key found for this signature in database
7 changed files with 38 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
web/src/components/queue/ProcessingQueueItem.svelte
View File

@ -96,7 +96,8 @@
// if only fetch workers are running, then we should
// show the sum of all running & completed fetch workers
if (running.size === 1 && running.has("fetch")) {
totalSize += info.pipelineResults.reduce((s, p) => s + (p?.size ?? 0), 0);
totalSize += Object.values(info.pipelineResults)
.reduce((s, p) => s + (p?.size ?? 0), 0);
}
const runningText = [...running].map(task => $t(`queue.state.running.${task}`)).join(", ");
@ -107,7 +108,7 @@
}
const firstUnstarted = info.pipeline.find(w => {
if (info.completedWorkers.has(w.workerId))
if (info.pipelineResults[w.workerId])
return false;
const task = currentTasks[w.workerId];
@ -134,7 +135,7 @@
};
const getWorkerProgress = (item: CobaltQueueItem, workerId: UUID): number | undefined => {
if (item.state === 'running' && item.completedWorkers.has(workerId)) {
if (item.state === 'running' && item.pipelineResults[workerId]) {
return 100;
}
@ -187,7 +188,7 @@
<ProgressBar
percentage={getWorkerProgress(info, task.workerId) || 0}
workerId={task.workerId}
completedWorkers={info.completedWorkers}
pipelineResults={info.pipelineResults}
/>
{/each}
</div>

8
web/src/components/queue/ProgressBar.svelte
View File

@ -1,14 +1,14 @@
<script lang="ts">
import Skeleton from "$components/misc/Skeleton.svelte";
import type { UUID } from "$lib/types/queue";
import type { CobaltQueueItemRunning, UUID } from "$lib/types/queue";
type Props = {
percentage?: number;
workerId: UUID;
completedWorkers: Set<string>;
pipelineResults: CobaltQueueItemRunning['pipelineResults'];
}
let { percentage = 0, workerId, completedWorkers }: Props = $props();
let { percentage = 0, workerId, pipelineResults }: Props = $props();
</script>
<div class="file-progress">
@ -17,7 +17,7 @@
class="progress"
style="width: {Math.min(100, percentage)}%"
></div>
{:else if completedWorkers.has(workerId)}
{:else if pipelineResults[workerId]}
<div
class="progress"
style="width: 100%"

14
web/src/lib/state/task-manager/queue.ts
View File

@ -6,11 +6,15 @@ import { clearCurrentTasks, removeWorkerFromQueue } from "$lib/state/task-manage
import type { CobaltQueue, CobaltQueueItem, CobaltQueueItemRunning, UUID } from "$lib/types/queue";
export const DUMMY_FILE = new File([], 'pipeline_result_deleted.bin');
const clearPipelineCache = (queueItem: CobaltQueueItem) => {
if (queueItem.state === "running") {
let item: File | undefined;
while ((item = queueItem.pipelineResults.pop())) {
for (const [ workerId, item ] of Object.entries(queueItem.pipelineResults)) {
if (item.name !== DUMMY_FILE.name) {
removeFromFileStorage(item.name);
queueItem.pipelineResults[workerId] = DUMMY_FILE;
}
}
} else if (queueItem.state === "done") {
removeFromFileStorage(queueItem.resultFile.name);
@ -75,8 +79,7 @@ export function pipelineTaskDone(id: UUID, workerId: UUID, file: File) {
const item = queueData[id];
if (item && item.state === 'running') {
item.pipelineResults.push(file);
item.completedWorkers.add(workerId);
item.pipelineResults[workerId] = file;
}
return queueData;
@ -92,8 +95,7 @@ export function itemRunning(id: UUID) {
if (data) {
data.state = 'running';
data.completedWorkers ??= new Set();
data.pipelineResults ??= [];
data.pipelineResults ??= {};
}
return queueData;

2
web/src/lib/task-manager/queue.ts
View File

@ -217,7 +217,7 @@ export const getProgress = (item: CobaltQueueItem, currentTasks: CobaltCurrentTa
let sum = 0;
for (const worker of item.pipeline) {
if (item.completedWorkers.has(worker.workerId)) {
if (item.pipelineResults[worker.workerId]) {
sum += 1;
} else {
const task = currentTasks[worker.workerId];

13
web/src/lib/task-manager/run-worker.ts
View File

@ -12,7 +12,7 @@ export const killWorker = (worker: Worker, unsubscribe: () => void, interval?: N
if (interval) clearInterval(interval);
}
export const startWorker = async ({ worker, workerId, parentId, workerArgs }: CobaltPipelineItem) => {
export const startWorker = async ({ worker, workerId, dependsOn, parentId, workerArgs }: CobaltPipelineItem) => {
let files: File[] = [];
switch (worker) {
@ -22,10 +22,15 @@ export const startWorker = async ({ worker, workerId, parentId, workerArgs }: Co
files = workerArgs.files;
}
if (files.length === 0) {
const parent = get(queue)[parentId];
if (parent.state === "running" && parent.pipelineResults.length) {
files = parent.pipelineResults;
if (parent?.state === "running" && dependsOn) {
for (const workerId of dependsOn) {
const file = parent.pipelineResults[workerId];
if (!file) {
return itemError(parentId, workerId, "queue.ffmpeg.no_args");
}
files.push(file);
}
}

13
web/src/lib/task-manager/scheduler.ts
View File

@ -1,7 +1,7 @@
import { get } from "svelte/store";
import { startWorker } from "$lib/task-manager/run-worker";
import { addWorkerToQueue, currentTasks } from "$lib/state/task-manager/current-tasks";
import { itemDone, itemError, itemRunning, queue } from "$lib/state/task-manager/queue";
import { DUMMY_FILE, itemDone, itemError, itemRunning, queue } from "$lib/state/task-manager/queue";
import type { CobaltPipelineItem } from "$lib/types/workers";
@ -27,8 +27,11 @@ export const schedule = () => {
// if all workers are completed, then return the
// the final file and go to the next task
if (task.completedWorkers.size === task.pipeline.length) {
const finalFile = task.pipelineResults.pop();
if (Object.keys(task.pipelineResults).length === task.pipeline.length) {
// swap final file for a dummy, so that it doesn't get
// deleted when we clean up the intermediate files
const finalFile = task.pipelineResults[finalWorker.workerId];
task.pipelineResults[finalWorker.workerId] = DUMMY_FILE;
if (finalFile) {
itemDone(task.id, finalFile);
@ -42,11 +45,11 @@ export const schedule = () => {
// if current worker is completed, but there are more workers,
// then start the next one and wait to be called again
for (const worker of task.pipeline) {
if (task.completedWorkers.has(worker.workerId) || ongoingTasks[worker.workerId]) {
if (task.pipelineResults[worker.workerId] || ongoingTasks[worker.workerId]) {
continue;
}
const needsToWait = worker.dependsOn?.some(id => !task.completedWorkers.has(id));
const needsToWait = worker.dependsOn?.some(id => !task.pipelineResults[id]);
if (needsToWait) {
break;
}

3
web/src/lib/types/queue.ts
View File

@ -19,8 +19,7 @@ type CobaltQueueItemWaiting = CobaltQueueBaseItem & {
export type CobaltQueueItemRunning = CobaltQueueBaseItem & {
state: "running",
completedWorkers: Set<UUID>,
pipelineResults: File[],
pipelineResults: Record<UUID, File>,
};
type CobaltQueueItemDone = CobaltQueueBaseItem & {