saulteafarmer/Stirling-PDF
1
0
Fork 0
mirror of https://github.com/Stirling-Tools/Stirling-PDF.git synced 2025-09-18 17:39:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

Simplifying adding raw files

Browse Source
This commit is contained in:
Connor Yoh 2025-09-10 20:19:56 +01:00
parent 055d3acc82
commit 3b3a2df392
4 changed files with 122 additions and 160 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

57
frontend/src/contexts/FileContext.tsx
View File

@ -22,13 +22,12 @@ import {
FileId, FileId,
StirlingFileStub, StirlingFileStub,
StirlingFile, StirlingFile,
createStirlingFile
} from '../types/fileContext'; } from '../types/fileContext';
// Import modular components // Import modular components
import { fileContextReducer, initialFileContextState } from './file/FileReducer'; import { fileContextReducer, initialFileContextState } from './file/FileReducer';
import { createFileSelectors } from './file/fileSelectors'; import { createFileSelectors } from './file/fileSelectors';
import { AddedFile, addFiles, addStirlingFileStubs, consumeFiles, undoConsumeFiles, createFileActions } from './file/fileActions'; import { addFiles, addStirlingFileStubs, consumeFiles, undoConsumeFiles, createFileActions } from './file/fileActions';
import { FileLifecycleManager } from './file/lifecycle'; import { FileLifecycleManager } from './file/lifecycle';
import { FileStateContext, FileActionsContext } from './file/contexts'; import { FileStateContext, FileActionsContext } from './file/contexts';
import { IndexedDBProvider, useIndexedDB } from './IndexedDBContext'; import { IndexedDBProvider, useIndexedDB } from './IndexedDBContext';
@ -73,56 +72,23 @@ function FileContextInner({
dispatch({ type: 'SET_UNSAVED_CHANGES', payload: { hasChanges } }); dispatch({ type: 'SET_UNSAVED_CHANGES', payload: { hasChanges } });
}, []); }, []);
const selectFiles = (addedFilesWithIds: AddedFile[]) => { const selectFiles = (stirlingFiles: StirlingFile[]) => {
const currentSelection = stateRef.current.ui.selectedFileIds; const currentSelection = stateRef.current.ui.selectedFileIds;
const newFileIds = addedFilesWithIds.map(({ id }) => id); const newFileIds = stirlingFiles.map(stirlingFile => stirlingFile.fileId);
dispatch({ type: 'SET_SELECTED_FILES', payload: { fileIds: [...currentSelection, ...newFileIds] } }); dispatch({ type: 'SET_SELECTED_FILES', payload: { fileIds: [...currentSelection, ...newFileIds] } });
} }
// File operations using unified addFiles helper with persistence // File operations using unified addFiles helper with persistence
const addRawFiles = useCallback(async (files: File[], options?: { insertAfterPageId?: string; selectFiles?: boolean }): Promise<StirlingFile[]> => { const addRawFiles = useCallback(async (files: File[], options?: { insertAfterPageId?: string; selectFiles?: boolean }): Promise<StirlingFile[]> => {
const addedFilesWithIds = await addFiles({ files, ...options }, stateRef, filesRef, dispatch, lifecycleManager); const stirlingFiles = await addFiles({ files, ...options }, stateRef, filesRef, dispatch, lifecycleManager, enablePersistence);
// Auto-select the newly added files if requested // Auto-select the newly added files if requested
if (options?.selectFiles && addedFilesWithIds.length > 0) { if (options?.selectFiles && stirlingFiles.length > 0) {
selectFiles(addedFilesWithIds); selectFiles(stirlingFiles);
} }
// Persist to IndexedDB if enabled and update StirlingFileStub with version info return stirlingFiles;
if (indexedDB && enablePersistence && addedFilesWithIds.length > 0) { }, [enablePersistence]);
await Promise.all(addedFilesWithIds.map(async ({ file, id, thumbnail }) => {
try {
const metadata = await indexedDB.saveFile(file, id, thumbnail);
// Update StirlingFileStub with version information from IndexedDB
if (metadata.versionNumber || metadata.originalFileId) {
dispatch({
type: 'UPDATE_FILE_RECORD',
payload: {
id,
updates: {
versionNumber: metadata.versionNumber,
originalFileId: metadata.originalFileId,
parentFileId: metadata.parentFileId,
toolHistory: metadata.toolHistory
}
}
});
if (DEBUG) console.log(`📄 FileContext: Updated raw file ${file.name} with IndexedDB history data:`, {
versionNumber: metadata.versionNumber,
originalFileId: metadata.originalFileId,
toolChainLength: metadata.toolHistory?.length || 0
});
}
} catch (error) {
console.error('Failed to persist file to IndexedDB:', file.name, error);
}
}));
}
return addedFilesWithIds.map(({ file, id }) => createStirlingFile(file, id));
}, [indexedDB, enablePersistence]);
const addStirlingFileStubsAction = useCallback(async (stirlingFileStubs: StirlingFileStub[], options?: { insertAfterPageId?: string; selectFiles?: boolean }): Promise<StirlingFile[]> => { const addStirlingFileStubsAction = useCallback(async (stirlingFileStubs: StirlingFileStub[], options?: { insertAfterPageId?: string; selectFiles?: boolean }): Promise<StirlingFile[]> => {
// StirlingFileStubs preserve all metadata - perfect for FileManager use case! // StirlingFileStubs preserve all metadata - perfect for FileManager use case!
@ -130,12 +96,7 @@ function FileContextInner({
// Auto-select the newly added files if requested // Auto-select the newly added files if requested
if (options?.selectFiles && result.length > 0) { if (options?.selectFiles && result.length > 0) {
// Convert StirlingFile[] to AddedFile[] format for selectFiles selectFiles(result);
const addedFilesWithIds = result.map(stirlingFile => ({
file: stirlingFile,
id: stirlingFile.fileId
}));
selectFiles(addedFilesWithIds);
} }
return result; return result;

19
frontend/src/contexts/IndexedDBContext.tsx
View File

@ -64,7 +64,24 @@ export function IndexedDBProvider({ children }: IndexedDBProviderProps) {
// Store in IndexedDB (no history data - that's handled by direct fileStorage calls now) // Store in IndexedDB (no history data - that's handled by direct fileStorage calls now)
const stirlingFile = createStirlingFile(file, fileId); const stirlingFile = createStirlingFile(file, fileId);
await fileStorage.storeStirlingFile(stirlingFile, thumbnail, true);
// Create minimal stub for storage
const stub: StirlingFileStub = {
id: fileId,
name: file.name,
size: file.size,
type: file.type,
lastModified: file.lastModified,
quickKey: `${file.name}|${file.size}|${file.lastModified}`,
thumbnailUrl: thumbnail,
isLeaf: true,
createdAt: Date.now(),
versionNumber: 1,
originalFileId: fileId,
toolHistory: []
};
await fileStorage.storeStirlingFile(stirlingFile, stub);
const storedFile = await fileStorage.getStirlingFileStub(fileId); const storedFile = await fileStorage.getStirlingFileStub(fileId);
// Cache the file object for immediate reuse // Cache the file object for immediate reuse

175
frontend/src/contexts/file/fileActions.ts
View File

@ -8,7 +8,8 @@ import {
FileContextState, FileContextState,
toStirlingFileStub, toStirlingFileStub,
createFileId, createFileId,
createQuickKey createQuickKey,
createStirlingFile,
} from '../../types/fileContext'; } from '../../types/fileContext';
import { FileId } from '../../types/file'; import { FileId } from '../../types/file';
import { generateThumbnailWithMetadata } from '../../utils/thumbnailUtils'; import { generateThumbnailWithMetadata } from '../../utils/thumbnailUtils';
@ -16,7 +17,6 @@ import { FileLifecycleManager } from './lifecycle';
import { buildQuickKeySet } from './fileSelectors'; import { buildQuickKeySet } from './fileSelectors';
import { StirlingFile } from '../../types/fileContext'; import { StirlingFile } from '../../types/fileContext';
import { fileStorage } from '../../services/fileStorage'; import { fileStorage } from '../../services/fileStorage';
const DEBUG = process.env.NODE_ENV === 'development'; const DEBUG = process.env.NODE_ENV === 'development';
/** /**
@ -120,13 +120,7 @@ export function createChildStub(
}; };
} }
/**
* File addition types
*/
type AddFileKind = 'raw' | 'processed';
interface AddFileOptions { interface AddFileOptions {
// For 'raw' files
files?: File[]; files?: File[];
// For 'processed' files // For 'processed' files
@ -139,12 +133,6 @@ interface AddFileOptions {
selectFiles?: boolean; selectFiles?: boolean;
} }
export interface AddedFile {
file: File;
id: FileId;
thumbnail?: string;
}
/** /**
* Unified file addition helper - replaces addFiles * Unified file addition helper - replaces addFiles
*/ */
@ -153,95 +141,114 @@ export async function addFiles(
stateRef: React.MutableRefObject<FileContextState>, stateRef: React.MutableRefObject<FileContextState>,
filesRef: React.MutableRefObject<Map<FileId, File>>, filesRef: React.MutableRefObject<Map<FileId, File>>,
dispatch: React.Dispatch<FileContextAction>, dispatch: React.Dispatch<FileContextAction>,
lifecycleManager: FileLifecycleManager lifecycleManager: FileLifecycleManager,
): Promise<AddedFile[]> { enablePersistence: boolean = false
): Promise<StirlingFile[]> {
// Acquire mutex to prevent race conditions // Acquire mutex to prevent race conditions
await addFilesMutex.lock(); await addFilesMutex.lock();
try { try {
const stirlingFileStubs: StirlingFileStub[] = []; const stirlingFileStubs: StirlingFileStub[] = [];
const addedFiles: AddedFile[] = []; const stirlingFiles: StirlingFile[] = [];
// Build quickKey lookup from existing files for deduplication // Build quickKey lookup from existing files for deduplication
const existingQuickKeys = buildQuickKeySet(stateRef.current.files.byId); const existingQuickKeys = buildQuickKeySet(stateRef.current.files.byId);
const { files = [] } = options; const { files = [] } = options;
if (DEBUG) console.log(`📄 addFiles(raw): Adding ${files.length} files with immediate thumbnail generation`); if (DEBUG) console.log(`📄 addFiles(raw): Adding ${files.length} files with immediate thumbnail generation`);
for (const file of files) { for (const file of files) {
const quickKey = createQuickKey(file); const quickKey = createQuickKey(file);
// Soft deduplication: Check if file already exists by metadata // Soft deduplication: Check if file already exists by metadata
if (existingQuickKeys.has(quickKey)) { if (existingQuickKeys.has(quickKey)) {
if (DEBUG) console.log(`📄 Skipping duplicate file: ${file.name} (quickKey: ${quickKey})`); if (DEBUG) console.log(`📄 Skipping duplicate file: ${file.name} (quickKey: ${quickKey})`);
continue; continue;
} }
if (DEBUG) console.log(`📄 Adding new file: ${file.name} (quickKey: ${quickKey})`); if (DEBUG) console.log(`📄 Adding new file: ${file.name} (quickKey: ${quickKey})`);
const fileId = createFileId(); const fileId = createFileId();
filesRef.current.set(fileId, file); filesRef.current.set(fileId, file);
// Generate thumbnail and page count immediately // Generate thumbnail and page count immediately
let thumbnail: string | undefined; let thumbnail: string | undefined;
let pageCount: number = 1; let pageCount: number = 1;
// Route based on file type - PDFs through full metadata pipeline, non-PDFs through simple path // Route based on file type - PDFs through full metadata pipeline, non-PDFs through simple path
if (file.type.startsWith('application/pdf')) { if (file.type.startsWith('application/pdf')) {
try { try {
if (DEBUG) console.log(`📄 Generating PDF metadata for ${file.name}`); if (DEBUG) console.log(`📄 Generating PDF metadata for ${file.name}`);
const result = await generateThumbnailWithMetadata(file); const result = await generateThumbnailWithMetadata(file);
thumbnail = result.thumbnail; thumbnail = result.thumbnail;
pageCount = result.pageCount; pageCount = result.pageCount;
if (DEBUG) console.log(`📄 Generated PDF metadata for ${file.name}: ${pageCount} pages, thumbnail: SUCCESS`); if (DEBUG) console.log(`📄 Generated PDF metadata for ${file.name}: ${pageCount} pages, thumbnail: SUCCESS`);
} catch (error) { } catch (error) {
if (DEBUG) console.warn(`📄 Failed to generate PDF metadata for ${file.name}:`, error); if (DEBUG) console.warn(`📄 Failed to generate PDF metadata for ${file.name}:`, error);
} }
} else { } else {
// Non-PDF files: simple thumbnail generation, no page count // Non-PDF files: simple thumbnail generation, no page count
try { try {
if (DEBUG) console.log(`📄 Generating simple thumbnail for non-PDF file ${file.name}`); if (DEBUG) console.log(`📄 Generating simple thumbnail for non-PDF file ${file.name}`);
const { generateThumbnailForFile } = await import('../../utils/thumbnailUtils'); const { generateThumbnailForFile } = await import('../../utils/thumbnailUtils');
thumbnail = await generateThumbnailForFile(file); thumbnail = await generateThumbnailForFile(file);
pageCount = 0; // Non-PDFs have no page count pageCount = 0; // Non-PDFs have no page count
if (DEBUG) console.log(`📄 Generated simple thumbnail for ${file.name}: no page count, thumbnail: SUCCESS`); if (DEBUG) console.log(`📄 Generated simple thumbnail for ${file.name}: no page count, thumbnail: SUCCESS`);
} catch (error) { } catch (error) {
if (DEBUG) console.warn(`📄 Failed to generate simple thumbnail for ${file.name}:`, error); if (DEBUG) console.warn(`📄 Failed to generate simple thumbnail for ${file.name}:`, error);
} }
} }
// Create record with immediate thumbnail and page metadata // Create record with immediate thumbnail and page metadata
const record = toStirlingFileStub(file, fileId, thumbnail); const record = toStirlingFileStub(file, fileId, thumbnail);
if (thumbnail) { if (thumbnail) {
// Track blob URLs for cleanup (images return blob URLs that need revocation) // Track blob URLs for cleanup (images return blob URLs that need revocation)
if (thumbnail.startsWith('blob:')) { if (thumbnail.startsWith('blob:')) {
lifecycleManager.trackBlobUrl(thumbnail); lifecycleManager.trackBlobUrl(thumbnail);
} }
} }
// Store insertion position if provided // Store insertion position if provided
if (options.insertAfterPageId !== undefined) { if (options.insertAfterPageId !== undefined) {
record.insertAfterPageId = options.insertAfterPageId; record.insertAfterPageId = options.insertAfterPageId;
} }
// Create initial processedFile metadata with page count // Create initial processedFile metadata with page count
if (pageCount > 0) { if (pageCount > 0) {
record.processedFile = createProcessedFile(pageCount, thumbnail); record.processedFile = createProcessedFile(pageCount, thumbnail);
if (DEBUG) console.log(`📄 addFiles(raw): Created initial processedFile metadata for ${file.name} with ${pageCount} pages`); if (DEBUG) console.log(`📄 addFiles(raw): Created initial processedFile metadata for ${file.name} with ${pageCount} pages`);
} }
// History metadata is now managed in IndexedDB, not in PDF metadata existingQuickKeys.add(quickKey);
stirlingFileStubs.push(record);
existingQuickKeys.add(quickKey); // Create StirlingFile directly
stirlingFileStubs.push(record); const stirlingFile = createStirlingFile(file, fileId);
addedFiles.push({ file, id: fileId, thumbnail }); stirlingFiles.push(stirlingFile);
} }
// Persist to storage if enabled using fileStorage service
if (enablePersistence && stirlingFiles.length > 0) {
await Promise.all(stirlingFiles.map(async (stirlingFile, index) => {
try {
// Get corresponding stub with all metadata
const fileStub = stirlingFileStubs[index];
// Store using the cleaner signature - pass StirlingFile + StirlingFileStub directly
await fileStorage.storeStirlingFile(stirlingFile, fileStub);
if (DEBUG) console.log(`📄 addFiles: Stored file ${stirlingFile.name} with metadata:`, fileStub);
} catch (error) {
console.error('Failed to persist file to storage:', stirlingFile.name, error);
}
}));
}
// Dispatch ADD_FILES action if we have new files // Dispatch ADD_FILES action if we have new files
if (stirlingFileStubs.length > 0) { if (stirlingFileStubs.length > 0) {
dispatch({ type: 'ADD_FILES', payload: { stirlingFileStubs } }); dispatch({ type: 'ADD_FILES', payload: { stirlingFileStubs } });
} }
return addedFiles; return stirlingFiles;
} finally { } finally {
// Always release mutex even if error occurs // Always release mutex even if error occurs
addFilesMutex.unlock(); addFilesMutex.unlock();
@ -300,17 +307,7 @@ export async function consumeFiles(
try { try {
// Use fileStorage directly with complete metadata from stub // Use fileStorage directly with complete metadata from stub
await fileStorage.storeStirlingFile( await fileStorage.storeStirlingFile(stirlingFile, stub);
stirlingFile,
stub.thumbnailUrl,
true, // isLeaf - new files are leaf nodes
{
versionNumber: stub.versionNumber || 1,
originalFileId: stub.originalFileId || stub.id,
parentFileId: stub.parentFileId,
toolHistory: stub.toolHistory || []
}
);
if (DEBUG) console.log(`📄 Saved StirlingFile ${stirlingFile.name} directly to storage with complete metadata:`, { if (DEBUG) console.log(`📄 Saved StirlingFile ${stirlingFile.name} directly to storage with complete metadata:`, {
fileId: stirlingFile.fileId, fileId: stirlingFile.fileId,

31
frontend/src/services/fileStorage.ts
View File

@ -39,22 +39,9 @@ class FileStorageService {
} }
/** /**
* Store a StirlingFile with its metadata * Store a StirlingFile with its metadata from StirlingFileStub
*/ */
async storeStirlingFile( async storeStirlingFile(stirlingFile: StirlingFile, stub: StirlingFileStub): Promise<void> {
stirlingFile: StirlingFile,
thumbnail?: string,
isLeaf: boolean = true,
historyData?: {
versionNumber: number;
originalFileId: string;
parentFileId: FileId | undefined;
toolHistory: Array<{
toolName: string;
timestamp: number;
}>;
}
): Promise<void> {
const db = await this.getDatabase(); const db = await this.getDatabase();
const arrayBuffer = await stirlingFile.arrayBuffer(); const arrayBuffer = await stirlingFile.arrayBuffer();
@ -67,14 +54,14 @@ class FileStorageService {
size: stirlingFile.size, size: stirlingFile.size,
lastModified: stirlingFile.lastModified, lastModified: stirlingFile.lastModified,
data: arrayBuffer, data: arrayBuffer,
thumbnail, thumbnail: stub.thumbnailUrl,
isLeaf, isLeaf: stub.isLeaf ?? true,
// History data - use provided data or defaults for original files // History data from stub
versionNumber: historyData?.versionNumber ?? 1, versionNumber: stub.versionNumber ?? 1,
originalFileId: historyData?.originalFileId ?? stirlingFile.fileId, originalFileId: stub.originalFileId ?? stirlingFile.fileId,
parentFileId: historyData?.parentFileId ?? undefined, parentFileId: stub.parentFileId ?? undefined,
toolHistory: historyData?.toolHistory ?? [] toolHistory: stub.toolHistory ?? []
}; };
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {