sqlite
This commit is contained in:
seb
190
vfs.md
Normal file
190
vfs.md
Normal file
@@ -0,0 +1,190 @@
|
||||
Here is a complete design and implementation document for a block-based Virtual File System (VFS) in SQL Server. This incorporates the architectural fixes, concurrency safety, and performance optimizations discussed earlier.
|
||||
|
||||
SQL Server Virtual File System (VFS) Implementation Guide
|
||||
1. Architecture Overview
|
||||
This VFS architecture mimics a traditional inode-based disk filesystem inside a relational database. Instead of storing large files in a single VARBINARY(MAX) column—which causes severe memory pressure and transaction log bloat—files are divided into fixed-size chunks (blocks) at the application layer and streamed into the database.
|
||||
|
||||
Key Benefits:
|
||||
|
||||
Low Memory Footprint: Applications can stream gigabytes of data using small, fixed-size memory buffers (e.g., 1MB at a time).
|
||||
|
||||
Resumable Uploads: If a network failure occurs, the client only needs to resume from the last successfully written block.
|
||||
|
||||
Azure SQL Compatibility: Provides FILESTREAM-like chunking benefits in PaaS environments where native FILESTREAM is unsupported.
|
||||
|
||||
2. Database Schema
|
||||
The schema separates metadata (Nodes) from the binary payload (Blocks). It includes a soft-delete flag to prevent massive transaction log spikes when deleting huge files.
|
||||
|
||||
SQL
|
||||
-- Represents files, directories, and metadata
|
||||
CREATE TABLE dbo.Filesystem_Nodes (
|
||||
NodeID BIGINT IDENTITY(1,1) PRIMARY KEY,
|
||||
ParentNodeID BIGINT NULL FOREIGN KEY REFERENCES dbo.Filesystem_Nodes(NodeID),
|
||||
Name NVARCHAR(255) NOT NULL,
|
||||
IsDirectory BIT NOT NULL DEFAULT 0,
|
||||
ByteSize BIGINT NOT NULL DEFAULT 0,
|
||||
IsDeleted BIT NOT NULL DEFAULT 0, -- Soft-delete flag for garbage collection
|
||||
CreatedAt DATETIME2 NOT NULL DEFAULT SYSUTCDATETIME(),
|
||||
ModifiedAt DATETIME2 NOT NULL DEFAULT SYSUTCDATETIME(),
|
||||
|
||||
CONSTRAINT UQ_Parent_Name_NotDeleted UNIQUE (ParentNodeID, Name, IsDeleted)
|
||||
);
|
||||
|
||||
-- Index to quickly find files pending deletion
|
||||
CREATE NONCLUSTERED INDEX IX_Filesystem_Nodes_Deleted
|
||||
ON dbo.Filesystem_Nodes(IsDeleted)
|
||||
INCLUDE (NodeID)
|
||||
WHERE IsDeleted = 1;
|
||||
|
||||
-- Represents the raw binary chunks of each file
|
||||
CREATE TABLE dbo.Filesystem_Blocks (
|
||||
NodeID BIGINT NOT NULL,
|
||||
BlockIndex BIGINT NOT NULL,
|
||||
Data VARBINARY(MAX) NOT NULL,
|
||||
|
||||
-- Crucial: Clustered PK ensures contiguous disk storage for sequential reads
|
||||
CONSTRAINT PK_Filesystem_Blocks PRIMARY KEY CLUSTERED (NodeID, BlockIndex)
|
||||
);
|
||||
|
||||
-- Foreign key is NO ACTION because we handle cascading deletes via background jobs
|
||||
ALTER TABLE dbo.Filesystem_Blocks
|
||||
ADD CONSTRAINT FK_Blocks_Nodes FOREIGN KEY (NodeID)
|
||||
REFERENCES dbo.Filesystem_Nodes(NodeID);
|
||||
3. Core Stored Procedures
|
||||
A. Create a Node (File or Directory)
|
||||
Initializes the metadata record.
|
||||
|
||||
SQL
|
||||
CREATE OR ALTER PROCEDURE dbo.sp_FS_CreateNode
|
||||
@ParentNodeID BIGINT = NULL,
|
||||
@Name NVARCHAR(255),
|
||||
@IsDirectory BIT = 0,
|
||||
@NodeID BIGINT OUTPUT
|
||||
AS
|
||||
BEGIN
|
||||
SET NOCOUNT ON;
|
||||
|
||||
INSERT INTO dbo.Filesystem_Nodes (ParentNodeID, Name, IsDirectory, ByteSize)
|
||||
VALUES (@ParentNodeID, @Name, @IsDirectory, 0);
|
||||
|
||||
SET @NodeID = SCOPE_IDENTITY();
|
||||
END;
|
||||
B. Write a Block (Concurrency-Safe)
|
||||
Handles writing or overwriting a specific chunk. Uses an UPDATE / INSERT pattern to avoid MERGE deadlocks and correctly calculates total file size based on a standard block size.
|
||||
|
||||
SQL
|
||||
CREATE OR ALTER PROCEDURE dbo.sp_FS_WriteBlock
|
||||
@NodeID BIGINT,
|
||||
@BlockIndex BIGINT,
|
||||
@Data VARBINARY(MAX),
|
||||
@StandardBlockSize BIGINT = 1048576 -- Default 1MB blocks
|
||||
AS
|
||||
BEGIN
|
||||
SET NOCOUNT ON;
|
||||
|
||||
-- 1. Upsert the block safely
|
||||
BEGIN TRAN;
|
||||
UPDATE dbo.Filesystem_Blocks
|
||||
SET Data = @Data
|
||||
WHERE NodeID = @NodeID AND BlockIndex = @BlockIndex;
|
||||
|
||||
IF @@ROWCOUNT = 0
|
||||
BEGIN
|
||||
INSERT INTO dbo.Filesystem_Blocks (NodeID, BlockIndex, Data)
|
||||
VALUES (@NodeID, @BlockIndex, @Data);
|
||||
END
|
||||
COMMIT TRAN;
|
||||
|
||||
-- 2. Calculate the file's current max boundary
|
||||
-- (Previous blocks * Standard Size) + Size of current block
|
||||
DECLARE @CalculatedSize BIGINT = (@BlockIndex * @StandardBlockSize) + DATALENGTH(@Data);
|
||||
|
||||
-- 3. Expand file size if we wrote past the current known end
|
||||
UPDATE dbo.Filesystem_Nodes
|
||||
SET ByteSize = CASE WHEN @CalculatedSize > ByteSize THEN @CalculatedSize ELSE ByteSize END,
|
||||
ModifiedAt = SYSUTCDATETIME()
|
||||
WHERE NodeID = @NodeID;
|
||||
END;
|
||||
C. Read a Block
|
||||
Retrieves a specific chunk for streaming back to the client.
|
||||
|
||||
SQL
|
||||
CREATE OR ALTER PROCEDURE dbo.sp_FS_ReadBlock
|
||||
@NodeID BIGINT,
|
||||
@BlockIndex BIGINT
|
||||
AS
|
||||
BEGIN
|
||||
SET NOCOUNT ON;
|
||||
|
||||
SELECT
|
||||
BlockIndex,
|
||||
Data,
|
||||
DATALENGTH(Data) AS BlockByteSize
|
||||
FROM dbo.Filesystem_Blocks
|
||||
WHERE NodeID = @NodeID
|
||||
AND BlockIndex = @BlockIndex;
|
||||
END;
|
||||
D. Delete a File (Soft Delete)
|
||||
Marks the file as deleted to immediately hide it from the UI/Application, preventing long synchronous locks.
|
||||
|
||||
SQL
|
||||
CREATE OR ALTER PROCEDURE dbo.sp_FS_DeleteNode
|
||||
@NodeID BIGINT
|
||||
AS
|
||||
BEGIN
|
||||
SET NOCOUNT ON;
|
||||
|
||||
-- Soft delete the node. A background job will clean up the physical blocks.
|
||||
UPDATE dbo.Filesystem_Nodes
|
||||
SET IsDeleted = 1,
|
||||
ModifiedAt = SYSUTCDATETIME(),
|
||||
-- Scramble the name slightly to free up the unique constraint
|
||||
-- so a new file with the same name can be created immediately
|
||||
Name = Name + '_deleted_' + CAST(NEWID() AS NVARCHAR(36))
|
||||
WHERE NodeID = @NodeID;
|
||||
END;
|
||||
4. Background Maintenance (Garbage Collection)
|
||||
Because large files can contain tens of thousands of blocks, a single DELETE statement would lock the table and bloat the transaction log. This script should be scheduled via SQL Server Agent to run every few minutes to physically delete blocks in small, manageable batches.
|
||||
|
||||
SQL
|
||||
CREATE OR ALTER PROCEDURE dbo.sp_FS_GarbageCollection
|
||||
@BatchSize INT = 2000
|
||||
AS
|
||||
BEGIN
|
||||
SET NOCOUNT ON;
|
||||
DECLARE @DeletedNodes TABLE (NodeID BIGINT);
|
||||
DECLARE @RowsAffected INT = 1;
|
||||
|
||||
-- Get a list of nodes pending deletion
|
||||
INSERT INTO @DeletedNodes (NodeID)
|
||||
SELECT NodeID FROM dbo.Filesystem_Nodes WHERE IsDeleted = 1;
|
||||
|
||||
-- Delete blocks in small batches to protect the Transaction Log
|
||||
WHILE @RowsAffected > 0
|
||||
BEGIN
|
||||
DELETE TOP (@BatchSize) B
|
||||
FROM dbo.Filesystem_Blocks B
|
||||
INNER JOIN @DeletedNodes D ON B.NodeID = D.NodeID;
|
||||
|
||||
SET @RowsAffected = @@ROWCOUNT;
|
||||
|
||||
-- Brief pause to yield to production workloads
|
||||
IF @RowsAffected > 0 WAITFOR DELAY '00:00:00.100';
|
||||
END
|
||||
|
||||
-- Once all blocks are gone, delete the metadata nodes
|
||||
DELETE N
|
||||
FROM dbo.Filesystem_Nodes N
|
||||
INNER JOIN @DeletedNodes D ON N.NodeID = D.NodeID
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM dbo.Filesystem_Blocks B WHERE B.NodeID = N.NodeID
|
||||
);
|
||||
END;
|
||||
5. Application Layer Implementation Notes
|
||||
To use this database architecture successfully, the application layer (e.g., C#, Python, Node.js) must follow these rules:
|
||||
|
||||
Enforce Block Sizes: The application MUST slice incoming files into exact standard sizes (e.g., 1MB) for every block except the final block, which can be smaller.
|
||||
|
||||
Concurrency: Multiple threads can upload different blocks of the same file simultaneously (e.g., Thread A writes Block 0, Thread B writes Block 1) without locking each other out.
|
||||
|
||||
Read Streaming: To download a file, the application should query sp_FS_ReadBlock in a loop (incrementing BlockIndex from 0 upwards) and pipe the binary result directly to the HTTP Response stream until an empty result is returned.
|
||||
Reference in New Issue
Block a user