sqlite

src/providers/sqlite3/index.js

@@ -1,15 +1,29 @@
 /**
- * Provider for a caller-owned `better-sqlite3` Database instance.
+ * Provider for `better-sqlite3` Database instances or connection options.
  *
  * The driver is synchronous, but this adapter exposes the asynchronous
  * provider contract used by LibDashStore.
  */
 
+import Database from 'better-sqlite3';
+import { SCHEMA_SCRIPTS } from './schema.js';
+
+function normalizeParamValue(value) {
+  if (value === undefined || value === null) return null;
+  if (value instanceof Date) {
+    return isNaN(value.getTime()) ? null : value.toISOString();
+  }
+  if (typeof value === 'boolean') {
+    return value ? 1 : 0;
+  }
+  return value;
+}
+
 function bindParams(params) {
   return Object.fromEntries(
     Object.entries(params).map(([key, value]) => [
       key.replace(/^[@:$]/, ''),
-      value === undefined ? null : value,
+      normalizeParamValue(value),
     ]),
   );
 }
@@ -20,10 +34,169 @@ function normalizeInsertId(value) {
   return value <= BigInt(Number.MAX_SAFE_INTEGER) ? Number(value) : value;
 }
 
+import { dirname, join } from 'node:path';
+import { mkdirSync, mkdtempSync, promises as fsPromises } from 'node:fs';
+import { tmpdir } from 'node:os';
+
+export function extFromMimetype(mimetype) {
+  if (!mimetype) return '';
+  const type = mimetype.trim().toLowerCase().split(';')[0];
+  const map = {
+    'application/pdf': '.pdf',
+    'text/plain': '.txt',
+    'text/csv': '.csv',
+    'text/html': '.html',
+    'text/markdown': '.md',
+    'image/avif': '.avif',
+    'image/png': '.png',
+    'image/jpeg': '.jpg',
+    'image/jpg': '.jpg',
+    'image/webp': '.webp',
+    'image/gif': '.gif',
+    'image/svg+xml': '.svg',
+    'application/json': '.json',
+    'application/zip': '.zip',
+    'application/xml': '.xml',
+    'application/octet-stream': '.bin',
+  };
+  if (map[type]) return map[type];
+  if (type.startsWith('image/')) return '.' + type.slice(6).replace('+xml', '');
+  if (type.startsWith('text/')) return '.' + type.slice(5);
+  return '';
+}
+
+async function safeWrite(filePath, buffer) {
+  if (buffer == null) return;
+  const buf = Buffer.isBuffer(buffer) ? buffer : Buffer.from(buffer);
+  await fsPromises.writeFile(filePath, buf);
+}
+
+async function findFileByHash(dir, hash, ext = '') {
+  if (!dir || !hash) return null;
+  if (ext) {
+    const candidate = join(dir, hash + ext);
+    try {
+      await fsPromises.access(candidate);
+      return candidate;
+    } catch {
+      // fallback
+    }
+  }
+  const candidateNoExt = join(dir, hash);
+  try {
+    await fsPromises.access(candidateNoExt);
+    return candidateNoExt;
+  } catch {
+    // fallback
+  }
+  try {
+    const files = await fsPromises.readdir(dir);
+    const match = files.find((f) => f === hash || f.startsWith(hash + '.'));
+    if (match) return join(dir, match);
+  } catch {
+    // ignore
+  }
+  return null;
+}
+
+async function safeReadByHash(dir, hash, ext = '') {
+  const filePath = await findFileByHash(dir, hash, ext);
+  if (!filePath) return null;
+  try {
+    return await fsPromises.readFile(filePath);
+  } catch (err) {
+    if (err.code === 'ENOENT') return null;
+    throw err;
+  }
+}
+
+async function safeDeleteByHash(dir, hash) {
+  if (!dir || !hash) return;
+  try {
+    const files = await fsPromises.readdir(dir);
+    const matches = files.filter((f) => f === hash || f.startsWith(hash + '.'));
+    await Promise.all(matches.map((f) => fsPromises.unlink(join(dir, f))));
+  } catch (err) {
+    if (err.code !== 'ENOENT') throw err;
+  }
+}
+
 export class BetterSqlite3Executor {
-  /** @param {import('better-sqlite3').Database} db */
-  constructor(db) {
+  /**
+   * @param {import('better-sqlite3').Database} db
+   * @param {string|null} [storageDir=null]
+   */
+  constructor(db, storageDir = null) {
     this.db = db;
+    this.dialect = 'sqlite3';
+    this.storageDir = storageDir;
+  }
+
+  async writeFileContent(hash, buffer, mimetype = null) {
+    if (!this.storageDir || !hash) return;
+    const ext = extFromMimetype(mimetype);
+    await safeWrite(join(this.storageDir, 'content', hash + ext), buffer);
+  }
+
+  async readFileContent(hash, mimetype = null) {
+    if (!this.storageDir || !hash) return null;
+    const ext = extFromMimetype(mimetype);
+    return safeReadByHash(join(this.storageDir, 'content'), hash, ext);
+  }
+
+  async writeThumbAvif(hash, buffer) {
+    if (!this.storageDir || !hash) return;
+    await safeWrite(join(this.storageDir, 'thumbavif', hash + '.avif'), buffer);
+  }
+
+  async readThumbAvif(hash) {
+    if (!this.storageDir || !hash) return null;
+    return safeReadByHash(join(this.storageDir, 'thumbavif'), hash, '.avif');
+  }
+
+  async writePicFull(hash, buffer) {
+    if (!this.storageDir || !hash) return;
+    await safeWrite(join(this.storageDir, 'fullbinary', hash + '.avif'), buffer);
+  }
+
+  async readPicFull(hash) {
+    if (!this.storageDir || !hash) return null;
+    return safeReadByHash(join(this.storageDir, 'fullbinary'), hash, '.avif');
+  }
+
+  async writePicThumb(hash, buffer) {
+    if (!this.storageDir || !hash) return;
+    await safeWrite(join(this.storageDir, 'thumbbinary', hash + '.avif'), buffer);
+  }
+
+  async readPicThumb(hash) {
+    if (!this.storageDir || !hash) return null;
+    return safeReadByHash(join(this.storageDir, 'thumbbinary'), hash, '.avif');
+  }
+
+  async deleteFile(hash) {
+    if (!this.storageDir || !hash) return;
+    await safeDeleteByHash(join(this.storageDir, 'content'), hash);
+    await safeDeleteByHash(join(this.storageDir, 'thumbavif'), hash);
+  }
+
+  async deletePic(hash) {
+    if (!this.storageDir || !hash) return;
+    await safeDeleteByHash(join(this.storageDir, 'fullbinary'), hash);
+    await safeDeleteByHash(join(this.storageDir, 'thumbbinary'), hash);
+  }
+
+  async purgeFiles() {
+    if (!this.storageDir) return;
+    for (const sub of ['content', 'thumbavif', 'fullbinary', 'thumbbinary']) {
+      const dir = join(this.storageDir, sub);
+      try {
+        const files = await fsPromises.readdir(dir);
+        await Promise.all(files.map((f) => fsPromises.unlink(join(dir, f))));
+      } catch {
+        // ignore
+      }
+    }
   }
 
   /** @returns {Promise<import('../provider.js').QueryResult>} */
@@ -49,11 +222,47 @@ export class BetterSqlite3Executor {
   }
 }
 
+function ensureDirectory(filename) {
+  if (typeof filename === 'string' && filename !== ':memory:' && !filename.startsWith(':memory:')) {
+    const dir = dirname(filename);
+    if (dir && dir !== '.') {
+      mkdirSync(dir, { recursive: true });
+    }
+  }
+}
+
 export class BetterSqlite3Provider extends BetterSqlite3Executor {
   /**
-   * @param {import('better-sqlite3').Database} db An open, caller-owned database.
+   * @param {import('better-sqlite3').Database | string | { filename: string, storageDir?: string, filesDir?: string, [key: string]: any }} [dbOrConfig]
+   *   An open better-sqlite3 Database, a file path (or ':memory:'), or a config object.
    */
-  constructor(db) {
+  constructor(dbOrConfig = ':memory:') {
+    let db = dbOrConfig;
+    let storageDir = null;
+
+    if (typeof dbOrConfig === 'string') {
+      ensureDirectory(dbOrConfig);
+      db = new Database(dbOrConfig);
+      storageDir = (dbOrConfig === ':memory:' || dbOrConfig.startsWith(':memory:'))
+        ? mkdtempSync(join(tmpdir(), 'dash-sqlite-storage-'))
+        : join(dirname(dbOrConfig), 'storage');
+    } else if (
+      dbOrConfig &&
+      typeof dbOrConfig === 'object' &&
+      typeof dbOrConfig.prepare !== 'function'
+    ) {
+      const { filename, storageDir: customStorageDir, filesDir, ...options } = dbOrConfig;
+      if (filename) ensureDirectory(filename);
+      db = new Database(filename || ':memory:', options);
+      storageDir = customStorageDir || filesDir || (
+        (!filename || filename === ':memory:' || filename.startsWith(':memory:'))
+          ? mkdtempSync(join(tmpdir(), 'dash-sqlite-storage-'))
+          : join(dirname(filename), 'storage')
+      );
+    } else if (db && typeof db.prepare === 'function') {
+      storageDir = mkdtempSync(join(tmpdir(), 'dash-sqlite-storage-'));
+    }
+
     if (
       !db ||
       typeof db.prepare !== 'function' ||
@@ -61,22 +270,29 @@ export class BetterSqlite3Provider extends BetterSqlite3Executor {
       typeof db.pragma !== 'function'
     ) {
       throw new TypeError(
-        'BetterSqlite3Provider requires an open better-sqlite3 Database',
+        'BetterSqlite3Provider requires an open better-sqlite3 Database or a filename string',
       );
     }
 
-    super(db);
-    this.dialect = 'sqlite3';
+    super(db, storageDir);
+    if (this.storageDir) {
+      for (const sub of ['content', 'thumbavif', 'fullbinary', 'thumbbinary']) {
+        mkdirSync(join(this.storageDir, sub), { recursive: true });
+      }
+    }
+    this.pool = this;
     db.pragma('foreign_keys = ON');
     db.pragma('journal_mode = WAL');
     db.pragma('busy_timeout = 5000');
   }
 
   #transactionTail = Promise.resolve();
+  #transactionDepth = 0;
 
   /**
    * Runs `fn(runner)` in an immediate write transaction. Transactions are
    * serialized because better-sqlite3 has one synchronous connection.
+   * Nested calls use SAVEPOINTs.
    *
    * @template T
    * @param {(runner: BetterSqlite3Executor) => Promise<T> | T} fn
@@ -87,15 +303,66 @@ export class BetterSqlite3Provider extends BetterSqlite3Executor {
       return Promise.reject(new TypeError('transaction requires a callback'));
     }
 
+    const bindRunner = (executor) => {
+      const runner = async (sql, params) => executor.query(sql, params);
+      runner.query = runner;
+      runner.dialect = 'sqlite3';
+      runner.storageDir = this.storageDir;
+      runner.writeFileContent = executor.writeFileContent.bind(executor);
+      runner.readFileContent = executor.readFileContent.bind(executor);
+      runner.writeThumbAvif = executor.writeThumbAvif.bind(executor);
+      runner.readThumbAvif = executor.readThumbAvif.bind(executor);
+      runner.writePicFull = executor.writePicFull.bind(executor);
+      runner.readPicFull = executor.readPicFull.bind(executor);
+      runner.writePicThumb = executor.writePicThumb.bind(executor);
+      runner.readPicThumb = executor.readPicThumb.bind(executor);
+      runner.deleteFile = executor.deleteFile.bind(executor);
+      runner.deletePic = executor.deletePic.bind(executor);
+      runner.purgeFiles = executor.purgeFiles.bind(executor);
+      return runner;
+    };
+
+    if (this.#transactionDepth > 0 || this.db.inTransaction) {
+      const savepoint = `sp_${++this.#transactionDepth}`;
+      this.db.exec(`SAVEPOINT ${savepoint}`);
+      const executor = new BetterSqlite3Executor(this.db, this.storageDir);
+      const runner = bindRunner(executor);
+
+      return Promise.resolve()
+        .then(() => fn(runner, executor))
+        .then(
+          (result) => {
+            this.db.exec(`RELEASE SAVEPOINT ${savepoint}`);
+            this.#transactionDepth--;
+            return result;
+          },
+          (err) => {
+            try {
+              this.db.exec(`ROLLBACK TO SAVEPOINT ${savepoint}`);
+              this.db.exec(`RELEASE SAVEPOINT ${savepoint}`);
+            } catch {
+              // ignore
+            }
+            this.#transactionDepth--;
+            throw err;
+          },
+        );
+    }
+
     const run = async () => {
+      this.#transactionDepth = 1;
       this.db.exec('BEGIN IMMEDIATE');
       try {
-        const result = await fn(new BetterSqlite3Executor(this.db));
+        const executor = new BetterSqlite3Executor(this.db, this.storageDir);
+        const runner = bindRunner(executor);
+        const result = await fn(runner, executor);
         this.db.exec('COMMIT');
         return result;
       } catch (err) {
         if (this.db.inTransaction) this.db.exec('ROLLBACK');
         throw err;
+      } finally {
+        this.#transactionDepth = 0;
       }
     };
 
@@ -108,18 +375,67 @@ export class BetterSqlite3Provider extends BetterSqlite3Executor {
     throw new Error('Stored procedures are not supported by better-sqlite3');
   }
 
+  /**
+   * Initializes the native SQLite schema.
+   * @returns {Promise<void>}
+   */
   async initialize() {
-    // SQLite migrations are provided by the SQLite schema layer.
+    for (const script of SCHEMA_SCRIPTS) {
+      if (script && script.trim()) {
+        this.db.exec(script);
+      }
+    }
   }
 
+  /**
+   * Purges all tables, views, and triggers from the SQLite database.
+   * @returns {Promise<void>}
+   */
   async purge() {
-    // SQLite schema teardown is provided by the SQLite schema layer.
+    await this.purgeFiles();
+    this.db.pragma('foreign_keys = OFF');
+    try {
+      const tables = this.db
+        .prepare(
+          "SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'",
+        )
+        .all()
+        .map((r) => r.name);
+      const views = this.db
+        .prepare("SELECT name FROM sqlite_master WHERE type = 'view'")
+        .all()
+        .map((r) => r.name);
+      const triggers = this.db
+        .prepare("SELECT name FROM sqlite_master WHERE type = 'trigger'")
+        .all()
+        .map((r) => r.name);
+
+      for (const trigger of triggers) {
+        this.db.exec(`DROP TRIGGER IF EXISTS "${trigger}"`);
+      }
+      for (const view of views) {
+        this.db.exec(`DROP VIEW IF EXISTS "${view}"`);
+      }
+      for (const table of tables) {
+        this.db.exec(`DROP TABLE IF EXISTS "${table}"`);
+      }
+    } finally {
+      this.db.pragma('foreign_keys = ON');
+    }
   }
 
-  /** Explicitly closes the caller-owned database. It is never closed implicitly. */
+  /** Explicitly closes the database. */
   close() {
     this.db.close();
   }
+
+  isUniqueConstraintError(err) {
+    return isUniqueConstraintError(err);
+  }
+
+  isForeignKeyConstraintError(err) {
+    return isForeignKeyConstraintError(err);
+  }
 }
 
 export function isUniqueConstraintError(err) {
@@ -131,4 +447,7 @@ export function isUniqueConstraintError(err) {
 
 export function isForeignKeyConstraintError(err) {
   return err?.code === 'SQLITE_CONSTRAINT_FOREIGNKEY';
-}
+}
+
+export { BetterSqlite3Provider as SqliteProvider };
+export { BetterSqlite3Executor as SqliteExecutor };

src/providers/sqlite3/queries/category.js

@@ -5,10 +5,60 @@ import { ARTICLE_LIST_COLUMNS, ARTICLE_LIST_JOINS } from './article.js';
 const queries = translateQueries(source, 'category');
 export const {
   CREATE, DELETE, GET_BY_PARENT_AND_NAME,
-  GET_NEXT_SIBLING_SORT, IS_ANCESTOR, LIST_CHILD_IDS,
-  MOVE, RESOLVE_CATEGORY_ACCESS, UPDATE_SORT,
+  GET_NEXT_SIBLING_SORT, LIST_CHILD_IDS,
+  MOVE, UPDATE_SORT,
 } = queries;
 
+export const RESOLVE_CATEGORY_ACCESS = `
+WITH RECURSIVE ancestors AS (
+    SELECT id, parent_id
+    FROM category
+    WHERE id = @categoryId
+
+    UNION ALL
+
+    SELECT c.id, c.parent_id
+    FROM category c
+    INNER JOIN ancestors a ON c.id = a.parent_id
+),
+root AS (
+    SELECT id AS rootCategoryId
+    FROM ancestors
+    WHERE parent_id IS NULL
+    LIMIT 1
+)
+SELECT
+    p.id AS categoryId,
+    r.rootCategoryId,
+    co.id AS companyId,
+    co.default_language_id AS languageId,
+    co.poscustomergroup_id AS posCustomerGroupId,
+    cu.id AS companyuserId
+FROM category p
+LEFT JOIN root r ON 1 = 1
+LEFT JOIN company co ON co.root_category_id = r.rootCategoryId
+LEFT JOIN companyuser cu
+    ON cu.company_id = co.id AND cu.google_auth_id = @googleAuthId
+WHERE p.id = @categoryId
+`;
+
+export const IS_ANCESTOR = `
+WITH RECURSIVE ancestors AS (
+    SELECT id, parent_id
+    FROM category
+    WHERE id = @parentId
+
+    UNION ALL
+
+    SELECT c.id, c.parent_id
+    FROM category c
+    INNER JOIN ancestors a ON c.id = a.parent_id
+)
+SELECT id
+FROM ancestors
+WHERE id = @id
+`;
+
 export const GET_CATEGORY_ARTICLES = `
 SELECT ${ARTICLE_LIST_COLUMNS}
 FROM categoryarticle ca

src/providers/sqlite3/queries/delivery.js

@@ -5,18 +5,41 @@ const queries = translateQueries(source, 'delivery');
 export const {
   CREATE_ATTACHMENT, CREATE_MANIFEST, DELETE_LINES_BY_MANIFEST,
   DELETE_MANIFEST, DELIVERY_TYPE, GET_DELIVERY_HEADER, GET_DELIVERY_LINES,
-  GET_NEXT_ATTACHMENT_SORT, LIST_DELIVERIES, LIST_DELIVERY_FILES,
+  GET_NEXT_ATTACHMENT_SORT, LIST_DELIVERIES,
   UPDATE_MANIFEST,
 } = queries;
 
 export const CREATE_FILE = `
 INSERT INTO "file" (
-  hash, mimetype, name, size, created, modified, content,
-  thumbavif, thumbx, thumby, meta
+  hash, mimetype, name, size, created, modified,
+  thumbx, thumby, meta
 )
 VALUES (
-  @hash, @mimetype, @name, @size, @created, @modified, @content,
-  @thumbavif, @thumbx, @thumby, @meta
+  @hash, @mimetype, @name, @size, @created, @modified,
+  @thumbx, @thumby, @meta
 )
 RETURNING id
+`;
+
+export const LIST_DELIVERY_FILES = `
+SELECT
+    ma.id,
+    ma.manifest_id AS manifestId,
+    ma.file_id AS fileId,
+    ma.sort,
+    f.hash,
+    f.mimetype,
+    f.name,
+    f.size,
+    f.created,
+    f.modified,
+    f.thumbx,
+    f.thumby,
+    f.meta
+FROM manifestattachment ma
+INNER JOIN "file" f ON f.id = ma.file_id
+INNER JOIN manifest m ON m.id = ma.manifest_id
+WHERE m.company_id = @companyId
+  AND m."type" = @type
+ORDER BY ma.manifest_id, ma.sort ASC, ma.id ASC
 `;

src/providers/sqlite3/queries/manifestattachment.js

@@ -4,18 +4,68 @@ import { translateQueries } from './translate.js';
 const queries = translateQueries(source, 'manifestattachment');
 export const {
   CHECK_MANIFEST_ACCESS, CREATE_ATTACHMENT, DELETE_ATTACHMENT,
-  DELETE_FILE, FIND_ATTACHMENT, GET_BY_ID, GET_NEXT_SORT, LIST_BY_MANIFEST,
+  DELETE_FILE, GET_NEXT_SORT,
   LIST_IDS_BY_MANIFEST, SET_TEMP_SORTS, UPDATE_SORT,
 } = queries;
 
+export const FIND_ATTACHMENT = `
+SELECT ma.id, ma.file_id AS fileId, f.hash
+FROM manifestattachment ma
+INNER JOIN "file" f ON f.id = ma.file_id
+WHERE ma.manifest_id = @manifestId
+  AND (ma.id = @id OR ma.file_id = @id)
+`;
+
 export const CREATE_FILE = `
 INSERT INTO "file" (
-  hash, mimetype, name, size, created, modified, content,
-  thumbavif, thumbx, thumby, meta
+  hash, mimetype, name, size, created, modified,
+  thumbx, thumby, meta
 )
 VALUES (
-  @hash, @mimetype, @name, @size, @created, @modified, @content,
-  @thumbavif, @thumbx, @thumby, @meta
+  @hash, @mimetype, @name, @size, @created, @modified,
+  @thumbx, @thumby, @meta
 )
 RETURNING id
+`;
+
+export const LIST_BY_MANIFEST = `
+SELECT 
+    ma.id,
+    ma.manifest_id AS manifestId,
+    ma.file_id AS fileId,
+    ma.sort,
+    f.hash,
+    f.mimetype,
+    f.name,
+    f.size,
+    f.created,
+    f.modified,
+    f.thumbx,
+    f.thumby,
+    f.meta
+FROM manifestattachment ma
+INNER JOIN "file" f ON f.id = ma.file_id
+WHERE ma.manifest_id = @manifestId
+ORDER BY ma.sort ASC, ma.id ASC
+`;
+
+export const GET_BY_ID = `
+SELECT 
+    ma.id,
+    ma.manifest_id AS manifestId,
+    ma.file_id AS fileId,
+    ma.sort,
+    f.hash,
+    f.mimetype,
+    f.name,
+    f.size,
+    f.created,
+    f.modified,
+    f.thumbx,
+    f.thumby,
+    f.meta
+FROM manifestattachment ma
+INNER JOIN "file" f ON f.id = ma.file_id
+WHERE ma.manifest_id = @manifestId
+  AND (ma.id = @id OR ma.file_id = @id)
 `;

src/providers/sqlite3/queries/pic.js

@@ -3,9 +3,31 @@ import { translateQueries } from './translate.js';
 
 const queries = translateQueries(source, 'pic');
 export const {
-  CREATE, GET_CATEGORY_ID_BY_HASH, GET_FULL_BY_HASH, GET_THUMB_BY_HASH,
+  GET_CATEGORY_ID_BY_HASH,
 } = queries;
 
+export const CREATE = `
+INSERT INTO pic (
+    w, h, size, hash, thumbw, thumbh, thumbhash, source
+)
+VALUES (
+    @w, @h, @size, @hash, @thumbw, @thumbh, @thumbhash, @source
+)
+RETURNING id
+`;
+
+export const GET_FULL_BY_HASH = `
+SELECT w, h, size, hash
+FROM pic
+WHERE hash = @hash
+`;
+
+export const GET_THUMB_BY_HASH = `
+SELECT thumbw AS w, thumbh AS h, hash
+FROM pic
+WHERE hash = @hash
+`;
+
 export const DELETE_BY_ARTICLE_AND_HASH = `
 DELETE FROM pic
 WHERE hash = @fullHash

src/providers/sqlite3/schema.js

@@ -42,9 +42,7 @@ CREATE TABLE IF NOT EXISTS pic (
     thumbw INTEGER NOT NULL,
     thumbh INTEGER NOT NULL,
     thumbhash TEXT NOT NULL,
-    source TEXT NOT NULL,
-    fullbinary BLOB NOT NULL,
-    thumbbinary BLOB NOT NULL
+    source TEXT NOT NULL
 );
 `;
 
@@ -542,8 +540,6 @@ CREATE TABLE IF NOT EXISTS "file" (
     size INTEGER NOT NULL,
     created TEXT NOT NULL DEFAULT (${UTC_NOW}),
     modified TEXT NOT NULL DEFAULT (${UTC_NOW}),
-    content BLOB NOT NULL,
-    thumbavif BLOB,
     thumbx INTEGER,
     thumby INTEGER,
     meta TEXT