/* ** 2017-10-20 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: ** ** May you do good and not evil. ** May you find forgiveness for yourself and forgive others. ** May you share freely, never taking more than you give. ** ****************************************************************************** ** ** This file implements a VFS shim that allows an SQLite database to be ** appended onto the end of some other file, such as an executable. ** ** A special record must appear at the end of the file that identifies the ** file as an appended database and provides an offset to page 1. For ** best performance page 1 should be located at a disk page boundary, though ** that is not required. ** ** When opening a database using this VFS, the connection might treat ** the file as an ordinary SQLite database, or it might treat is as a ** database appended onto some other file. Here are the rules: ** ** (1) When opening a new empty file, that file is treated as an ordinary ** database. ** ** (2) When opening a file that begins with the standard SQLite prefix ** string "SQLite format 3", that file is treated as an ordinary ** database. ** ** (3) When opening a file that ends with the appendvfs trailer string ** "Start-Of-SQLite3-NNNNNNNN" that file is treated as an appended ** database. ** ** (4) If none of the above apply and the SQLITE_OPEN_CREATE flag is ** set, then a new database is appended to the already existing file. ** ** (5) Otherwise, SQLITE_CANTOPEN is returned. ** ** To avoid unnecessary complications with the PENDING_BYTE, the size of ** the file containing the database is limited to 1GB. This VFS will refuse ** to read or write past the 1GB mark. This restriction might be lifted in ** future versions. For now, if you need a large database, then keep the ** database in a separate file. ** ** If the file being opened is not an appended database, then this shim is ** a pass-through into the default underlying VFS. **/ #include "sqlite3ext.h" SQLITE_EXTENSION_INIT1 #include <string.h> #include <assert.h> /* The append mark at the end of the database is: ** ** Start-Of-SQLite3-NNNNNNNN ** 123456789 123456789 12345 ** ** The NNNNNNNN represents a 64-bit big-endian unsigned integer which is ** the offset to page 1. */ #define APND_MARK_PREFIX "Start-Of-SQLite3-" #define APND_MARK_PREFIX_SZ 17 #define APND_MARK_SIZE 25 /* ** Maximum size of the combined prefix + database + append-mark. This ** must be less than 0x40000000 to avoid locking issues on Windows. */ #define APND_MAX_SIZE (65536*15259) /* ** Forward declaration of objects used by this utility */ typedef struct sqlite3_vfs ApndVfs; typedef struct ApndFile ApndFile; /* Access to a lower-level VFS that (might) implement dynamic loading, ** access to randomness, etc. */ #define ORIGVFS(p) ((sqlite3_vfs*)((p)->pAppData)) #define ORIGFILE(p) ((sqlite3_file*)(((ApndFile*)(p))+1)) /* An open file */ struct ApndFile { sqlite3_file base; /* IO methods */ sqlite3_int64 iPgOne; /* File offset to page 1 */ sqlite3_int64 iMark; /* Start of the append-mark */ }; /* ** Methods for ApndFile */ static int apndClose(sqlite3_file*); static int apndRead(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst); static int apndWrite(sqlite3_file*,const void*,int iAmt, sqlite3_int64 iOfst); static int apndTruncate(sqlite3_file*, sqlite3_int64 size); static int apndSync(sqlite3_file*, int flags); static int apndFileSize(sqlite3_file*, sqlite3_int64 *pSize); static int apndLock(sqlite3_file*, int); static int apndUnlock(sqlite3_file*, int); static int apndCheckReservedLock(sqlite3_file*, int *pResOut); static int apndFileControl(sqlite3_file*, int op, void *pArg); static int apndSectorSize(sqlite3_file*); static int apndDeviceCharacteristics(sqlite3_file*); static int apndShmMap(sqlite3_file*, int iPg, int pgsz, int, void volatile**); static int apndShmLock(sqlite3_file*, int offset, int n, int flags); static void apndShmBarrier(sqlite3_file*); static int apndShmUnmap(sqlite3_file*, int deleteFlag); static int apndFetch(sqlite3_file*, sqlite3_int64 iOfst, int iAmt, void **pp); static int apndUnfetch(sqlite3_file*, sqlite3_int64 iOfst, void *p); /* ** Methods for ApndVfs */ static int apndOpen(sqlite3_vfs*, const char *, sqlite3_file*, int , int *); static int apndDelete(sqlite3_vfs*, const char *zName, int syncDir); static int apndAccess(sqlite3_vfs*, const char *zName, int flags, int *); static int apndFullPathname(sqlite3_vfs*, const char *zName, int, char *zOut); static void *apndDlOpen(sqlite3_vfs*, const char *zFilename); static void apndDlError(sqlite3_vfs*, int nByte, char *zErrMsg); static void (*apndDlSym(sqlite3_vfs *pVfs, void *p, const char*zSym))(void); static void apndDlClose(sqlite3_vfs*, void*); static int apndRandomness(sqlite3_vfs*, int nByte, char *zOut); static int apndSleep(sqlite3_vfs*, int microseconds); static int apndCurrentTime(sqlite3_vfs*, double*); static int apndGetLastError(sqlite3_vfs*, int, char *); static int apndCurrentTimeInt64(sqlite3_vfs*, sqlite3_int64*); static int apndSetSystemCall(sqlite3_vfs*, const char*,sqlite3_syscall_ptr); static sqlite3_syscall_ptr apndGetSystemCall(sqlite3_vfs*, const char *z); static const char *apndNextSystemCall(sqlite3_vfs*, const char *zName); static sqlite3_vfs apnd_vfs = { 3, /* iVersion (set when registered) */ 0, /* szOsFile (set when registered) */ 1024, /* mxPathname */ 0, /* pNext */ "apndvfs", /* zName */ 0, /* pAppData (set when registered) */ apndOpen, /* xOpen */ apndDelete, /* xDelete */ apndAccess, /* xAccess */ apndFullPathname, /* xFullPathname */ apndDlOpen, /* xDlOpen */ apndDlError, /* xDlError */ apndDlSym, /* xDlSym */ apndDlClose, /* xDlClose */ apndRandomness, /* xRandomness */ apndSleep, /* xSleep */ apndCurrentTime, /* xCurrentTime */ apndGetLastError, /* xGetLastError */ apndCurrentTimeInt64, /* xCurrentTimeInt64 */ apndSetSystemCall, /* xSetSystemCall */ apndGetSystemCall, /* xGetSystemCall */ apndNextSystemCall /* xNextSystemCall */ }; static const sqlite3_io_methods apnd_io_methods = { 3, /* iVersion */ apndClose, /* xClose */ apndRead, /* xRead */ apndWrite, /* xWrite */ apndTruncate, /* xTruncate */ apndSync, /* xSync */ apndFileSize, /* xFileSize */ apndLock, /* xLock */ apndUnlock, /* xUnlock */ apndCheckReservedLock, /* xCheckReservedLock */ apndFileControl, /* xFileControl */ apndSectorSize, /* xSectorSize */ apndDeviceCharacteristics, /* xDeviceCharacteristics */ apndShmMap, /* xShmMap */ apndShmLock, /* xShmLock */ apndShmBarrier, /* xShmBarrier */ apndShmUnmap, /* xShmUnmap */ apndFetch, /* xFetch */ apndUnfetch /* xUnfetch */ }; /* ** Close an apnd-file. */ static int apndClose(sqlite3_file *pFile){ pFile = ORIGFILE(pFile); return pFile->pMethods->xClose(pFile); } /* ** Read data from an apnd-file. */ static int apndRead( sqlite3_file *pFile, void *zBuf, int iAmt, sqlite_int64 iOfst ){ ApndFile *p = (ApndFile *)pFile; pFile = ORIGFILE(pFile); return pFile->pMethods->xRead(pFile, zBuf, iAmt, iOfst+p->iPgOne); } /* ** Add the append-mark onto the end of the file. */ static int apndWriteMark(ApndFile *p, sqlite3_file *pFile){ int i; unsigned char a[APND_MARK_SIZE]; memcpy(a, APND_MARK_PREFIX, APND_MARK_PREFIX_SZ); for(i=0; i<8; i++){ a[APND_MARK_PREFIX_SZ+i] = (p->iPgOne >> (56 - i*8)) & 0xff; } return pFile->pMethods->xWrite(pFile, a, APND_MARK_SIZE, p->iMark); } /* ** Write data to an apnd-file. */ static int apndWrite( sqlite3_file *pFile, const void *zBuf, int iAmt, sqlite_int64 iOfst ){ int rc; ApndFile *p = (ApndFile *)pFile; pFile = ORIGFILE(pFile); if( iOfst+iAmt>=APND_MAX_SIZE ) return SQLITE_FULL; rc = pFile->pMethods->xWrite(pFile, zBuf, iAmt, iOfst+p->iPgOne); if( rc==SQLITE_OK && iOfst + iAmt + p->iPgOne > p->iMark ){ sqlite3_int64 sz = 0; rc = pFile->pMethods->xFileSize(pFile, &sz); if( rc==SQLITE_OK ){ p->iMark = sz - APND_MARK_SIZE; if( iOfst + iAmt + p->iPgOne > p->iMark ){ p->iMark = p->iPgOne + iOfst + iAmt; rc = apndWriteMark(p, pFile); } } } return rc; } /* ** Truncate an apnd-file. */ static int apndTruncate(sqlite3_file *pFile, sqlite_int64 size){ int rc; ApndFile *p = (ApndFile *)pFile; pFile = ORIGFILE(pFile); rc = pFile->pMethods->xTruncate(pFile, size+p->iPgOne+APND_MARK_SIZE); if( rc==SQLITE_OK ){ p->iMark = p->iPgOne+size; rc = apndWriteMark(p, pFile); } return rc; } /* ** Sync an apnd-file. */ static int apndSync(sqlite3_file *pFile, int flags){ pFile = ORIGFILE(pFile); return pFile->pMethods->xSync(pFile, flags); } /* ** Return the current file-size of an apnd-file. */ static int apndFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){ ApndFile *p = (ApndFile *)pFile; int rc; pFile = ORIGFILE(p); rc = pFile->pMethods->xFileSize(pFile, pSize); if( rc==SQLITE_OK && p->iPgOne ){ *pSize -= p->iPgOne + APND_MARK_SIZE; } return rc; } /* ** Lock an apnd-file. */ static int apndLock(sqlite3_file *pFile, int eLock){ pFile = ORIGFILE(pFile); return pFile->pMethods->xLock(pFile, eLock); } /* ** Unlock an apnd-file. */ static int apndUnlock(sqlite3_file *pFile, int eLock){ pFile = ORIGFILE(pFile); return pFile->pMethods->xUnlock(pFile, eLock); } /* ** Check if another file-handle holds a RESERVED lock on an apnd-file. */ static int apndCheckReservedLock(sqlite3_file *pFile, int *pResOut){ pFile = ORIGFILE(pFile); return pFile->pMethods->xCheckReservedLock(pFile, pResOut); } /* ** File control method. For custom operations on an apnd-file. */ static int apndFileControl(sqlite3_file *pFile, int op, void *pArg){ ApndFile *p = (ApndFile *)pFile; int rc; pFile = ORIGFILE(pFile); rc = pFile->pMethods->xFileControl(pFile, op, pArg); if( rc==SQLITE_OK && op==SQLITE_FCNTL_VFSNAME ){ *(char**)pArg = sqlite3_mprintf("apnd(%lld)/%z", p->iPgOne, *(char**)pArg); } return rc; } /* ** Return the sector-size in bytes for an apnd-file. */ static int apndSectorSize(sqlite3_file *pFile){ pFile = ORIGFILE(pFile); return pFile->pMethods->xSectorSize(pFile); } /* ** Return the device characteristic flags supported by an apnd-file. */ static int apndDeviceCharacteristics(sqlite3_file *pFile){ pFile = ORIGFILE(pFile); return pFile->pMethods->xDeviceCharacteristics(pFile); } /* Create a shared memory file mapping */ static int apndShmMap( sqlite3_file *pFile, int iPg, int pgsz, int bExtend, void volatile **pp ){ pFile = ORIGFILE(pFile); return pFile->pMethods->xShmMap(pFile,iPg,pgsz,bExtend,pp); } /* Perform locking on a shared-memory segment */ static int apndShmLock(sqlite3_file *pFile, int offset, int n, int flags){ pFile = ORIGFILE(pFile); return pFile->pMethods->xShmLock(pFile,offset,n,flags); } /* Memory barrier operation on shared memory */ static void apndShmBarrier(sqlite3_file *pFile){ pFile = ORIGFILE(pFile); pFile->pMethods->xShmBarrier(pFile); } /* Unmap a shared memory segment */ static int apndShmUnmap(sqlite3_file *pFile, int deleteFlag){ pFile = ORIGFILE(pFile); return pFile->pMethods->xShmUnmap(pFile,deleteFlag); } /* Fetch a page of a memory-mapped file */ static int apndFetch( sqlite3_file *pFile, sqlite3_int64 iOfst, int iAmt, void **pp ){ ApndFile *p = (ApndFile *)pFile; pFile = ORIGFILE(pFile); return pFile->pMethods->xFetch(pFile, iOfst+p->iPgOne, iAmt, pp); } /* Release a memory-mapped page */ static int apndUnfetch(sqlite3_file *pFile, sqlite3_int64 iOfst, void *pPage){ ApndFile *p = (ApndFile *)pFile; pFile = ORIGFILE(pFile); return pFile->pMethods->xUnfetch(pFile, iOfst+p->iPgOne, pPage); } /* ** Check to see if the file is an ordinary SQLite database file. */ static int apndIsOrdinaryDatabaseFile(sqlite3_int64 sz, sqlite3_file *pFile){ int rc; char zHdr[16]; static const char aSqliteHdr[] = "SQLite format 3"; if( sz<512 ) return 0; rc = pFile->pMethods->xRead(pFile, zHdr, sizeof(zHdr), 0); if( rc ) return 0; return memcmp(zHdr, aSqliteHdr, sizeof(zHdr))==0; } /* ** Try to read the append-mark off the end of a file. Return the ** start of the appended database if the append-mark is present. If ** there is no append-mark, return -1; */ static sqlite3_int64 apndReadMark(sqlite3_int64 sz, sqlite3_file *pFile){ int rc, i; sqlite3_int64 iMark; unsigned char a[APND_MARK_SIZE]; if( sz<=APND_MARK_SIZE ) return -1; rc = pFile->pMethods->xRead(pFile, a, APND_MARK_SIZE, sz-APND_MARK_SIZE); if( rc ) return -1; if( memcmp(a, APND_MARK_PREFIX, APND_MARK_PREFIX_SZ)!=0 ) return -1; iMark = ((sqlite3_int64)(a[APND_MARK_PREFIX_SZ]&0x7f))<<56; for(i=1; i<8; i++){ iMark += (sqlite3_int64)a[APND_MARK_PREFIX_SZ+i]<<(56-8*i); } return iMark; } /* ** Open an apnd file handle. */ static int apndOpen( sqlite3_vfs *pVfs, const char *zName, sqlite3_file *pFile, int flags, int *pOutFlags ){ ApndFile *p; sqlite3_file *pSubFile; sqlite3_vfs *pSubVfs; int rc; sqlite3_int64 sz; pSubVfs = ORIGVFS(pVfs); if( (flags & SQLITE_OPEN_MAIN_DB)==0 ){ return pSubVfs->xOpen(pSubVfs, zName, pFile, flags, pOutFlags); } p = (ApndFile*)pFile; memset(p, 0, sizeof(*p)); pSubFile = ORIGFILE(pFile); pFile->pMethods = &apnd_io_methods; rc = pSubVfs->xOpen(pSubVfs, zName, pSubFile, flags, pOutFlags); if( rc ) goto apnd_open_done; rc = pSubFile->pMethods->xFileSize(pSubFile, &sz); if( rc ){ pSubFile->pMethods->xClose(pSubFile); goto apnd_open_done; } if( apndIsOrdinaryDatabaseFile(sz, pSubFile) ){ memmove(pFile, pSubFile, pSubVfs->szOsFile); return SQLITE_OK; } p->iMark = 0; p->iPgOne = apndReadMark(sz, pFile); if( p->iPgOne>0 ){ return SQLITE_OK; } if( (flags & SQLITE_OPEN_CREATE)==0 ){ pSubFile->pMethods->xClose(pSubFile); rc = SQLITE_CANTOPEN; } p->iPgOne = (sz+0xfff) & ~(sqlite3_int64)0xfff; apnd_open_done: if( rc ) pFile->pMethods = 0; return rc; } /* ** All other VFS methods are pass-thrus. */ static int apndDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){ return ORIGVFS(pVfs)->xDelete(ORIGVFS(pVfs), zPath, dirSync); } static int apndAccess( sqlite3_vfs *pVfs, const char *zPath, int flags, int *pResOut ){ return ORIGVFS(pVfs)->xAccess(ORIGVFS(pVfs), zPath, flags, pResOut); } static int apndFullPathname( sqlite3_vfs *pVfs, const char *zPath, int nOut, char *zOut ){ return ORIGVFS(pVfs)->xFullPathname(ORIGVFS(pVfs),zPath,nOut,zOut); } static void *apndDlOpen(sqlite3_vfs *pVfs, const char *zPath){ return ORIGVFS(pVfs)->xDlOpen(ORIGVFS(pVfs), zPath); } static void apndDlError(sqlite3_vfs *pVfs, int nByte, char *zErrMsg){ ORIGVFS(pVfs)->xDlError(ORIGVFS(pVfs), nByte, zErrMsg); } static void (*apndDlSym(sqlite3_vfs *pVfs, void *p, const char *zSym))(void){ return ORIGVFS(pVfs)->xDlSym(ORIGVFS(pVfs), p, zSym); } static void apndDlClose(sqlite3_vfs *pVfs, void *pHandle){ ORIGVFS(pVfs)->xDlClose(ORIGVFS(pVfs), pHandle); } static int apndRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){ return ORIGVFS(pVfs)->xRandomness(ORIGVFS(pVfs), nByte, zBufOut); } static int apndSleep(sqlite3_vfs *pVfs, int nMicro){ return ORIGVFS(pVfs)->xSleep(ORIGVFS(pVfs), nMicro); } static int apndCurrentTime(sqlite3_vfs *pVfs, double *pTimeOut){ return ORIGVFS(pVfs)->xCurrentTime(ORIGVFS(pVfs), pTimeOut); } static int apndGetLastError(sqlite3_vfs *pVfs, int a, char *b){ return ORIGVFS(pVfs)->xGetLastError(ORIGVFS(pVfs), a, b); } static int apndCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *p){ return ORIGVFS(pVfs)->xCurrentTimeInt64(ORIGVFS(pVfs), p); } static int apndSetSystemCall( sqlite3_vfs *pVfs, const char *zName, sqlite3_syscall_ptr pCall ){ return ORIGVFS(pVfs)->xSetSystemCall(ORIGVFS(pVfs),zName,pCall); } static sqlite3_syscall_ptr apndGetSystemCall( sqlite3_vfs *pVfs, const char *zName ){ return ORIGVFS(pVfs)->xGetSystemCall(ORIGVFS(pVfs),zName); } static const char *apndNextSystemCall(sqlite3_vfs *pVfs, const char *zName){ return ORIGVFS(pVfs)->xNextSystemCall(ORIGVFS(pVfs), zName); } #ifdef _WIN32 __declspec(dllexport) #endif /* ** This routine is called when the extension is loaded. ** Register the new VFS. */ int sqlite3_appendvfs_init( sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi ){ int rc = SQLITE_OK; sqlite3_vfs *pOrig; SQLITE_EXTENSION_INIT2(pApi); (void)pzErrMsg; (void)db; pOrig = sqlite3_vfs_find(0); apnd_vfs.iVersion = pOrig->iVersion; apnd_vfs.pAppData = pOrig; apnd_vfs.szOsFile = pOrig->szOsFile + sizeof(ApndFile); rc = sqlite3_vfs_register(&apnd_vfs, 0); #ifdef APPENDVFS_TEST if( rc==SQLITE_OK ){ rc = sqlite3_auto_extension((void(*)(void))apndvfsRegister); } #endif if( rc==SQLITE_OK ) rc = SQLITE_OK_LOAD_PERMANENTLY; return rc; }