SQLite

Check-in [41045be752]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Cleanups in the wasmfs/opfs integration but disable it in order to get the build into a known-working state before continuing with experimentation.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | fiddle-opfs
Files: files | file ages | folders
SHA3-256: 41045be752a5bd7966849638f3ca56f4905308df70f79f2cb6196ca7dce9d525
User & Date: stephan 2022-08-13 13:56:00.886
Context
2022-08-13
16:11
Add scratchpad/test app for WASMFS/OPFS running in the main window thread. Enable WASMFS by default in the library build. (check-in: ae24ac0f7d user: stephan tags: fiddle-opfs)
13:56
Cleanups in the wasmfs/opfs integration but disable it in order to get the build into a known-working state before continuing with experimentation. (check-in: 41045be752 user: stephan tags: fiddle-opfs)
13:51
Remove OPFS from the fiddle build for the time being - will re-enable once the breakage is figured out via testing with the core API. (check-in: 3bc510a614 user: stephan tags: fiddle-opfs)
Changes
Unified Diff Ignore Whitespace Patch
Changes to ext/wasm/GNUmakefile.
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139

sqlite3-api.jses := \
  $(dir.api)/sqlite3-api-prologue.js \
  $(dir.common)/whwasmutil.js \
  $(dir.jacc)/jaccwabyt.js \
  $(dir.api)/sqlite3-api-glue.js \
  $(dir.api)/sqlite3-api-oo1.js \
  $(dir.api)/sqlite3-api-worker.js \
  $(dir.api)/sqlite3-api-opfs.js \
  $(dir.api)/sqlite3-api-cleanup.js

sqlite3-api.js := $(dir.api)/sqlite3-api.js
CLEAN_FILES += $(sqlite3-api.js)
$(sqlite3-api.js): $(sqlite3-api.jses) $(MAKEFILE)
	@echo "Making $@..."
	@for i in $(sqlite3-api.jses); do \
		echo "/* BEGIN FILE: $$i */"; \







|
|
|







123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139

sqlite3-api.jses := \
  $(dir.api)/sqlite3-api-prologue.js \
  $(dir.common)/whwasmutil.js \
  $(dir.jacc)/jaccwabyt.js \
  $(dir.api)/sqlite3-api-glue.js \
  $(dir.api)/sqlite3-api-oo1.js \
  $(dir.api)/sqlite3-api-worker.js
#sqlite3-api.jses += $(dir.api)/sqlite3-api-opfs.js 
sqlite3-api.jses += $(dir.api)/sqlite3-api-cleanup.js

sqlite3-api.js := $(dir.api)/sqlite3-api.js
CLEAN_FILES += $(sqlite3-api.js)
$(sqlite3-api.js): $(sqlite3-api.jses) $(MAKEFILE)
	@echo "Making $@..."
	@for i in $(sqlite3-api.jses); do \
		echo "/* BEGIN FILE: $$i */"; \
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185


186



187
188
189
190
191
192
193
emcc.cflags += -I. -I$(dir.top) # $(SQLITE_OPT)
emcc.cflags += -pthread

########################################################################
# emcc flags specific to building the final .js/.wasm file...
emcc.jsflags := -fPIC
emcc.jsflags += --no-entry
emcc.jsflags += -sENVIRONMENT=web,worker
emcc.jsflags += -sMODULARIZE
emcc.jsflags += -sSTRICT_JS
emcc.jsflags += -sDYNAMIC_EXECUTION=0
emcc.jsflags += -sNO_POLYFILL
emcc.jsflags += -sEXPORTED_FUNCTIONS=@$(dir.wasm)/EXPORTED_FUNCTIONS.api
emcc.jsflags += -sEXPORTED_RUNTIME_METHODS=FS,wasmMemory # wasmMemory==>for -sIMPORTED_MEMORY
emcc.jsflags += -sUSE_CLOSURE_COMPILER=0
emcc.jsflags += -sIMPORTED_MEMORY


emcc.jsflags += -pthread -sWASMFS



#emcc.jsflags += -sINITIAL_MEMORY=13107200
#emcc.jsflags += -sTOTAL_STACK=4194304
emcc.jsflags += -sEXPORT_NAME=sqlite3InitModule
emcc.jsflags += -sGLOBAL_BASE=4096 # HYPOTHETICALLY keep func table indexes from overlapping w/ heap addr.
emcc.jsflags +=--post-js=$(post-js.js)
#emcc.jsflags += -sSTRICT # fails due to missing __syscall_...()
#emcc.jsflags += -sALLOW_UNIMPLEMENTED_SYSCALLS







<








>
>
|
>
>
>







170
171
172
173
174
175
176

177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
emcc.cflags += -I. -I$(dir.top) # $(SQLITE_OPT)
emcc.cflags += -pthread

########################################################################
# emcc flags specific to building the final .js/.wasm file...
emcc.jsflags := -fPIC
emcc.jsflags += --no-entry

emcc.jsflags += -sMODULARIZE
emcc.jsflags += -sSTRICT_JS
emcc.jsflags += -sDYNAMIC_EXECUTION=0
emcc.jsflags += -sNO_POLYFILL
emcc.jsflags += -sEXPORTED_FUNCTIONS=@$(dir.wasm)/EXPORTED_FUNCTIONS.api
emcc.jsflags += -sEXPORTED_RUNTIME_METHODS=FS,wasmMemory # wasmMemory==>for -sIMPORTED_MEMORY
emcc.jsflags += -sUSE_CLOSURE_COMPILER=0
emcc.jsflags += -sIMPORTED_MEMORY
emcc.environment := -sENVIRONMENT=web
ifeq (0,1)
  emcc.jsflags += -pthread -sWASMFS -sPTHREAD_POOL_SIZE=2
  emcc.environment := $(emcc.environment),worker
endif
emcc.jsflags += $(emcc.environment)
#emcc.jsflags += -sINITIAL_MEMORY=13107200
#emcc.jsflags += -sTOTAL_STACK=4194304
emcc.jsflags += -sEXPORT_NAME=sqlite3InitModule
emcc.jsflags += -sGLOBAL_BASE=4096 # HYPOTHETICALLY keep func table indexes from overlapping w/ heap addr.
emcc.jsflags +=--post-js=$(post-js.js)
#emcc.jsflags += -sSTRICT # fails due to missing __syscall_...()
#emcc.jsflags += -sALLOW_UNIMPLEMENTED_SYSCALLS
231
232
233
234
235
236
237
238

239
240
241
242



243
244
245
246
247
248
249
sqlite3.js := $(dir.api)/sqlite3.js
sqlite3.wasm := $(dir.api)/sqlite3.wasm
$(dir.api)/sqlite3-wasm.o: emcc.cflags += $(SQLITE_OPT)
$(dir.api)/sqlite3-wasm.o: $(dir.top)/sqlite3.c
$(dir.api)/wasm_util.o: emcc.cflags += $(SQLITE_OPT)
sqlite3.wasm.c := $(dir.api)/sqlite3-wasm.c \
    $(dir.jacc)/jaccwabyt_test.c
# ^^^ FIXME (how?): jaccwabyt_test.c is only needed for the test

# apps. However, we want to test the release builds with those apps,
# so we cannot simply elide that file in release builds. That
# component is critical to the VFS bindings so needs to be tested
# along with the core APIs.



define WASM_C_COMPILE
$(1).o := $$(subst .c,.o,$(1))
sqlite3.wasm.obj += $$($(1).o)
$$($(1).o): $$(MAKEFILE) $(1)
	$$(emcc.bin) $$(emcc_opt) $$(emcc.flags) $$(emcc.cflags) -c $(1) -o $$@
CLEAN_FILES += $$($(1).o)
endef







|
>
|
|
<
|
>
>
>







235
236
237
238
239
240
241
242
243
244
245

246
247
248
249
250
251
252
253
254
255
256
sqlite3.js := $(dir.api)/sqlite3.js
sqlite3.wasm := $(dir.api)/sqlite3.wasm
$(dir.api)/sqlite3-wasm.o: emcc.cflags += $(SQLITE_OPT)
$(dir.api)/sqlite3-wasm.o: $(dir.top)/sqlite3.c
$(dir.api)/wasm_util.o: emcc.cflags += $(SQLITE_OPT)
sqlite3.wasm.c := $(dir.api)/sqlite3-wasm.c \
    $(dir.jacc)/jaccwabyt_test.c
# ^^^ FIXME (how?): jaccwabyt_test.c is only needed for the test apps,
# so we don't really want to include it in release builds. However, we
# want to test the release builds with those apps, so we cannot simply
# elide that file in release builds. That component is critical to the

# VFS bindings so needs to be tested along with the core APIs.
ifneq (,$(filter -sWASMFS,$(emcc.jsflags)))
  $(dir.api)/sqlite3-wasm.o: emcc.cflags+=-DSQLITE_WASM_OPFS
endif
define WASM_C_COMPILE
$(1).o := $$(subst .c,.o,$(1))
sqlite3.wasm.obj += $$($(1).o)
$$($(1).o): $$(MAKEFILE) $(1)
	$$(emcc.bin) $$(emcc_opt) $$(emcc.flags) $$(emcc.cflags) -c $(1) -o $$@
CLEAN_FILES += $$($(1).o)
endef
Changes to ext/wasm/api/sqlite3-api-prologue.js.
571
572
573
574
575
576
577

















































578
579
580
581
582
583
584
  */
  capi.wasm.bindingSignatures.int64 = [
      ["sqlite3_bind_int64","int", ["sqlite3_stmt*", "int", "i64"]],
      ["sqlite3_changes64","i64", ["sqlite3*"]],
      ["sqlite3_column_int64","i64", ["sqlite3_stmt*", "int"]],
      ["sqlite3_total_changes64", "i64", ["sqlite3*"]]
  ];


















































  /* The remainder of the API will be set up in later steps. */
  return {
    capi,
    postInit: [
      /* some pieces of the API may install functions into this array,
         and each such function will be called, passed (self,sqlite3),







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
  */
  capi.wasm.bindingSignatures.int64 = [
      ["sqlite3_bind_int64","int", ["sqlite3_stmt*", "int", "i64"]],
      ["sqlite3_changes64","i64", ["sqlite3*"]],
      ["sqlite3_column_int64","i64", ["sqlite3_stmt*", "int"]],
      ["sqlite3_total_changes64", "i64", ["sqlite3*"]]
  ];

  /** State for sqlite3_web_persistent_dir(). */
  let __persistentDir;
  /**
     An experiment. Do not use.

     If the wasm environment has a persistent storage directory,
     its path is returned by this function. If it does not then
     it returns one of:

     - `undefined` if initIfNeeded is false and this function has
       never been called before.

     - `""` if no persistent storage is available.

     Note that in both cases the return value is falsy.
  */
  capi.sqlite3_web_persistent_dir = function(initIfNeeded=true){
    if(undefined !== __persistentDir) return __persistentDir;
    else if(!initIfNeeded) return;
    // If we have no OPFS, there is no persistent dir
    if(!self.FileSystemHandle || !self.FileSystemDirectoryHandle
       || !self.FileSystemFileHandle){
      return __persistentDir = "";
    }
    try{
      if(0===this.wasm.xCall('sqlite3_wasm_init_opfs')){
        /** OPFS does not support locking and will trigger errors if
            we try to lock. We don't _really_ want to
            _unconditionally_ install a non-locking sqlite3 VFS as the
            default, but we do so here for simplicy's sake for the
            time being. That said: locking is a no-op on all of the
            current WASM storage, so this isn't (currently) as bad as
            it may initially seem. */
        const pVfs = this.sqlite3_vfs_find("unix-none");
        if(pVfs){
          this.sqlite3_vfs_register(pVfs,1);
          //warn("Installed 'unix-none' as the default sqlite3 VFS.");
        }
        return __persistentDir =
          "/persistent" /* name is hard-coded in sqlite3_wasm_init_opfs()!*/;
      }else{
        return __persistentDir = "";
      }
    }catch(e){
      // sqlite3_wasm_init_opfs() is not available
      return __persistentDir = "";
    }
  }.bind(capi);

  /* The remainder of the API will be set up in later steps. */
  return {
    capi,
    postInit: [
      /* some pieces of the API may install functions into this array,
         and each such function will be called, passed (self,sqlite3),
Changes to ext/wasm/api/sqlite3-wasm.c.
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447


448
449
450

451
452
453
454
455










456
457
458




459
  sqlite3_vfs * const pVfs = sqlite3_vfs_find(0);
  if( zName && pVfs && pVfs->xDelete ){
    rc = pVfs->xDelete(pVfs, zName, 1);
  }
  return rc;
}

#ifdef __EMSCRIPTEN__
#include <emscripten/wasmfs.h>
#include <emscripten/console.h>
/*
** This function is NOT part of the sqlite3 public API. It is strictly
** for use by the sqlite project's own JS/WASM bindings.
**
** This function should only be called if the JS side detects the
** existence of the Origin-Private FileSystem (OPFS) APIs in the
** client. The first time it is called, this function instantiates a
** WASMFS backend impl for OPFS. On success, subsequent calls are
** no-ops.
**
** Returns 0 on success, SQLITE_NOMEM if intantiation of the backend
** object fails.


*/
int sqlite3_wasm_init_opfs(void){
  static backend_t pOpfs = 0;

  if( !pOpfs ){
    pOpfs = wasmfs_create_opfs_backend();
    if( pOpfs ){
      emscripten_console_log("Created OPFS WASMFS backend.");
    }










  }
  return pOpfs ? 0 : SQLITE_NOMEM;
}




#endif /* __EMSCRIPTEN__ */







|












|
|
>
>



>



|

>
>
>
>
>
>
>
>
>
>



>
>
>
>
|
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
  sqlite3_vfs * const pVfs = sqlite3_vfs_find(0);
  if( zName && pVfs && pVfs->xDelete ){
    rc = pVfs->xDelete(pVfs, zName, 1);
  }
  return rc;
}

#if defined(__EMSCRIPTEN__) && defined(SQLITE_WASM_OPFS)
#include <emscripten/wasmfs.h>
#include <emscripten/console.h>
/*
** This function is NOT part of the sqlite3 public API. It is strictly
** for use by the sqlite project's own JS/WASM bindings.
**
** This function should only be called if the JS side detects the
** existence of the Origin-Private FileSystem (OPFS) APIs in the
** client. The first time it is called, this function instantiates a
** WASMFS backend impl for OPFS. On success, subsequent calls are
** no-ops.
**
** Returns 0 on success, SQLITE_NOMEM if instantiation of the backend
** object fails, SQLITE_IOERR if mkdir() of the "/persistent" dir in
** the virtual FS fails. In builds compiled without SQLITE_WASM_OPFS
** defined, SQLITE_NOTFOUND is returned without side effects.
*/
int sqlite3_wasm_init_opfs(void){
  static backend_t pOpfs = 0;
  static const char * zDir = "/persistent";
  if( !pOpfs ){
    pOpfs = wasmfs_create_opfs_backend();
    if( pOpfs ){
      emscripten_console_log("Created WASMFS OPFS backend.");
    }
  }
  /** It's not enough to instantiate the backend. We have to create a
      mountpoint in the VFS and attach the backend to it. */
  if( pOpfs && 0!=access(zDir, F_OK) ){
    /* mkdir() simply hangs when called from fiddle app. Cause is
       not yet determined but the hypothesis is an init-order
       issue. */
    const int rc = wasmfs_create_directory(zDir, 0777, pOpfs);
    emscripten_console_log(rc ? "OPFS mkdir failed." : "OPFS mkdir ok.");
    if(rc) return SQLITE_IOERR;
  }
  return pOpfs ? 0 : SQLITE_NOMEM;
}
#else
int sqlite3_wasm_init_opfs(void){
  return SQLITE_NOTFOUND;
}
#endif /* __EMSCRIPTEN__ && SQLITE_WASM_OPFS */
Changes to ext/wasm/testing1.js.
15
16
17
18
19
20
21
22

23
24
25
26
27
28
29
*/
'use strict';
(function(){
  const T = self.SqliteTestUtil;
  const toss = function(...args){throw new Error(args.join(' '))};
  const debug = console.debug.bind(console);
  const eOutput = document.querySelector('#test-output');
  const log = console.log.bind(console)

  const logHtml = function(...args){
    log.apply(this, args);
    const ln = document.createElement('div');
    ln.append(document.createTextNode(args.join(' ')));
    eOutput.append(ln);
  };








|
>







15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
*/
'use strict';
(function(){
  const T = self.SqliteTestUtil;
  const toss = function(...args){throw new Error(args.join(' '))};
  const debug = console.debug.bind(console);
  const eOutput = document.querySelector('#test-output');
  const log = console.log.bind(console),
        warn = console.warn.bind(console);
  const logHtml = function(...args){
    log.apply(this, args);
    const ln = document.createElement('div');
    ln.append(document.createTextNode(args.join(' ')));
    eOutput.append(ln);
  };

1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
    //log("Module",Module);
    const sqlite3 = Module.sqlite3,
          capi = sqlite3.capi,
          oo = sqlite3.oo1,
          wasm = capi.wasm;
    log("Loaded module:",capi.sqlite3_libversion(), capi.sqlite3_sourceid());
    log("Build options:",wasm.compileOptionUsed());

    if(1){
      /* Let's grab those last few lines of test coverage for
         sqlite3-api.js... */
      const rc = wasm.compileOptionUsed(['COMPILER']);
      T.assert(1 === rc.COMPILER);
      const obj = {COMPILER:undefined};
      wasm.compileOptionUsed(obj);







|







1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
    //log("Module",Module);
    const sqlite3 = Module.sqlite3,
          capi = sqlite3.capi,
          oo = sqlite3.oo1,
          wasm = capi.wasm;
    log("Loaded module:",capi.sqlite3_libversion(), capi.sqlite3_sourceid());
    log("Build options:",wasm.compileOptionUsed());
    capi.sqlite3_web_persistent_dir()/*will install OPFS if available, plus a and non-locking VFS*/;
    if(1){
      /* Let's grab those last few lines of test coverage for
         sqlite3-api.js... */
      const rc = wasm.compileOptionUsed(['COMPILER']);
      T.assert(1 === rc.COMPILER);
      const obj = {COMPILER:undefined};
      wasm.compileOptionUsed(obj);
Changes to ext/wasm/testing2.js.
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
(function(){
  const T = self.SqliteTestUtil;
  const SW = new Worker("api/sqlite3-worker.js");
  const DbState = {
    id: undefined
  };
  const eOutput = document.querySelector('#test-output');
  const log = console.log.bind(console)
  const logHtml = function(cssClass,...args){
    log.apply(this, args);
    const ln = document.createElement('div');
    if(cssClass) ln.classList.add(cssClass);
    ln.append(document.createTextNode(args.join(' ')));
    eOutput.append(ln);
  };







|







16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
(function(){
  const T = self.SqliteTestUtil;
  const SW = new Worker("api/sqlite3-worker.js");
  const DbState = {
    id: undefined
  };
  const eOutput = document.querySelector('#test-output');
  const log = console.log.bind(console);
  const logHtml = function(cssClass,...args){
    log.apply(this, args);
    const ln = document.createElement('div');
    if(cssClass) ln.classList.add(cssClass);
    ln.append(document.createTextNode(args.join(' ')));
    eOutput.append(ln);
  };