SQLite

Check-in [ae24ac0f7d]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add scratchpad/test app for WASMFS/OPFS running in the main window thread. Enable WASMFS by default in the library build.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | fiddle-opfs
Files: files | file ages | folders
SHA3-256: ae24ac0f7dd9e12a40de0f6ccd61a16f010804da454085f886c217cc600cdba4
User & Date: stephan 2022-08-13 16:11:38.873
References
2022-08-13
17:13
Add worker-style variant of the tests added in [ae24ac0f7dd9], but building this with wasmfs causes them to throw inexplicable exceptions from the Emscripten glue (without wasmfs it builds and runs fine, but storage is not persistent). (check-in: 6401595e59 user: stephan tags: fiddle-opfs)
Context
2022-08-13
16:36
wasmfs: use unix-none VFS by default to avoid locking errors in non-OPFS mode. (check-in: 75561dea1a user: stephan tags: fiddle-opfs)
16:11
Add scratchpad/test app for WASMFS/OPFS running in the main window thread. Enable WASMFS by default in the library build. (check-in: ae24ac0f7d user: stephan tags: fiddle-opfs)
13:56
Cleanups in the wasmfs/opfs integration but disable it in order to get the build into a known-working state before continuing with experimentation. (check-in: 41045be752 user: stephan tags: fiddle-opfs)
Changes
Unified Diff Ignore Whitespace Patch
Changes to ext/wasm/GNUmakefile.
︙︙
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185

186

187
188





189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238


########################################################################
# emcc flags for .c/.o.
emcc.cflags :=
emcc.cflags += -std=c99 -fPIC
# -------------^^^^^^^^ we currently need c99 for WASM-specific sqlite3 APIs.
emcc.cflags += -I. -I$(dir.top) # $(SQLITE_OPT)
emcc.cflags += -pthread

########################################################################
# emcc flags specific to building the final .js/.wasm file...
emcc.jsflags := -fPIC
emcc.jsflags += --no-entry
emcc.jsflags += -sMODULARIZE
emcc.jsflags += -sSTRICT_JS
emcc.jsflags += -sDYNAMIC_EXECUTION=0
emcc.jsflags += -sNO_POLYFILL
emcc.jsflags += -sEXPORTED_FUNCTIONS=@$(dir.wasm)/EXPORTED_FUNCTIONS.api
emcc.jsflags += -sEXPORTED_RUNTIME_METHODS=FS,wasmMemory # wasmMemory==>for -sIMPORTED_MEMORY
emcc.jsflags += -sUSE_CLOSURE_COMPILER=0
emcc.jsflags += -sIMPORTED_MEMORY
emcc.environment := -sENVIRONMENT=web

ifeq (0,1)

  emcc.jsflags += -pthread -sWASMFS -sPTHREAD_POOL_SIZE=2
  emcc.environment := $(emcc.environment),worker





endif
emcc.jsflags += $(emcc.environment)
#emcc.jsflags += -sINITIAL_MEMORY=13107200
#emcc.jsflags += -sTOTAL_STACK=4194304
emcc.jsflags += -sEXPORT_NAME=sqlite3InitModule
emcc.jsflags += -sGLOBAL_BASE=4096 # HYPOTHETICALLY keep func table indexes from overlapping w/ heap addr.
emcc.jsflags +=--post-js=$(post-js.js)
#emcc.jsflags += -sSTRICT # fails due to missing __syscall_...()
#emcc.jsflags += -sALLOW_UNIMPLEMENTED_SYSCALLS
#emcc.jsflags += -sFILESYSTEM=0 # only for experimentation. sqlite3 needs the FS API
#emcc.jsflags += -sABORTING_MALLOC
emcc.jsflags += -sALLOW_MEMORY_GROWTH
emcc.jsflags += -sALLOW_TABLE_GROWTH
emcc.jsflags += -Wno-limited-postlink-optimizations
# ^^^^^ it likes to warn when we have "limited optimizations" via the -g3 flag.
#emcc.jsflags += -sMALLOC=emmalloc
#emcc.jsflags += -sMALLOC=dlmalloc # a good 8k larger than emmalloc
#emcc.jsflags += -sSTANDALONE_WASM # causes OOM errors, not sure why
#emcc.jsflags += --import=foo_bar
#emcc.jsflags += --no-gc-sections
# https://lld.llvm.org/WebAssembly.html
emcc.jsflags += -sERROR_ON_UNDEFINED_SYMBOLS=0
emcc.jsflags += -sLLD_REPORT_UNDEFINED
#emcc.jsflags += --allow-undefined
emcc.jsflags += --import-undefined
#emcc.jsflags += --unresolved-symbols=import-dynamic --experimental-pic
#emcc.jsflags +=  --experimental-pic --unresolved-symbols=ingore-all --import-undefined
#emcc.jsflags += --unresolved-symbols=ignore-all
enable_bigint ?= 1
ifneq (0,$(enable_bigint))
emcc.jsflags += -sWASM_BIGINT
endif
emcc.jsflags += -sMEMORY64=0
# ^^^^ MEMORY64=1 fails to load, erroring with:
#  invalid memory limits flags 0x5
#    (enable via --experimental-wasm-memory64)
#
# ^^^^ MEMORY64=2 builds and loads but dies when we do things like:
#
#  new Uint8Array(heapWrappers().HEAP8U.buffer, ptr, n)
#
# because ptr is now a BigInt, so is invalid for passing to arguments
# which have strict must-be-a-number requirements.
########################################################################


sqlite3.js := $(dir.api)/sqlite3.js
sqlite3.wasm := $(dir.api)/sqlite3.wasm
$(dir.api)/sqlite3-wasm.o: emcc.cflags += $(SQLITE_OPT)
$(dir.api)/sqlite3-wasm.o: $(dir.top)/sqlite3.c








<














>
|
>


>
>
>
>
>






|




<



<
<

<
<






|















|








164
165
166
167
168
169
170

171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205

206
207
208


209


210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239


########################################################################
# emcc flags for .c/.o.
emcc.cflags :=
emcc.cflags += -std=c99 -fPIC
# -------------^^^^^^^^ we currently need c99 for WASM-specific sqlite3 APIs.
emcc.cflags += -I. -I$(dir.top) # $(SQLITE_OPT)


########################################################################
# emcc flags specific to building the final .js/.wasm file...
emcc.jsflags := -fPIC
emcc.jsflags += --no-entry
emcc.jsflags += -sMODULARIZE
emcc.jsflags += -sSTRICT_JS
emcc.jsflags += -sDYNAMIC_EXECUTION=0
emcc.jsflags += -sNO_POLYFILL
emcc.jsflags += -sEXPORTED_FUNCTIONS=@$(dir.wasm)/EXPORTED_FUNCTIONS.api
emcc.jsflags += -sEXPORTED_RUNTIME_METHODS=FS,wasmMemory # wasmMemory==>for -sIMPORTED_MEMORY
emcc.jsflags += -sUSE_CLOSURE_COMPILER=0
emcc.jsflags += -sIMPORTED_MEMORY
emcc.environment := -sENVIRONMENT=web
ENABLE_WASMFS := 1
ifneq (0,$(ENABLE_WASMFS))
  emcc.cflags += -pthread
  emcc.jsflags += -pthread -sWASMFS -sPTHREAD_POOL_SIZE=2
  emcc.environment := $(emcc.environment),worker
else
  emcc.jsflags += -sALLOW_MEMORY_GROWTH
  # emcc: warning: USE_PTHREADS + ALLOW_MEMORY_GROWTH may run non-wasm code
  #       slowly, see https://github.com/WebAssembly/design/issues/1271
  #       [-Wpthreads-mem-growth]
endif
emcc.jsflags += $(emcc.environment)
#emcc.jsflags += -sINITIAL_MEMORY=13107200
#emcc.jsflags += -sTOTAL_STACK=4194304
emcc.jsflags += -sEXPORT_NAME=sqlite3InitModule
emcc.jsflags += -sGLOBAL_BASE=4096 # HYPOTHETICALLY keep func table indexes from overlapping w/ heap addr.
emcc.jsflags += --post-js=$(post-js.js)
#emcc.jsflags += -sSTRICT # fails due to missing __syscall_...()
#emcc.jsflags += -sALLOW_UNIMPLEMENTED_SYSCALLS
#emcc.jsflags += -sFILESYSTEM=0 # only for experimentation. sqlite3 needs the FS API
#emcc.jsflags += -sABORTING_MALLOC

emcc.jsflags += -sALLOW_TABLE_GROWTH
emcc.jsflags += -Wno-limited-postlink-optimizations
# ^^^^^ it likes to warn when we have "limited optimizations" via the -g3 flag.


#emcc.jsflags += -sSTANDALONE_WASM # causes OOM errors, not sure why


# https://lld.llvm.org/WebAssembly.html
emcc.jsflags += -sERROR_ON_UNDEFINED_SYMBOLS=0
emcc.jsflags += -sLLD_REPORT_UNDEFINED
#emcc.jsflags += --allow-undefined
emcc.jsflags += --import-undefined
#emcc.jsflags += --unresolved-symbols=import-dynamic --experimental-pic
#emcc.jsflags += --experimental-pic --unresolved-symbols=ingore-all --import-undefined
#emcc.jsflags += --unresolved-symbols=ignore-all
enable_bigint ?= 1
ifneq (0,$(enable_bigint))
emcc.jsflags += -sWASM_BIGINT
endif
emcc.jsflags += -sMEMORY64=0
# ^^^^ MEMORY64=1 fails to load, erroring with:
#  invalid memory limits flags 0x5
#    (enable via --experimental-wasm-memory64)
#
# ^^^^ MEMORY64=2 builds and loads but dies when we do things like:
#
#  new Uint8Array(heapWrappers().HEAP8U.buffer, ptr, n)
#
# because ptr is now a BigInt, so is invalid for passing to arguments
# which have strict must-be-a-Number requirements.
########################################################################


sqlite3.js := $(dir.api)/sqlite3.js
sqlite3.wasm := $(dir.api)/sqlite3.wasm
$(dir.api)/sqlite3-wasm.o: emcc.cflags += $(SQLITE_OPT)
$(dir.api)/sqlite3-wasm.o: $(dir.top)/sqlite3.c
︙︙
Added ext/wasm/scratchpad-opfs-main.html. 


















































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40

<!doctype html>
<html lang="en-us">
  <head>
    <meta charset="utf-8">
    <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
    <link rel="shortcut icon" href="data:image/x-icon;," type="image/x-icon">
    <link rel="stylesheet" href="common/emscripten.css"/>
    <link rel="stylesheet" href="common/testing.css"/>
    <title>sqlite3 WASMFS/OPFS Main-thread Scratchpad</title>
  </head>
  <body>
    <header id='titlebar'><span>sqlite3 WASMFS/OPFS Main-thread Scratchpad</span></header>
    <!-- emscripten bits -->
    <figure id="module-spinner">
      <div class="spinner"></div>
      <div class='center'><strong>Initializing app...</strong></div>
      <div class='center'>
        On a slow internet connection this may take a moment.  If this
        message displays for "a long time", intialization may have
        failed and the JavaScript console may contain clues as to why.
      </div>
    </figure>
    <div class="emscripten" id="module-status">Downloading...</div>
    <div class="emscripten">
      <progress value="0" max="100" id="module-progress" hidden='1'></progress>  
    </div><!-- /emscripten bits -->
    <p>Scratchpad/test app for the WASMF/OPFS integration in the
      main window thread. This page requires that the sqlite3 API have
      been built with WASMFS support. If OPFS support is available then
      it "should" persist a database across reloads (watch the dev console
      output), otherwise it will not.
    </p>
    <p>All stuff on this page happens in the dev console.</p>
    <hr>
    <div id='test-output'></div>
    <script src="api/sqlite3.js"></script>
    <script src="common/SqliteTestUtil.js"></script>
    <script src="scratchpad-opfs-main.js"></script>
  </body>
</html>
Added ext/wasm/scratchpad-opfs-main.js. 




















































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73

/*
  2022-05-22

  The author disclaims copyright to this source code.  In place of a
  legal notice, here is a blessing:

  *   May you do good and not evil.
  *   May you find forgiveness for yourself and forgive others.
  *   May you share freely, never taking more than you give.

  ***********************************************************************

  A basic test script for sqlite3-api.js. This file must be run in
  main JS thread and sqlite3.js must have been loaded before it.
*/
'use strict';
(function(){
  const toss = function(...args){throw new Error(args.join(' '))};
  const log = console.log.bind(console),
        warn = console.warn.bind(console),
        error = console.error.bind(console);

  const stdout = log;
  const stderr = error;

  const test1 = function(db){
    db.execMulti("create table if not exists t(a);")
      .callInTransaction(function(db){
        db.prepare("insert into t(a) values(?)")
          .bind(new Date().getTime())
          .stepFinalize();
        stdout("Number of values in table t:",
            db.selectValue("select count(*) from t"));
      });
  };

  const runTests = function(Module){
    //stdout("Module",Module);
    self._MODULE = Module /* this is only to facilitate testing from the console */;
    const sqlite3 = Module.sqlite3,
          capi = sqlite3.capi,
          oo = sqlite3.oo1,
          wasm = capi.wasm;
    stdout("Loaded sqlite3:",capi.sqlite3_libversion(), capi.sqlite3_sourceid());
    const persistentDir = capi.sqlite3_web_persistent_dir();
    if(persistentDir){
      stdout("Persistent storage dir:",persistentDir);
    }else{
      stderr("No persistent storage available.");
    }
    const startTime = performance.now();
    let db;
    try {
      db = new oo.DB(persistentDir+'/foo.db');
      stdout("DB filename:",db.filename,db.fileName());
      const banner1 = '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>',
            banner2 = '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<';
      [
        test1
      ].forEach((f)=>{
        const n = performance.now();
        stdout(banner1,"Running",f.name+"()...");
        f(db, sqlite3, Module);
        stdout(banner2,f.name+"() took ",(performance.now() - n),"ms");
      });
    }finally{
      if(db) db.close();
    }
    stdout("Total test time:",(performance.now() - startTime),"ms");
  };

  sqlite3InitModule(self.sqlite3TestModule).then(runTests);
})();