SQLite

Check-in [f2e590700d]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix an issue with fts5 synonyms and NEAR(...) queries.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | fts5-incompatible
Files: files | file ages | folders
SHA1: f2e590700d8eb30da8ac3a92f6ca92a2bbe9c1fe
User & Date: dan 2015-09-02 18:56:01.484
Context
2015-09-02
19:48
Further tests to raise coverage of fts5 synonym code to 100%. Fix a dropped error code in the same. (check-in: bdedd838bb user: dan tags: fts5-incompatible)
18:56
Fix an issue with fts5 synonyms and NEAR(...) queries. (check-in: f2e590700d user: dan tags: fts5-incompatible)
17:34
Fix a problem with fts5 synonyms and phrase queries. Also fix an OOM handling bug in fts5. (check-in: a4c35fa2c9 user: dan tags: fts5-incompatible)
Changes
Unified Diff Ignore Whitespace Patch
Changes to ext/fts5/fts5_expr.c.
︙︙
738
739
740
741
742
743
744
745
746
747
748
749
750

751
752
753
754
755
756
757
758
759
760
761

762
763
764
765
766
767
768
769
770
771
772
773
774

  return 0;
}

static int fts5ExprSynonymAdvanceto(
  Fts5ExprTerm *pTerm,            /* Term iterator to advance */
  int bDesc,                      /* True if iterator is "rowid DESC" */
  i64 *piLast,                    /* IN/OUT: Lastest rowid seen so far */
  int *pRc,                       /* OUT: Error code */
  int *pbEof                      /* OUT: Set to true if EOF */
){
  int rc = SQLITE_OK;
  i64 iLast = *piLast;
  Fts5ExprTerm *p;


  for(p=pTerm; rc==SQLITE_OK && p; p=p->pSynonym){
    if( sqlite3Fts5IterEof(p->pIter)==0 ){
      i64 iRowid = sqlite3Fts5IterRowid(p->pIter);
      if( (bDesc==0 && iLast>iRowid) || (bDesc && iLast<iRowid) ){
        rc = sqlite3Fts5IterNextFrom(p->pIter, iLast);
      }
    }
  }

  if( rc!=SQLITE_OK ){

    *pbEof = 1;
  }else{
    *piLast = fts5ExprSynonymRowid(pTerm, bDesc, pbEof);
  }

  return rc;
}

/*
** IN/OUT parameter (*pa) points to a position list n bytes in size. If
** the position list contains entries for column iCol, then (*pa) is set
** to point to the sub-position-list for that column and the number of
** bytes in it returned. Or, if the argument position list does not








|
<




>











>
|

|

<
|








738
739
740
741
742
743
744
745

746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766

767
768
769
770
771
772
773
774

  return 0;
}

static int fts5ExprSynonymAdvanceto(
  Fts5ExprTerm *pTerm,            /* Term iterator to advance */
  int bDesc,                      /* True if iterator is "rowid DESC" */
  i64 *piLast,                    /* IN/OUT: Lastest rowid seen so far */
  int *pRc                        /* OUT: Error code */

){
  int rc = SQLITE_OK;
  i64 iLast = *piLast;
  Fts5ExprTerm *p;
  int bEof = 0;

  for(p=pTerm; rc==SQLITE_OK && p; p=p->pSynonym){
    if( sqlite3Fts5IterEof(p->pIter)==0 ){
      i64 iRowid = sqlite3Fts5IterRowid(p->pIter);
      if( (bDesc==0 && iLast>iRowid) || (bDesc && iLast<iRowid) ){
        rc = sqlite3Fts5IterNextFrom(p->pIter, iLast);
      }
    }
  }

  if( rc!=SQLITE_OK ){
    *pRc = rc;
    bEof = 1;
  }else{
    *piLast = fts5ExprSynonymRowid(pTerm, bDesc, &bEof);
  }

  return bEof;
}

/*
** IN/OUT parameter (*pa) points to a position list n bytes in size. If
** the position list contains entries for column iCol, then (*pa) is set
** to point to the sub-position-list for that column and the number of
** bytes in it returned. Or, if the argument position list does not
︙︙
948
949
950
951
952
953
954
955

956
957
958
959
960
961
962
963
964
965

        Fts5ExprTerm *pTerm = &pPhrase->aTerm[j];
        if( pTerm->pSynonym ){
          Fts5ExprTerm *p;
          int bEof = 1;
          i64 iRowid = fts5ExprSynonymRowid(pTerm, bDesc, 0);
          if( iRowid==iLast ) continue;
          bMatch = 0;
          if( fts5ExprSynonymAdvanceto(pTerm,bDesc,&iLast,&rc,&pNode->bEof) ){

            return rc;
          }

        }else{
          Fts5IndexIter *pIter = pPhrase->aTerm[j].pIter;
          i64 iRowid = sqlite3Fts5IterRowid(pIter);
          if( iRowid==iLast ) continue;
          bMatch = 0;
          if( fts5ExprAdvanceto(pIter, bDesc, &iLast, &rc, &pNode->bEof) ){
            return rc;








|
>


<








948
949
950
951
952
953
954
955
956
957
958

959
960
961
962
963
964
965

        Fts5ExprTerm *pTerm = &pPhrase->aTerm[j];
        if( pTerm->pSynonym ){
          Fts5ExprTerm *p;
          int bEof = 1;
          i64 iRowid = fts5ExprSynonymRowid(pTerm, bDesc, 0);
          if( iRowid==iLast ) continue;
          bMatch = 0;
          if( fts5ExprSynonymAdvanceto(pTerm, bDesc, &iLast, &rc) ){
            pNode->bEof = 1;
            return rc;
          }

        }else{
          Fts5IndexIter *pIter = pPhrase->aTerm[j].pIter;
          i64 iRowid = sqlite3Fts5IterRowid(pIter);
          if( iRowid==iLast ) continue;
          bMatch = 0;
          if( fts5ExprAdvanceto(pIter, bDesc, &iLast, &rc, &pNode->bEof) ){
            return rc;
︙︙
Changes to ext/fts5/test/fts5synonym.test.
︙︙
329
330
331
332
333
334
335









336
337
338
339
340
341
342
343

      for {set i 2} {$i<=10} {incr i} {
        sqlite3_fts5_token -colo [string repeat $w $i] $iStart $iEnd
      }
    }
  }
}










do_test 6.0 {
  execsql { 
    CREATE VIRTUAL TABLE t2 USING fts5(a, b, tokenize=tcl)
  }
  foreach {rowid a b} {
    1 {yyyy vvvvv qq oo yyyyyy vvvv eee} {ffff uu r qq aaaa}
    2 {ww oooooo bbbbb ssssss mm} {ffffff yy iiii rr s ccc qqqqq}
    3 {zzzz llll gggggg cccc uu} {hhhhhh aaaa ppppp rr ee jjjj}








>
>
>
>
>
>
>
>
>
|








329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352

      for {set i 2} {$i<=10} {incr i} {
        sqlite3_fts5_token -colo [string repeat $w $i] $iStart $iEnd
      }
    }
  }
}

do_execsql_test 6.0.1 {
  CREATE VIRTUAL TABLE t1 USING fts5(x, tokenize=tcl);
  INSERT INTO t1 VALUES('yy xx qq');
  INSERT INTO t1 VALUES('yy xx xx');
}
do_execsql_test 6.0.2 {
  SELECT * FROM t1 WHERE t1 MATCH 'NEAR(y q)';
} {{yy xx qq}}

do_test 6.0.3 {
  execsql { 
    CREATE VIRTUAL TABLE t2 USING fts5(a, b, tokenize=tcl)
  }
  foreach {rowid a b} {
    1 {yyyy vvvvv qq oo yyyyyy vvvv eee} {ffff uu r qq aaaa}
    2 {ww oooooo bbbbb ssssss mm} {ffffff yy iiii rr s ccc qqqqq}
    3 {zzzz llll gggggg cccc uu} {hhhhhh aaaa ppppp rr ee jjjj}
︙︙
368
369
370
371
372
373
374





375
376
377
378
379
380
381

  3 {o OR (q AND a)} {
    1 {yyyy vvvvv [qq] [oo] yyyyyy vvvv eee} {ffff uu r [qq] [aaaa]}
    2 {ww [oooooo] bbbbb ssssss mm} {ffffff yy iiii rr s ccc qqqqq}
    5 {fffff mm vvvv [ooo] ffffff kkkk tttt} {cccccc bb e zzz d n}
    6 {iii dddd hh [qqqq] ddd [ooo]} {ttt d c b [aaaaaa] [qqqq]}
    9 {kkkkk qqq [oooo] e tttttt mmm} {e ss qqqqqq hhhh llllll gg}
  }





} {
  do_execsql_test 6.1.$tn {
    SELECT rowid, highlight(t2, 0, '[', ']'), highlight(t2, 1, '[', ']')
    FROM t2 WHERE t2 MATCH $q
  } $res
}









>
>
>
>
>








377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395

  3 {o OR (q AND a)} {
    1 {yyyy vvvvv [qq] [oo] yyyyyy vvvv eee} {ffff uu r [qq] [aaaa]}
    2 {ww [oooooo] bbbbb ssssss mm} {ffffff yy iiii rr s ccc qqqqq}
    5 {fffff mm vvvv [ooo] ffffff kkkk tttt} {cccccc bb e zzz d n}
    6 {iii dddd hh [qqqq] ddd [ooo]} {ttt d c b [aaaaaa] [qqqq]}
    9 {kkkkk qqq [oooo] e tttttt mmm} {e ss qqqqqq hhhh llllll gg}
  }

  4 {NEAR(q y, 20)} {
    1 {[yyyy] vvvvv [qq] oo [yyyyyy] vvvv eee} {ffff uu r qq aaaa}
    2 {ww oooooo bbbbb ssssss mm} {ffffff [yy] iiii rr s ccc [qqqqq]}
  }
} {
  do_execsql_test 6.1.$tn {
    SELECT rowid, highlight(t2, 0, '[', ']'), highlight(t2, 1, '[', ']')
    FROM t2 WHERE t2 MATCH $q
  } $res
}
︙︙