000001 /* 000002 ** 2001 September 15 000003 ** 000004 ** The author disclaims copyright to this source code. In place of 000005 ** a legal notice, here is a blessing: 000006 ** 000007 ** May you do good and not evil. 000008 ** May you find forgiveness for yourself and forgive others. 000009 ** May you share freely, never taking more than you give. 000010 ** 000011 ************************************************************************* 000012 ** An tokenizer for SQL 000013 ** 000014 ** This file contains C code that splits an SQL input string up into 000015 ** individual tokens and sends those tokens one-by-one over to the 000016 ** parser for analysis. 000017 */ 000018 #include "sqliteInt.h" 000019 #include <stdlib.h> 000020 000021 /* Character classes for tokenizing 000022 ** 000023 ** In the sqlite3GetToken() function, a switch() on aiClass[c] is implemented 000024 ** using a lookup table, whereas a switch() directly on c uses a binary search. 000025 ** The lookup table is much faster. To maximize speed, and to ensure that 000026 ** a lookup table is used, all of the classes need to be small integers and 000027 ** all of them need to be used within the switch. 000028 */ 000029 #define CC_X 0 /* The letter 'x', or start of BLOB literal */ 000030 #define CC_KYWD 1 /* Alphabetics or '_'. Usable in a keyword */ 000031 #define CC_ID 2 /* unicode characters usable in IDs */ 000032 #define CC_DIGIT 3 /* Digits */ 000033 #define CC_DOLLAR 4 /* '$' */ 000034 #define CC_VARALPHA 5 /* '@', '#', ':'. Alphabetic SQL variables */ 000035 #define CC_VARNUM 6 /* '?'. Numeric SQL variables */ 000036 #define CC_SPACE 7 /* Space characters */ 000037 #define CC_QUOTE 8 /* '"', '\'', or '`'. String literals, quoted ids */ 000038 #define CC_QUOTE2 9 /* '['. [...] style quoted ids */ 000039 #define CC_PIPE 10 /* '|'. Bitwise OR or concatenate */ 000040 #define CC_MINUS 11 /* '-'. Minus or SQL-style comment */ 000041 #define CC_LT 12 /* '<'. Part of < or <= or <> */ 000042 #define CC_GT 13 /* '>'. Part of > or >= */ 000043 #define CC_EQ 14 /* '='. Part of = or == */ 000044 #define CC_BANG 15 /* '!'. Part of != */ 000045 #define CC_SLASH 16 /* '/'. / or c-style comment */ 000046 #define CC_LP 17 /* '(' */ 000047 #define CC_RP 18 /* ')' */ 000048 #define CC_SEMI 19 /* ';' */ 000049 #define CC_PLUS 20 /* '+' */ 000050 #define CC_STAR 21 /* '*' */ 000051 #define CC_PERCENT 22 /* '%' */ 000052 #define CC_COMMA 23 /* ',' */ 000053 #define CC_AND 24 /* '&' */ 000054 #define CC_TILDA 25 /* '~' */ 000055 #define CC_DOT 26 /* '.' */ 000056 #define CC_ILLEGAL 27 /* Illegal character */ 000057 #define CC_NUL 28 /* 0x00 */ 000058 000059 static const unsigned char aiClass[] = { 000060 #ifdef SQLITE_ASCII 000061 /* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xa xb xc xd xe xf */ 000062 /* 0x */ 28, 27, 27, 27, 27, 27, 27, 27, 27, 7, 7, 27, 7, 7, 27, 27, 000063 /* 1x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 000064 /* 2x */ 7, 15, 8, 5, 4, 22, 24, 8, 17, 18, 21, 20, 23, 11, 26, 16, 000065 /* 3x */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 5, 19, 12, 14, 13, 6, 000066 /* 4x */ 5, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 000067 /* 5x */ 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 9, 27, 27, 27, 1, 000068 /* 6x */ 8, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 000069 /* 7x */ 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 27, 10, 27, 25, 27, 000070 /* 8x */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 000071 /* 9x */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 000072 /* Ax */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 000073 /* Bx */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 000074 /* Cx */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 000075 /* Dx */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 000076 /* Ex */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 000077 /* Fx */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 000078 #endif 000079 #ifdef SQLITE_EBCDIC 000080 /* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xa xb xc xd xe xf */ 000081 /* 0x */ 27, 27, 27, 27, 27, 7, 27, 27, 27, 27, 27, 27, 7, 7, 27, 27, 000082 /* 1x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 000083 /* 2x */ 27, 27, 27, 27, 27, 7, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 000084 /* 3x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 000085 /* 4x */ 7, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 26, 12, 17, 20, 10, 000086 /* 5x */ 24, 27, 27, 27, 27, 27, 27, 27, 27, 27, 15, 4, 21, 18, 19, 27, 000087 /* 6x */ 11, 16, 27, 27, 27, 27, 27, 27, 27, 27, 27, 23, 22, 1, 13, 6, 000088 /* 7x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 8, 5, 5, 5, 8, 14, 8, 000089 /* 8x */ 27, 1, 1, 1, 1, 1, 1, 1, 1, 1, 27, 27, 27, 27, 27, 27, 000090 /* 9x */ 27, 1, 1, 1, 1, 1, 1, 1, 1, 1, 27, 27, 27, 27, 27, 27, 000091 /* Ax */ 27, 25, 1, 1, 1, 1, 1, 0, 1, 1, 27, 27, 27, 27, 27, 27, 000092 /* Bx */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 9, 27, 27, 27, 27, 27, 000093 /* Cx */ 27, 1, 1, 1, 1, 1, 1, 1, 1, 1, 27, 27, 27, 27, 27, 27, 000094 /* Dx */ 27, 1, 1, 1, 1, 1, 1, 1, 1, 1, 27, 27, 27, 27, 27, 27, 000095 /* Ex */ 27, 27, 1, 1, 1, 1, 1, 0, 1, 1, 27, 27, 27, 27, 27, 27, 000096 /* Fx */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 27, 27, 27, 27, 27, 27, 000097 #endif 000098 }; 000099 000100 /* 000101 ** The charMap() macro maps alphabetic characters (only) into their 000102 ** lower-case ASCII equivalent. On ASCII machines, this is just 000103 ** an upper-to-lower case map. On EBCDIC machines we also need 000104 ** to adjust the encoding. The mapping is only valid for alphabetics 000105 ** which are the only characters for which this feature is used. 000106 ** 000107 ** Used by keywordhash.h 000108 */ 000109 #ifdef SQLITE_ASCII 000110 # define charMap(X) sqlite3UpperToLower[(unsigned char)X] 000111 #endif 000112 #ifdef SQLITE_EBCDIC 000113 # define charMap(X) ebcdicToAscii[(unsigned char)X] 000114 const unsigned char ebcdicToAscii[] = { 000115 /* 0 1 2 3 4 5 6 7 8 9 A B C D E F */ 000116 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x */ 000117 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1x */ 000118 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */ 000119 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 3x */ 000120 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 4x */ 000121 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 5x */ 000122 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, /* 6x */ 000123 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 7x */ 000124 0, 97, 98, 99,100,101,102,103,104,105, 0, 0, 0, 0, 0, 0, /* 8x */ 000125 0,106,107,108,109,110,111,112,113,114, 0, 0, 0, 0, 0, 0, /* 9x */ 000126 0, 0,115,116,117,118,119,120,121,122, 0, 0, 0, 0, 0, 0, /* Ax */ 000127 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* Bx */ 000128 0, 97, 98, 99,100,101,102,103,104,105, 0, 0, 0, 0, 0, 0, /* Cx */ 000129 0,106,107,108,109,110,111,112,113,114, 0, 0, 0, 0, 0, 0, /* Dx */ 000130 0, 0,115,116,117,118,119,120,121,122, 0, 0, 0, 0, 0, 0, /* Ex */ 000131 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* Fx */ 000132 }; 000133 #endif 000134 000135 /* 000136 ** The sqlite3KeywordCode function looks up an identifier to determine if 000137 ** it is a keyword. If it is a keyword, the token code of that keyword is 000138 ** returned. If the input is not a keyword, TK_ID is returned. 000139 ** 000140 ** The implementation of this routine was generated by a program, 000141 ** mkkeywordhash.c, located in the tool subdirectory of the distribution. 000142 ** The output of the mkkeywordhash.c program is written into a file 000143 ** named keywordhash.h and then included into this source file by 000144 ** the #include below. 000145 */ 000146 #include "keywordhash.h" 000147 000148 000149 /* 000150 ** If X is a character that can be used in an identifier then 000151 ** IdChar(X) will be true. Otherwise it is false. 000152 ** 000153 ** For ASCII, any character with the high-order bit set is 000154 ** allowed in an identifier. For 7-bit characters, 000155 ** sqlite3IsIdChar[X] must be 1. 000156 ** 000157 ** For EBCDIC, the rules are more complex but have the same 000158 ** end result. 000159 ** 000160 ** Ticket #1066. the SQL standard does not allow '$' in the 000161 ** middle of identifiers. But many SQL implementations do. 000162 ** SQLite will allow '$' in identifiers for compatibility. 000163 ** But the feature is undocumented. 000164 */ 000165 #ifdef SQLITE_ASCII 000166 #define IdChar(C) ((sqlite3CtypeMap[(unsigned char)C]&0x46)!=0) 000167 #endif 000168 #ifdef SQLITE_EBCDIC 000169 const char sqlite3IsEbcdicIdChar[] = { 000170 /* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */ 000171 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 4x */ 000172 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 0, 0, /* 5x */ 000173 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, /* 6x */ 000174 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, /* 7x */ 000175 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 0, /* 8x */ 000176 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 0, /* 9x */ 000177 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 0, /* Ax */ 000178 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* Bx */ 000179 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, /* Cx */ 000180 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, /* Dx */ 000181 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, /* Ex */ 000182 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0, /* Fx */ 000183 }; 000184 #define IdChar(C) (((c=C)>=0x42 && sqlite3IsEbcdicIdChar[c-0x40])) 000185 #endif 000186 000187 /* Make the IdChar function accessible from ctime.c and alter.c */ 000188 int sqlite3IsIdChar(u8 c){ return IdChar(c); } 000189 000190 #ifndef SQLITE_OMIT_WINDOWFUNC 000191 /* 000192 ** Return the id of the next token in string (*pz). Before returning, set 000193 ** (*pz) to point to the byte following the parsed token. 000194 */ 000195 static int getToken(const unsigned char **pz){ 000196 const unsigned char *z = *pz; 000197 int t; /* Token type to return */ 000198 do { 000199 z += sqlite3GetToken(z, &t); 000200 }while( t==TK_SPACE ); 000201 if( t==TK_ID 000202 || t==TK_STRING 000203 || t==TK_JOIN_KW 000204 || t==TK_WINDOW 000205 || t==TK_OVER 000206 || sqlite3ParserFallback(t)==TK_ID 000207 ){ 000208 t = TK_ID; 000209 } 000210 *pz = z; 000211 return t; 000212 } 000213 000214 /* 000215 ** The following three functions are called immediately after the tokenizer 000216 ** reads the keywords WINDOW, OVER and FILTER, respectively, to determine 000217 ** whether the token should be treated as a keyword or an SQL identifier. 000218 ** This cannot be handled by the usual lemon %fallback method, due to 000219 ** the ambiguity in some constructions. e.g. 000220 ** 000221 ** SELECT sum(x) OVER ... 000222 ** 000223 ** In the above, "OVER" might be a keyword, or it might be an alias for the 000224 ** sum(x) expression. If a "%fallback ID OVER" directive were added to 000225 ** grammar, then SQLite would always treat "OVER" as an alias, making it 000226 ** impossible to call a window-function without a FILTER clause. 000227 ** 000228 ** WINDOW is treated as a keyword if: 000229 ** 000230 ** * the following token is an identifier, or a keyword that can fallback 000231 ** to being an identifier, and 000232 ** * the token after than one is TK_AS. 000233 ** 000234 ** OVER is a keyword if: 000235 ** 000236 ** * the previous token was TK_RP, and 000237 ** * the next token is either TK_LP or an identifier. 000238 ** 000239 ** FILTER is a keyword if: 000240 ** 000241 ** * the previous token was TK_RP, and 000242 ** * the next token is TK_LP. 000243 */ 000244 static int analyzeWindowKeyword(const unsigned char *z){ 000245 int t; 000246 t = getToken(&z); 000247 if( t!=TK_ID ) return TK_ID; 000248 t = getToken(&z); 000249 if( t!=TK_AS ) return TK_ID; 000250 return TK_WINDOW; 000251 } 000252 static int analyzeOverKeyword(const unsigned char *z, int lastToken){ 000253 if( lastToken==TK_RP ){ 000254 int t = getToken(&z); 000255 if( t==TK_LP || t==TK_ID ) return TK_OVER; 000256 } 000257 return TK_ID; 000258 } 000259 static int analyzeFilterKeyword(const unsigned char *z, int lastToken){ 000260 if( lastToken==TK_RP && getToken(&z)==TK_LP ){ 000261 return TK_FILTER; 000262 } 000263 return TK_ID; 000264 } 000265 #endif /* SQLITE_OMIT_WINDOWFUNC */ 000266 000267 /* 000268 ** Return the length (in bytes) of the token that begins at z[0]. 000269 ** Store the token type in *tokenType before returning. 000270 */ 000271 int sqlite3GetToken(const unsigned char *z, int *tokenType){ 000272 int i, c; 000273 switch( aiClass[*z] ){ /* Switch on the character-class of the first byte 000274 ** of the token. See the comment on the CC_ defines 000275 ** above. */ 000276 case CC_SPACE: { 000277 testcase( z[0]==' ' ); 000278 testcase( z[0]=='\t' ); 000279 testcase( z[0]=='\n' ); 000280 testcase( z[0]=='\f' ); 000281 testcase( z[0]=='\r' ); 000282 for(i=1; sqlite3Isspace(z[i]); i++){} 000283 *tokenType = TK_SPACE; 000284 return i; 000285 } 000286 case CC_MINUS: { 000287 if( z[1]=='-' ){ 000288 for(i=2; (c=z[i])!=0 && c!='\n'; i++){} 000289 *tokenType = TK_SPACE; /* IMP: R-22934-25134 */ 000290 return i; 000291 } 000292 *tokenType = TK_MINUS; 000293 return 1; 000294 } 000295 case CC_LP: { 000296 *tokenType = TK_LP; 000297 return 1; 000298 } 000299 case CC_RP: { 000300 *tokenType = TK_RP; 000301 return 1; 000302 } 000303 case CC_SEMI: { 000304 *tokenType = TK_SEMI; 000305 return 1; 000306 } 000307 case CC_PLUS: { 000308 *tokenType = TK_PLUS; 000309 return 1; 000310 } 000311 case CC_STAR: { 000312 *tokenType = TK_STAR; 000313 return 1; 000314 } 000315 case CC_SLASH: { 000316 if( z[1]!='*' || z[2]==0 ){ 000317 *tokenType = TK_SLASH; 000318 return 1; 000319 } 000320 for(i=3, c=z[2]; (c!='*' || z[i]!='/') && (c=z[i])!=0; i++){} 000321 if( c ) i++; 000322 *tokenType = TK_SPACE; /* IMP: R-22934-25134 */ 000323 return i; 000324 } 000325 case CC_PERCENT: { 000326 *tokenType = TK_REM; 000327 return 1; 000328 } 000329 case CC_EQ: { 000330 *tokenType = TK_EQ; 000331 return 1 + (z[1]=='='); 000332 } 000333 case CC_LT: { 000334 if( (c=z[1])=='=' ){ 000335 *tokenType = TK_LE; 000336 return 2; 000337 }else if( c=='>' ){ 000338 *tokenType = TK_NE; 000339 return 2; 000340 }else if( c=='<' ){ 000341 *tokenType = TK_LSHIFT; 000342 return 2; 000343 }else{ 000344 *tokenType = TK_LT; 000345 return 1; 000346 } 000347 } 000348 case CC_GT: { 000349 if( (c=z[1])=='=' ){ 000350 *tokenType = TK_GE; 000351 return 2; 000352 }else if( c=='>' ){ 000353 *tokenType = TK_RSHIFT; 000354 return 2; 000355 }else{ 000356 *tokenType = TK_GT; 000357 return 1; 000358 } 000359 } 000360 case CC_BANG: { 000361 if( z[1]!='=' ){ 000362 *tokenType = TK_ILLEGAL; 000363 return 1; 000364 }else{ 000365 *tokenType = TK_NE; 000366 return 2; 000367 } 000368 } 000369 case CC_PIPE: { 000370 if( z[1]!='|' ){ 000371 *tokenType = TK_BITOR; 000372 return 1; 000373 }else{ 000374 *tokenType = TK_CONCAT; 000375 return 2; 000376 } 000377 } 000378 case CC_COMMA: { 000379 *tokenType = TK_COMMA; 000380 return 1; 000381 } 000382 case CC_AND: { 000383 *tokenType = TK_BITAND; 000384 return 1; 000385 } 000386 case CC_TILDA: { 000387 *tokenType = TK_BITNOT; 000388 return 1; 000389 } 000390 case CC_QUOTE: { 000391 int delim = z[0]; 000392 testcase( delim=='`' ); 000393 testcase( delim=='\'' ); 000394 testcase( delim=='"' ); 000395 for(i=1; (c=z[i])!=0; i++){ 000396 if( c==delim ){ 000397 if( z[i+1]==delim ){ 000398 i++; 000399 }else{ 000400 break; 000401 } 000402 } 000403 } 000404 if( c=='\'' ){ 000405 *tokenType = TK_STRING; 000406 return i+1; 000407 }else if( c!=0 ){ 000408 *tokenType = TK_ID; 000409 return i+1; 000410 }else{ 000411 *tokenType = TK_ILLEGAL; 000412 return i; 000413 } 000414 } 000415 case CC_DOT: { 000416 #ifndef SQLITE_OMIT_FLOATING_POINT 000417 if( !sqlite3Isdigit(z[1]) ) 000418 #endif 000419 { 000420 *tokenType = TK_DOT; 000421 return 1; 000422 } 000423 /* If the next character is a digit, this is a floating point 000424 ** number that begins with ".". Fall thru into the next case */ 000425 } 000426 case CC_DIGIT: { 000427 testcase( z[0]=='0' ); testcase( z[0]=='1' ); testcase( z[0]=='2' ); 000428 testcase( z[0]=='3' ); testcase( z[0]=='4' ); testcase( z[0]=='5' ); 000429 testcase( z[0]=='6' ); testcase( z[0]=='7' ); testcase( z[0]=='8' ); 000430 testcase( z[0]=='9' ); 000431 *tokenType = TK_INTEGER; 000432 #ifndef SQLITE_OMIT_HEX_INTEGER 000433 if( z[0]=='0' && (z[1]=='x' || z[1]=='X') && sqlite3Isxdigit(z[2]) ){ 000434 for(i=3; sqlite3Isxdigit(z[i]); i++){} 000435 return i; 000436 } 000437 #endif 000438 for(i=0; sqlite3Isdigit(z[i]); i++){} 000439 #ifndef SQLITE_OMIT_FLOATING_POINT 000440 if( z[i]=='.' ){ 000441 i++; 000442 while( sqlite3Isdigit(z[i]) ){ i++; } 000443 *tokenType = TK_FLOAT; 000444 } 000445 if( (z[i]=='e' || z[i]=='E') && 000446 ( sqlite3Isdigit(z[i+1]) 000447 || ((z[i+1]=='+' || z[i+1]=='-') && sqlite3Isdigit(z[i+2])) 000448 ) 000449 ){ 000450 i += 2; 000451 while( sqlite3Isdigit(z[i]) ){ i++; } 000452 *tokenType = TK_FLOAT; 000453 } 000454 #endif 000455 while( IdChar(z[i]) ){ 000456 *tokenType = TK_ILLEGAL; 000457 i++; 000458 } 000459 return i; 000460 } 000461 case CC_QUOTE2: { 000462 for(i=1, c=z[0]; c!=']' && (c=z[i])!=0; i++){} 000463 *tokenType = c==']' ? TK_ID : TK_ILLEGAL; 000464 return i; 000465 } 000466 case CC_VARNUM: { 000467 *tokenType = TK_VARIABLE; 000468 for(i=1; sqlite3Isdigit(z[i]); i++){} 000469 return i; 000470 } 000471 case CC_DOLLAR: 000472 case CC_VARALPHA: { 000473 int n = 0; 000474 testcase( z[0]=='$' ); testcase( z[0]=='@' ); 000475 testcase( z[0]==':' ); testcase( z[0]=='#' ); 000476 *tokenType = TK_VARIABLE; 000477 for(i=1; (c=z[i])!=0; i++){ 000478 if( IdChar(c) ){ 000479 n++; 000480 #ifndef SQLITE_OMIT_TCL_VARIABLE 000481 }else if( c=='(' && n>0 ){ 000482 do{ 000483 i++; 000484 }while( (c=z[i])!=0 && !sqlite3Isspace(c) && c!=')' ); 000485 if( c==')' ){ 000486 i++; 000487 }else{ 000488 *tokenType = TK_ILLEGAL; 000489 } 000490 break; 000491 }else if( c==':' && z[i+1]==':' ){ 000492 i++; 000493 #endif 000494 }else{ 000495 break; 000496 } 000497 } 000498 if( n==0 ) *tokenType = TK_ILLEGAL; 000499 return i; 000500 } 000501 case CC_KYWD: { 000502 for(i=1; aiClass[z[i]]<=CC_KYWD; i++){} 000503 if( IdChar(z[i]) ){ 000504 /* This token started out using characters that can appear in keywords, 000505 ** but z[i] is a character not allowed within keywords, so this must 000506 ** be an identifier instead */ 000507 i++; 000508 break; 000509 } 000510 *tokenType = TK_ID; 000511 return keywordCode((char*)z, i, tokenType); 000512 } 000513 case CC_X: { 000514 #ifndef SQLITE_OMIT_BLOB_LITERAL 000515 testcase( z[0]=='x' ); testcase( z[0]=='X' ); 000516 if( z[1]=='\'' ){ 000517 *tokenType = TK_BLOB; 000518 for(i=2; sqlite3Isxdigit(z[i]); i++){} 000519 if( z[i]!='\'' || i%2 ){ 000520 *tokenType = TK_ILLEGAL; 000521 while( z[i] && z[i]!='\'' ){ i++; } 000522 } 000523 if( z[i] ) i++; 000524 return i; 000525 } 000526 #endif 000527 /* If it is not a BLOB literal, then it must be an ID, since no 000528 ** SQL keywords start with the letter 'x'. Fall through */ 000529 } 000530 case CC_ID: { 000531 i = 1; 000532 break; 000533 } 000534 case CC_NUL: { 000535 *tokenType = TK_ILLEGAL; 000536 return 0; 000537 } 000538 default: { 000539 *tokenType = TK_ILLEGAL; 000540 return 1; 000541 } 000542 } 000543 while( IdChar(z[i]) ){ i++; } 000544 *tokenType = TK_ID; 000545 return i; 000546 } 000547 000548 /* 000549 ** Run the parser on the given SQL string. The parser structure is 000550 ** passed in. An SQLITE_ status code is returned. If an error occurs 000551 ** then an and attempt is made to write an error message into 000552 ** memory obtained from sqlite3_malloc() and to make *pzErrMsg point to that 000553 ** error message. 000554 */ 000555 int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){ 000556 int nErr = 0; /* Number of errors encountered */ 000557 void *pEngine; /* The LEMON-generated LALR(1) parser */ 000558 int n = 0; /* Length of the next token token */ 000559 int tokenType; /* type of the next token */ 000560 int lastTokenParsed = -1; /* type of the previous token */ 000561 sqlite3 *db = pParse->db; /* The database connection */ 000562 int mxSqlLen; /* Max length of an SQL string */ 000563 #ifdef sqlite3Parser_ENGINEALWAYSONSTACK 000564 yyParser sEngine; /* Space to hold the Lemon-generated Parser object */ 000565 #endif 000566 VVA_ONLY( u8 startedWithOom = db->mallocFailed ); 000567 000568 assert( zSql!=0 ); 000569 mxSqlLen = db->aLimit[SQLITE_LIMIT_SQL_LENGTH]; 000570 if( db->nVdbeActive==0 ){ 000571 db->u1.isInterrupted = 0; 000572 } 000573 pParse->rc = SQLITE_OK; 000574 pParse->zTail = zSql; 000575 assert( pzErrMsg!=0 ); 000576 #ifdef SQLITE_DEBUG 000577 if( db->flags & SQLITE_ParserTrace ){ 000578 printf("parser: [[[%s]]]\n", zSql); 000579 sqlite3ParserTrace(stdout, "parser: "); 000580 }else{ 000581 sqlite3ParserTrace(0, 0); 000582 } 000583 #endif 000584 #ifdef sqlite3Parser_ENGINEALWAYSONSTACK 000585 pEngine = &sEngine; 000586 sqlite3ParserInit(pEngine, pParse); 000587 #else 000588 pEngine = sqlite3ParserAlloc(sqlite3Malloc, pParse); 000589 if( pEngine==0 ){ 000590 sqlite3OomFault(db); 000591 return SQLITE_NOMEM_BKPT; 000592 } 000593 #endif 000594 assert( pParse->pNewTable==0 ); 000595 assert( pParse->pNewTrigger==0 ); 000596 assert( pParse->nVar==0 ); 000597 assert( pParse->pVList==0 ); 000598 pParse->pParentParse = db->pParse; 000599 db->pParse = pParse; 000600 while( 1 ){ 000601 n = sqlite3GetToken((u8*)zSql, &tokenType); 000602 mxSqlLen -= n; 000603 if( mxSqlLen<0 ){ 000604 pParse->rc = SQLITE_TOOBIG; 000605 break; 000606 } 000607 #ifndef SQLITE_OMIT_WINDOWFUNC 000608 if( tokenType>=TK_WINDOW ){ 000609 assert( tokenType==TK_SPACE || tokenType==TK_OVER || tokenType==TK_FILTER 000610 || tokenType==TK_ILLEGAL || tokenType==TK_WINDOW 000611 ); 000612 #else 000613 if( tokenType>=TK_SPACE ){ 000614 assert( tokenType==TK_SPACE || tokenType==TK_ILLEGAL ); 000615 #endif /* SQLITE_OMIT_WINDOWFUNC */ 000616 if( db->u1.isInterrupted ){ 000617 pParse->rc = SQLITE_INTERRUPT; 000618 break; 000619 } 000620 if( tokenType==TK_SPACE ){ 000621 zSql += n; 000622 continue; 000623 } 000624 if( zSql[0]==0 ){ 000625 /* Upon reaching the end of input, call the parser two more times 000626 ** with tokens TK_SEMI and 0, in that order. */ 000627 if( lastTokenParsed==TK_SEMI ){ 000628 tokenType = 0; 000629 }else if( lastTokenParsed==0 ){ 000630 break; 000631 }else{ 000632 tokenType = TK_SEMI; 000633 } 000634 n = 0; 000635 #ifndef SQLITE_OMIT_WINDOWFUNC 000636 }else if( tokenType==TK_WINDOW ){ 000637 assert( n==6 ); 000638 tokenType = analyzeWindowKeyword((const u8*)&zSql[6]); 000639 }else if( tokenType==TK_OVER ){ 000640 assert( n==4 ); 000641 tokenType = analyzeOverKeyword((const u8*)&zSql[4], lastTokenParsed); 000642 }else if( tokenType==TK_FILTER ){ 000643 assert( n==6 ); 000644 tokenType = analyzeFilterKeyword((const u8*)&zSql[6], lastTokenParsed); 000645 #endif /* SQLITE_OMIT_WINDOWFUNC */ 000646 }else{ 000647 sqlite3ErrorMsg(pParse, "unrecognized token: \"%.*s\"", n, zSql); 000648 break; 000649 } 000650 } 000651 pParse->sLastToken.z = zSql; 000652 pParse->sLastToken.n = n; 000653 sqlite3Parser(pEngine, tokenType, pParse->sLastToken); 000654 lastTokenParsed = tokenType; 000655 zSql += n; 000656 assert( db->mallocFailed==0 || pParse->rc!=SQLITE_OK || startedWithOom ); 000657 if( pParse->rc!=SQLITE_OK ) break; 000658 } 000659 assert( nErr==0 ); 000660 #ifdef YYTRACKMAXSTACKDEPTH 000661 sqlite3_mutex_enter(sqlite3MallocMutex()); 000662 sqlite3StatusHighwater(SQLITE_STATUS_PARSER_STACK, 000663 sqlite3ParserStackPeak(pEngine) 000664 ); 000665 sqlite3_mutex_leave(sqlite3MallocMutex()); 000666 #endif /* YYDEBUG */ 000667 #ifdef sqlite3Parser_ENGINEALWAYSONSTACK 000668 sqlite3ParserFinalize(pEngine); 000669 #else 000670 sqlite3ParserFree(pEngine, sqlite3_free); 000671 #endif 000672 if( db->mallocFailed ){ 000673 pParse->rc = SQLITE_NOMEM_BKPT; 000674 } 000675 if( pParse->rc!=SQLITE_OK && pParse->rc!=SQLITE_DONE && pParse->zErrMsg==0 ){ 000676 pParse->zErrMsg = sqlite3MPrintf(db, "%s", sqlite3ErrStr(pParse->rc)); 000677 } 000678 assert( pzErrMsg!=0 ); 000679 if( pParse->zErrMsg ){ 000680 *pzErrMsg = pParse->zErrMsg; 000681 sqlite3_log(pParse->rc, "%s in \"%s\"", 000682 *pzErrMsg, pParse->zTail); 000683 pParse->zErrMsg = 0; 000684 nErr++; 000685 } 000686 pParse->zTail = zSql; 000687 if( pParse->pVdbe && pParse->nErr>0 && pParse->nested==0 ){ 000688 sqlite3VdbeDelete(pParse->pVdbe); 000689 pParse->pVdbe = 0; 000690 } 000691 #ifndef SQLITE_OMIT_SHARED_CACHE 000692 if( pParse->nested==0 ){ 000693 sqlite3DbFree(db, pParse->aTableLock); 000694 pParse->aTableLock = 0; 000695 pParse->nTableLock = 0; 000696 } 000697 #endif 000698 #ifndef SQLITE_OMIT_VIRTUALTABLE 000699 sqlite3_free(pParse->apVtabLock); 000700 #endif 000701 000702 if( !IN_SPECIAL_PARSE ){ 000703 /* If the pParse->declareVtab flag is set, do not delete any table 000704 ** structure built up in pParse->pNewTable. The calling code (see vtab.c) 000705 ** will take responsibility for freeing the Table structure. 000706 */ 000707 sqlite3DeleteTable(db, pParse->pNewTable); 000708 } 000709 if( !IN_RENAME_OBJECT ){ 000710 sqlite3DeleteTrigger(db, pParse->pNewTrigger); 000711 } 000712 000713 if( pParse->pWithToFree ) sqlite3WithDelete(db, pParse->pWithToFree); 000714 sqlite3DbFree(db, pParse->pVList); 000715 while( pParse->pAinc ){ 000716 AutoincInfo *p = pParse->pAinc; 000717 pParse->pAinc = p->pNext; 000718 sqlite3DbFreeNN(db, p); 000719 } 000720 while( pParse->pZombieTab ){ 000721 Table *p = pParse->pZombieTab; 000722 pParse->pZombieTab = p->pNextZombie; 000723 sqlite3DeleteTable(db, p); 000724 } 000725 db->pParse = pParse->pParentParse; 000726 pParse->pParentParse = 0; 000727 assert( nErr==0 || pParse->rc!=SQLITE_OK ); 000728 return nErr; 000729 } 000730 000731 000732 #ifdef SQLITE_ENABLE_NORMALIZE 000733 /* 000734 ** Insert a single space character into pStr if the current string 000735 ** ends with an identifier 000736 */ 000737 static void addSpaceSeparator(sqlite3_str *pStr){ 000738 if( pStr->nChar && sqlite3IsIdChar(pStr->zText[pStr->nChar-1]) ){ 000739 sqlite3_str_append(pStr, " ", 1); 000740 } 000741 } 000742 000743 /* 000744 ** Compute a normalization of the SQL given by zSql[0..nSql-1]. Return 000745 ** the normalization in space obtained from sqlite3DbMalloc(). Or return 000746 ** NULL if anything goes wrong or if zSql is NULL. 000747 */ 000748 char *sqlite3Normalize( 000749 Vdbe *pVdbe, /* VM being reprepared */ 000750 const char *zSql /* The original SQL string */ 000751 ){ 000752 sqlite3 *db; /* The database connection */ 000753 int i; /* Next unread byte of zSql[] */ 000754 int n; /* length of current token */ 000755 int tokenType; /* type of current token */ 000756 int prevType = 0; /* Previous non-whitespace token */ 000757 int nParen; /* Number of nested levels of parentheses */ 000758 int iStartIN; /* Start of RHS of IN operator in z[] */ 000759 int nParenAtIN; /* Value of nParent at start of RHS of IN operator */ 000760 u32 j; /* Bytes of normalized SQL generated so far */ 000761 sqlite3_str *pStr; /* The normalized SQL string under construction */ 000762 000763 db = sqlite3VdbeDb(pVdbe); 000764 tokenType = -1; 000765 nParen = iStartIN = nParenAtIN = 0; 000766 pStr = sqlite3_str_new(db); 000767 assert( pStr!=0 ); /* sqlite3_str_new() never returns NULL */ 000768 for(i=0; zSql[i] && pStr->accError==0; i+=n){ 000769 if( tokenType!=TK_SPACE ){ 000770 prevType = tokenType; 000771 } 000772 n = sqlite3GetToken((unsigned char*)zSql+i, &tokenType); 000773 if( NEVER(n<=0) ) break; 000774 switch( tokenType ){ 000775 case TK_SPACE: { 000776 break; 000777 } 000778 case TK_NULL: { 000779 if( prevType==TK_IS || prevType==TK_NOT ){ 000780 sqlite3_str_append(pStr, " NULL", 5); 000781 break; 000782 } 000783 /* Fall through */ 000784 } 000785 case TK_STRING: 000786 case TK_INTEGER: 000787 case TK_FLOAT: 000788 case TK_VARIABLE: 000789 case TK_BLOB: { 000790 sqlite3_str_append(pStr, "?", 1); 000791 break; 000792 } 000793 case TK_LP: { 000794 nParen++; 000795 if( prevType==TK_IN ){ 000796 iStartIN = pStr->nChar; 000797 nParenAtIN = nParen; 000798 } 000799 sqlite3_str_append(pStr, "(", 1); 000800 break; 000801 } 000802 case TK_RP: { 000803 if( iStartIN>0 && nParen==nParenAtIN ){ 000804 assert( pStr->nChar>=(u32)iStartIN ); 000805 pStr->nChar = iStartIN+1; 000806 sqlite3_str_append(pStr, "?,?,?", 5); 000807 iStartIN = 0; 000808 } 000809 nParen--; 000810 sqlite3_str_append(pStr, ")", 1); 000811 break; 000812 } 000813 case TK_ID: { 000814 iStartIN = 0; 000815 j = pStr->nChar; 000816 if( sqlite3Isquote(zSql[i]) ){ 000817 char *zId = sqlite3DbStrNDup(db, zSql+i, n); 000818 int nId; 000819 int eType = 0; 000820 if( zId==0 ) break; 000821 sqlite3Dequote(zId); 000822 if( zSql[i]=='"' && sqlite3VdbeUsesDoubleQuotedString(pVdbe, zId) ){ 000823 sqlite3_str_append(pStr, "?", 1); 000824 sqlite3DbFree(db, zId); 000825 break; 000826 } 000827 nId = sqlite3Strlen30(zId); 000828 if( sqlite3GetToken((u8*)zId, &eType)==nId && eType==TK_ID ){ 000829 addSpaceSeparator(pStr); 000830 sqlite3_str_append(pStr, zId, nId); 000831 }else{ 000832 sqlite3_str_appendf(pStr, "\"%w\"", zId); 000833 } 000834 sqlite3DbFree(db, zId); 000835 }else{ 000836 addSpaceSeparator(pStr); 000837 sqlite3_str_append(pStr, zSql+i, n); 000838 } 000839 while( j<pStr->nChar ){ 000840 pStr->zText[j] = sqlite3Tolower(pStr->zText[j]); 000841 j++; 000842 } 000843 break; 000844 } 000845 case TK_SELECT: { 000846 iStartIN = 0; 000847 /* fall through */ 000848 } 000849 default: { 000850 if( sqlite3IsIdChar(zSql[i]) ) addSpaceSeparator(pStr); 000851 j = pStr->nChar; 000852 sqlite3_str_append(pStr, zSql+i, n); 000853 while( j<pStr->nChar ){ 000854 pStr->zText[j] = sqlite3Toupper(pStr->zText[j]); 000855 j++; 000856 } 000857 break; 000858 } 000859 } 000860 } 000861 if( tokenType!=TK_SEMI ) sqlite3_str_append(pStr, ";", 1); 000862 return sqlite3_str_finish(pStr); 000863 } 000864 #endif /* SQLITE_ENABLE_NORMALIZE */